@dcl/protocol 1.0.0-2731649607.commit-890a990 → 1.0.0-27358757907.commit-3c70e8a

package/protoc-gen-bitwise/generator_csharp.py

@@ -0,0 +1,176 @@
+"""
+C# code generator for the protoc-gen-bitwise plugin.
+
+For every proto message that contains at least one uint32 field annotated with
+[(quantized)], this module emits a C# partial class that adds a computed float
+property named {FieldName}Quantized.  The getter decodes the stored uint32 to
+a float; the setter encodes a float back to a uint32.  Standard protobuf
+handles serialization of the uint32 wire field; this class adds a typed float
+accessor on top.
+
+Protobuf encodes small uint32 values via varint, so a value that fits in
+2^20 costs 3 bytes on the wire — cheaper than a raw IEEE 754 float (4 bytes).
+
+Only uint32 fields are supported.  bit_packed and unannotated fields are
+passed through without generating any accessor.
+"""
+
+from google.protobuf import descriptor_pb2
+
+from options_pb2 import get_field_options
+
+# FieldDescriptorProto type constants (aliased for readability)
+_FT = descriptor_pb2.FieldDescriptorProto
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _snake_to_pascal(name: str) -> str:
+    """position_x → PositionX"""
+    return ''.join(word.capitalize() for word in name.split('_'))
+
+
+def _package_to_namespace(package: str) -> str:
+    """decentraland.kernel.comms.v3 → Decentraland.Kernel.Comms.V3"""
+    if not package:
+        return 'Generated'
+    return '.'.join(part.capitalize() for part in package.split('.'))
+
+
+def _format_float(value: float) -> str:
+    """Format a Python float as a C# float literal (e.g. -100.0f)."""
+    text = f'{value:.8g}'
+    if '.' not in text and 'e' not in text and 'E' not in text:
+        text += '.0'
+    return text + 'f'
+
+
+def _format_step(step: float) -> str:
+    """Format a quantization step size for display in a doc comment (e.g. ≈ 0.003)."""
+    return f'\u2248 {step:.6g}'
+
+
+# ---------------------------------------------------------------------------
+# Per-message code generation
+# ---------------------------------------------------------------------------
+
+def _gen_message(msg_proto, indent: str = '    ') -> list[str] | None:
+    """
+    Generate a C# partial class for a proto message.
+
+    For each uint32 field with a [(quantized)] annotation, emits a cached float
+    property {FieldName}Quantized backed by the raw uint32 field.
+
+    Returns a list of lines (without trailing newline) or None if the message
+    has no quantized uint32 fields.
+    """
+    props: list[tuple[str, str, str, int, float]] = []  # (prop_name, mn, mx, bits, step)
+
+    for field in msg_proto.field:
+        # Repeated/map fields are not supported
+        if field.label == _FT.LABEL_REPEATED:
+            continue
+
+        # Only uint32 fields are candidates for quantized accessors
+        if field.type != _FT.TYPE_UINT32:
+            continue
+
+        quantized, _ = get_field_options(field.options)
+        if quantized is None:
+            continue
+
+        step = (quantized.max - quantized.min) / ((1 << quantized.bits) - 1)
+
+        props.append((
+            _snake_to_pascal(field.name),
+            _format_float(quantized.min),
+            _format_float(quantized.max),
+            quantized.bits,
+            step,
+        ))
+
+    if not props:
+        return None
+
+    i = indent
+    backings: list[str] = []
+    lines: list[str] = []
+    lines.append(f'public partial class {msg_proto.name}')
+    lines.append('{')
+
+    for prop_name, mn, mx, bits, step in props:
+        backing = '_' + prop_name[0].lower() + prop_name[1:]
+        backings.append(backing)
+        lines.append(f'{i}private float? {backing};')
+        lines.append(f'{i}/// <summary>Float accessor for <see cref="{prop_name}"/>. Range [{mn}, {mx}], {bits} bits, step {_format_step(step)}.</summary>')
+        lines.append(f'{i}public float {prop_name}Quantized')
+        lines.append(f'{i}{{')
+        lines.append(f'{i}{i}get => {backing} ??= Quantize.Decode({prop_name}, {mn}, {mx}, {bits});')
+        lines.append(f'{i}{i}set {{ {backing} = value; {prop_name} = Quantize.Encode(value, {mn}, {mx}, {bits}); }}')
+        lines.append(f'{i}}}')
+        lines.append('')
+
+    lines.append(f'{i}/// <summary>Clears all cached decoded values. Call after mutating raw uint32 fields directly.</summary>')
+    lines.append(f'{i}public void ResetDecodedCache()')
+    lines.append(f'{i}{{')
+    for backing in backings:
+        lines.append(f'{i}{i}{backing} = null;')
+    lines.append(f'{i}}}')
+
+    lines.append('}')
+    return lines
+
+
+# ---------------------------------------------------------------------------
+# Per-file code generation (public entry point)
+# ---------------------------------------------------------------------------
+
+def generate_csharp(file_proto) -> dict | None:
+    """
+    Generate a C# source file for a FileDescriptorProto.
+
+    Returns a dict with keys 'name' (output path) and 'content' (C# source),
+    or None if the file contains no quantized uint32 fields.
+    """
+    namespace = _package_to_namespace(file_proto.package)
+
+    # Output is placed flat in the output root, matching --csharp_out convention.
+    # e.g. "decentraland/kernel/comms/v3/my_message.proto"
+    #    → "MyMessage.Bitwise.cs"
+    proto_file = file_proto.name.rsplit('/', 1)[-1]
+    stem = _snake_to_pascal(proto_file.replace('.proto', ''))
+    out_name = f'{stem}.Bitwise.cs'
+
+    header = [
+        '// <auto-generated>',
+        '//   Generated by protoc-gen-bitwise. DO NOT EDIT.',
+        f'//   Source: {file_proto.name}',
+        '// </auto-generated>',
+        '',
+        'using Decentraland.Networking.Bitwise;',
+        '',
+        f'namespace {namespace}',
+        '{',
+    ]
+    footer = [
+        '',
+        f'}} // namespace {namespace}',
+    ]
+
+    body: list[str] = []
+    for msg in file_proto.message_type:
+        msg_lines = _gen_message(msg)
+        if msg_lines is None:
+            continue
+        # Indent each line by 4 spaces (inside the namespace block)
+        for line in msg_lines:
+            body.append(('    ' + line) if line else '')
+        body.append('')
+
+    if not body:
+        return None  # nothing to emit
+
+    content = '\n'.join(header + body + footer) + '\n'
+    return {'name': out_name, 'content': content}

package/protoc-gen-bitwise/options_pb2.py

@@ -0,0 +1,171 @@
+"""
+Manual parser for the custom bitwise field options defined in options.proto.
+
+Rather than relying on protobuf extension registration (which requires a properly
+compiled _pb2 module), this module parses the raw serialized FieldOptions bytes
+directly using the protobuf binary wire format.  All protobuf runtimes preserve
+unknown/unregistered extension bytes when round-tripping, so
+field.options.SerializeToString() always contains the extension data even when
+the extension is not registered in the Python runtime.
+
+Wire format reference:
+  tag  = (field_number << 3) | wire_type
+  wire_type 0 = varint, 1 = 64-bit, 2 = length-delimited, 5 = 32-bit
+"""
+
+import struct
+
+# Extension field numbers as defined in options.proto
+QUANTIZED_FIELD_NUMBER = 50001
+BIT_PACKED_FIELD_NUMBER = 50002
+
+
+# ---------------------------------------------------------------------------
+# Low-level wire-format helpers
+# ---------------------------------------------------------------------------
+
+def _read_varint(data: bytes, pos: int):
+    """Decode a protobuf varint starting at *pos*. Returns (value, new_pos)."""
+    result = 0
+    shift = 0
+    while pos < len(data):
+        byte = data[pos]
+        pos += 1
+        result |= (byte & 0x7F) << shift
+        if not (byte & 0x80):
+            break
+        shift += 7
+    return result, pos
+
+
+def _read_float32(data: bytes, pos: int):
+    """Decode a little-endian 32-bit float. Returns (value, new_pos)."""
+    value, = struct.unpack_from('<f', data, pos)
+    return value, pos + 4
+
+
+def _skip_field(data: bytes, pos: int, wire_type: int) -> int:
+    """Advance *pos* past a field with the given wire_type."""
+    if wire_type == 0:
+        _, pos = _read_varint(data, pos)
+    elif wire_type == 1:
+        pos += 8
+    elif wire_type == 2:
+        length, pos = _read_varint(data, pos)
+        pos += length
+    elif wire_type == 5:
+        pos += 4
+    # wire types 3 and 4 (start/end group) are deprecated; skip gracefully
+    return pos
+
+
+# ---------------------------------------------------------------------------
+# Option message classes
+# ---------------------------------------------------------------------------
+
+class QuantizedFloatOptions:
+    """Mirrors the QuantizedFloatOptions proto message."""
+
+    __slots__ = ('min', 'max', 'bits')
+
+    def __init__(self, min_val: float = 0.0, max_val: float = 0.0, bits: int = 0):
+        self.min = min_val
+        self.max = max_val
+        self.bits = bits
+
+    @classmethod
+    def from_bytes(cls, data: bytes) -> 'QuantizedFloatOptions':
+        obj = cls()
+        pos = 0
+        while pos < len(data):
+            tag, pos = _read_varint(data, pos)
+            field_num = tag >> 3
+            wire_type = tag & 0x7
+            if field_num == 1 and wire_type == 5:    # min  (float)
+                obj.min, pos = _read_float32(data, pos)
+            elif field_num == 2 and wire_type == 5:  # max  (float)
+                obj.max, pos = _read_float32(data, pos)
+            elif field_num == 3 and wire_type == 0:  # bits (uint32)
+                obj.bits, pos = _read_varint(data, pos)
+            else:
+                pos = _skip_field(data, pos, wire_type)
+        return obj
+
+    def __repr__(self):
+        return f'QuantizedFloatOptions(min={self.min}, max={self.max}, bits={self.bits})'
+
+
+class BitPackedOptions:
+    """Mirrors the BitPackedOptions proto message."""
+
+    __slots__ = ('bits',)
+
+    def __init__(self, bits: int = 0):
+        self.bits = bits
+
+    @classmethod
+    def from_bytes(cls, data: bytes) -> 'BitPackedOptions':
+        obj = cls()
+        pos = 0
+        while pos < len(data):
+            tag, pos = _read_varint(data, pos)
+            field_num = tag >> 3
+            wire_type = tag & 0x7
+            if field_num == 1 and wire_type == 0:    # bits (uint32)
+                obj.bits, pos = _read_varint(data, pos)
+            else:
+                pos = _skip_field(data, pos, wire_type)
+        return obj
+
+    def __repr__(self):
+        return f'BitPackedOptions(bits={self.bits})'
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+def get_field_options(field_options_proto):
+    """
+    Extract custom bitwise options from a FieldDescriptorProto.options object.
+
+    Serialises the options message to bytes and walks the wire-format stream
+    looking for extension fields 50001 (quantized) and 50002 (bit_packed).
+
+    Args:
+        field_options_proto: google.protobuf.descriptor_pb2.FieldOptions instance
+            (may be a default/empty instance when no options are set).
+
+    Returns:
+        tuple[QuantizedFloatOptions | None, BitPackedOptions | None]
+    """
+    try:
+        raw = field_options_proto.SerializeToString()
+    except Exception:
+        return None, None
+
+    if not raw:
+        return None, None
+
+    quantized = None
+    bit_packed = None
+    pos = 0
+
+    while pos < len(raw):
+        tag, pos = _read_varint(raw, pos)
+        field_num = tag >> 3
+        wire_type = tag & 0x7
+
+        if wire_type == 2:                              # length-delimited
+            length, pos = _read_varint(raw, pos)
+            value_bytes = raw[pos:pos + length]
+            pos += length
+            if field_num == QUANTIZED_FIELD_NUMBER:
+                quantized = QuantizedFloatOptions.from_bytes(value_bytes)
+            elif field_num == BIT_PACKED_FIELD_NUMBER:
+                bit_packed = BitPackedOptions.from_bytes(value_bytes)
+            # else: unknown length-delimited field — already consumed
+        else:
+            pos = _skip_field(raw, pos, wire_type)
+
+    return quantized, bit_packed

package/protoc-gen-bitwise/plugin.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+"""
+protoc-gen-bitwise — protoc plugin that generates C# bitwise serialization code.
+
+Protocol:
+  1. protoc writes a serialised CodeGeneratorRequest to this process's stdin.
+  2. This plugin reads it, generates C# partial classes with Encode / Decode
+     methods for every message that carries [(quantized)] or [(bit_packed)]
+     field annotations.
+  3. A serialised CodeGeneratorResponse is written to stdout.
+
+Usage (from project root):
+    protoc \\
+      --proto_path=proto \\
+      --bitwise_out=generated/ \\
+      --plugin=protoc-gen-bitwise \\
+      proto/my_messages.proto
+
+Windows invocation (plugin not on PATH as executable):
+    protoc \\
+      --proto_path=proto \\
+      --bitwise_out=generated/ \\
+      --plugin=protoc-gen-bitwise=python protoc-gen-bitwise/plugin.py \\
+      proto/my_messages.proto
+
+Dependencies:
+    pip install grpcio-tools   # or: pip install protobuf
+"""
+
+import os
+import sys
+
+# Ensure sibling modules (generator_csharp, options_pb2) are importable
+# regardless of where protoc invokes this script from.
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+# On Windows, stdin/stdout are opened in text mode by default which corrupts
+# the binary protobuf payload.
+if sys.platform == 'win32':
+    import msvcrt
+    msvcrt.setmode(sys.stdin.fileno(),  os.O_BINARY)
+    msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
+
+from google.protobuf.compiler import plugin_pb2
+
+from generator_csharp import generate_csharp
+
+
+def main() -> None:
+    request_bytes = sys.stdin.buffer.read()
+
+    request = plugin_pb2.CodeGeneratorRequest()
+    request.ParseFromString(request_bytes)
+
+    response = plugin_pb2.CodeGeneratorResponse()
+    # Advertise proto3-optional support so protoc does not reject the plugin.
+    response.supported_features = (
+        plugin_pb2.CodeGeneratorResponse.FEATURE_PROTO3_OPTIONAL
+    )
+
+    # Build a lookup map for all file descriptors (needed for imports, though
+    # the current generator only uses the directly requested files).
+    file_by_name = {f.name: f for f in request.proto_file}
+
+    for file_name in request.file_to_generate:
+        # Skip the options definition file itself — it has no messages to generate.
+        if file_name == 'decentraland/common/options.proto':
+            continue
+
+        file_proto = file_by_name.get(file_name)
+        if file_proto is None:
+            continue
+
+        try:
+            generated = generate_csharp(file_proto)
+        except Exception as exc:  # noqa: BLE001
+            error = response.file.add()
+            error.name = ''  # empty name signals an error to protoc
+            # Append error text; protoc will print it and fail.
+            response.error = f'protoc-gen-bitwise: error processing {file_name}: {exc}'
+            continue
+
+        if generated is not None:
+            out = response.file.add()
+            out.name    = generated['name']
+            out.content = generated['content']
+
+    sys.stdout.buffer.write(response.SerializeToString())
+
+
+if __name__ == '__main__':
+    main()

package/protoc-gen-bitwise/runtime/cs/BitReader.cs

@@ -0,0 +1,112 @@
+// Decentraland.Networking.Bitwise — BitReader
+// Copy this file into your Unity project alongside generated *.Bitwise.cs files.
+
+using System;
+
+namespace Decentraland.Networking.Bitwise
+{
+    /// <summary>
+    /// Reads bits from a byte buffer, MSB first within each byte (big-endian bit
+    /// order).  Symmetric counterpart of <see cref="BitWriter"/>: every
+    /// Write… call has a corresponding Read… call with identical arguments that
+    /// reproduces the original value.
+    /// </summary>
+    public sealed class BitReader
+    {
+        private readonly byte[] _buffer;
+        private int _bitPos;
+
+        /// <param name="buffer">Source buffer filled by a <see cref="BitWriter"/>.</param>
+        public BitReader(byte[] buffer)
+        {
+            _buffer = buffer ?? throw new ArgumentNullException(nameof(buffer));
+            _bitPos = 0;
+        }
+
+        /// <summary>Current read position in bits.</summary>
+        public int BitPosition => _bitPos;
+
+        /// <summary>
+        /// Returns <c>true</c> when all written bits have been consumed
+        /// (i.e. <see cref="BitPosition"/> has reached the end of the buffer).
+        /// </summary>
+        public bool IsAtEnd => _bitPos >= _buffer.Length * 8;
+
+        // -----------------------------------------------------------------
+        // Core primitive
+        // -----------------------------------------------------------------
+
+        /// <summary>
+        /// Reads <paramref name="bits"/> bits and returns them as the
+        /// least-significant bits of a <see cref="uint"/>, MSB first.
+        /// </summary>
+        public uint ReadBits(int bits)
+        {
+            uint value = 0;
+            for (int i = bits - 1; i >= 0; i--)
+            {
+                int byteIdx = _bitPos / 8;
+                int bitIdx  = 7 - (_bitPos % 8);
+
+                if ((_buffer[byteIdx] >> bitIdx & 1) == 1)
+                    value |= 1u << i;
+
+                _bitPos++;
+            }
+            return value;
+        }
+
+        // -----------------------------------------------------------------
+        // Quantized float
+        // -----------------------------------------------------------------
+
+        /// <summary>
+        /// Reads a quantized float encoded with <see cref="BitWriter.WriteQuantizedFloat"/>.
+        /// Arguments must match those used during encoding exactly.
+        /// </summary>
+        public float ReadQuantizedFloat(float min, float max, int bits)
+        {
+            uint  maxQ       = (1u << bits) - 1;
+            uint  quantized  = ReadBits(bits);
+            float normalized = (float)quantized / maxQ;
+            return min + normalized * (max - min);
+        }
+
+        // -----------------------------------------------------------------
+        // Standard IEEE 754 helpers
+        // -----------------------------------------------------------------
+
+        /// <summary>Reads a 32-bit IEEE 754 float written by <see cref="BitWriter.WriteFloat"/>.</summary>
+        public float ReadFloat()
+        {
+            uint bits = ReadBits(32);
+            byte[] bytes =
+            {
+                (byte)(bits        & 0xFF),
+                (byte)((bits >> 8)  & 0xFF),
+                (byte)((bits >> 16) & 0xFF),
+                (byte)((bits >> 24) & 0xFF),
+            };
+            return BitConverter.ToSingle(bytes, 0);
+        }
+
+        /// <summary>Reads a 64-bit IEEE 754 double written by <see cref="BitWriter.WriteDouble"/>.</summary>
+        public double ReadDouble()
+        {
+            uint hi = ReadBits(32);
+            uint lo = ReadBits(32);
+            byte[] bytes =
+            {
+                (byte)(lo         & 0xFF),
+                (byte)((lo >> 8)  & 0xFF),
+                (byte)((lo >> 16) & 0xFF),
+                (byte)((lo >> 24) & 0xFF),
+                (byte)(hi         & 0xFF),
+                (byte)((hi >> 8)  & 0xFF),
+                (byte)((hi >> 16) & 0xFF),
+                (byte)((hi >> 24) & 0xFF),
+            };
+            return BitConverter.ToDouble(bytes, 0);
+        }
+    }
+}

package/protoc-gen-bitwise/runtime/cs/BitWriter.cs

@@ -0,0 +1,117 @@
+// Decentraland.Networking.Bitwise — BitWriter
+// Copy this file into your Unity project alongside generated *.Bitwise.cs files.
+
+using System;
+
+namespace Decentraland.Networking.Bitwise
+{
+    /// <summary>
+    /// Writes bits into a pre-allocated byte buffer, MSB first within each byte
+    /// (big-endian bit order).  This matches the layout expected by
+    /// <see cref="BitReader"/> so that encode → decode is always a round-trip no-op.
+    /// </summary>
+    public sealed class BitWriter
+    {
+        private readonly byte[] _buffer;
+        private int _bitPos;
+
+        /// <param name="buffer">Destination buffer (must be large enough for all writes).</param>
+        public BitWriter(byte[] buffer)
+        {
+            _buffer = buffer ?? throw new ArgumentNullException(nameof(buffer));
+            _bitPos = 0;
+        }
+
+        /// <summary>Current write position in bits.</summary>
+        public int BitPosition => _bitPos;
+
+        /// <summary>Number of bytes written (rounded up to the nearest byte).</summary>
+        public int ByteLength => (_bitPos + 7) / 8;
+
+        /// <summary>Returns a copy of the written bytes (trimmed to <see cref="ByteLength"/>).</summary>
+        public byte[] ToArray()
+        {
+            var result = new byte[ByteLength];
+            Array.Copy(_buffer, result, ByteLength);
+            return result;
+        }
+
+        // -----------------------------------------------------------------
+        // Core primitive
+        // -----------------------------------------------------------------
+
+        /// <summary>
+        /// Writes the <paramref name="bits"/> least-significant bits of
+        /// <paramref name="value"/>, MSB first.
+        /// </summary>
+        public void WriteBits(uint value, int bits)
+        {
+            for (int i = bits - 1; i >= 0; i--)
+            {
+                int byteIdx = _bitPos / 8;
+                int bitIdx  = 7 - (_bitPos % 8);
+
+                if ((value >> i & 1u) == 1u)
+                    _buffer[byteIdx] |=  (byte)(1 << bitIdx);
+                else
+                    _buffer[byteIdx] &= (byte)~(1 << bitIdx);
+
+                _bitPos++;
+            }
+        }
+
+        // -----------------------------------------------------------------
+        // Quantized float
+        // -----------------------------------------------------------------
+
+        /// <summary>
+        /// Quantizes <paramref name="value"/> into <paramref name="bits"/> bits
+        /// using the range [<paramref name="min"/>, <paramref name="max"/>] and
+        /// writes it to the buffer.
+        ///
+        /// Uses <c>Math.Round</c> (banker's rounding → ties-to-even) to guarantee
+        /// that encode → decode is a round-trip no-op.
+        /// </summary>
+        public void WriteQuantizedFloat(float value, float min, float max, int bits)
+        {
+            uint  maxQ       = (1u << bits) - 1;
+            float clamped    = Math.Clamp(value, min, max);
+            float normalized = (clamped - min) / (max - min);
+            uint  quantized  = (uint)Math.Round(normalized * maxQ);
+            WriteBits(quantized, bits);
+        }
+
+        // -----------------------------------------------------------------
+        // Standard IEEE 754 helpers (used for un-annotated float/double fields)
+        // -----------------------------------------------------------------
+
+        /// <summary>Writes a 32-bit IEEE 754 float (4 bytes).</summary>
+        public void WriteFloat(float value)
+        {
+            byte[] bytes = BitConverter.GetBytes(value);
+            // GetBytes is little-endian on all platforms; write MSB first.
+            uint bits = (uint)bytes[0]
+                      | ((uint)bytes[1] << 8)
+                      | ((uint)bytes[2] << 16)
+                      | ((uint)bytes[3] << 24);
+            WriteBits(bits, 32);
+        }
+
+        /// <summary>Writes a 64-bit IEEE 754 double (8 bytes).</summary>
+        public void WriteDouble(double value)
+        {
+            byte[] bytes = BitConverter.GetBytes(value);
+            uint lo = (uint)bytes[0]
+                    | ((uint)bytes[1] << 8)
+                    | ((uint)bytes[2] << 16)
+                    | ((uint)bytes[3] << 24);
+            uint hi = (uint)bytes[4]
+                    | ((uint)bytes[5] << 8)
+                    | ((uint)bytes[6] << 16)
+                    | ((uint)bytes[7] << 24);
+            // Write high 32 bits first so the bit stream is big-endian at word level too.
+            WriteBits(hi, 32);
+            WriteBits(lo, 32);
+        }
+    }
+}