twilic 3.0.0

data/lib/twilic/core/dictionary.rb

@@ -0,0 +1,236 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module Twilic
+  module Core
+    module Dictionary
+      module_function
+
+      def decode_trained_dictionary_payload(payload)
+        reader = Wire::Reader.new(payload)
+        n = reader.read_varuint
+        values = []
+        n.times do
+          values << reader.read_string
+        end
+        raise Errors.invalid_data("trained dictionary payload trailing bytes") unless reader.eof?
+
+        values
+      end
+
+      def encode_trained_dictionary_block(values, dictionary)
+        if values.empty?
+          out = +""
+          out << "\x00"
+          Wire.encode_varuint(0, out)
+          return [out, true]
+        end
+        by_value = {}
+        dictionary.each_with_index { |v, idx| by_value[v] = idx }
+        ids = values.map do |value|
+          id = by_value[value]
+          return [nil, false] unless id
+
+          id
+        end
+        raw = +""
+        raw << "\x00"
+        Wire.encode_varuint(ids.length, raw)
+        ids.each { |id| Wire.encode_varuint(id, raw) }
+        max_id = ids.max || 0
+        bit_width = max_id.zero? ? 0 : (64 - max_id.to_s(2).length)
+        packed = +""
+        pack_fixed_width_u64(ids, bit_width, packed)
+        bitpacked = +""
+        bitpacked << "\x01"
+        Wire.encode_varuint(ids.length, bitpacked)
+        bitpacked << bit_width.chr
+        bitpacked << packed
+        return [bitpacked, true] if bitpacked.bytesize < raw.bytesize
+
+        [raw, true]
+      end
+
+      def decode_trained_dictionary_block(block, dictionary)
+        reader = Wire::Reader.new(block)
+        mode = reader.read_u8
+        n = reader.read_varuint
+        ids = case mode
+              when 0
+                Array.new(n) { reader.read_varuint }
+              when 1
+                bit_width = reader.read_u8
+                remaining = block.bytesize - reader.position
+                packed = reader.read_exact(remaining)
+                unpack_fixed_width_u64(packed, n, bit_width)
+              else
+                raise Errors.invalid_data("trained dictionary block mode")
+              end
+        raise Errors.invalid_data("trained dictionary block trailing bytes") unless reader.eof?
+
+        ids.map do |id|
+          raise Errors.invalid_data("trained dictionary block id") if id >= dictionary.length
+
+          dictionary[id]
+        end
+      end
+
+      WideU128 = Data.define(:lo, :hi)
+
+      def wide_from_u64(v)
+        WideU128.new(lo: v, hi: 0)
+      end
+
+      def wide_mask(width)
+        if width == 64
+          WideU128.new(lo: 0xFFFFFFFFFFFFFFFF, hi: 0xFFFFFFFFFFFFFFFF)
+        elsif width.zero?
+          WideU128.new(lo: 0, hi: 0)
+        elsif width <= 64
+          WideU128.new(lo: (1 << width) - 1, hi: 0)
+        else
+          WideU128.new(lo: 0xFFFFFFFFFFFFFFFF, hi: (1 << (width - 64)) - 1)
+        end
+      end
+
+      def pack_fixed_width_u64(values, width, out)
+        raise Errors.invalid_data("fixed-width u64 bit width") if width > 64
+
+        if width.zero?
+          values.each do |value|
+            raise Errors.invalid_data("fixed-width u64 value overflow") unless value.zero?
+          end
+          return
+        end
+        acc = WideU128.new(lo: 0, hi: 0)
+        acc_bits = 0
+        values.each do |value|
+          raise Errors.invalid_data("fixed-width u64 value overflow") if width < 64 && (value >> width) != 0
+
+          acc = wide_or(acc, wide_shl(wide_from_u64(value), acc_bits))
+          acc_bits += width
+          while acc_bits >= 8
+            out << (acc.lo & 0xFF).chr
+            acc = wide_shr(acc, 8)
+            acc_bits -= 8
+          end
+        end
+        out << (acc.lo & 0xFF).chr if acc_bits.positive?
+      end
+
+      def unpack_fixed_width_u64(bytes, count, width)
+        raise Errors.invalid_data("fixed-width u64 bit width") if width > 64
+
+        if width.zero?
+          bytes.each { |b| raise Errors.invalid_data("fixed-width u64 trailing bytes") unless b.zero? }
+          return Array.new(count, 0)
+        end
+        out = []
+        acc = WideU128.new(lo: 0, hi: 0)
+        acc_bits = 0
+        idx = 0
+        mask = wide_mask(width)
+        count.times do
+          while acc_bits < width
+            raise Errors.invalid_data("fixed-width u64 underflow") if idx >= bytes.bytesize
+
+            acc = wide_or(acc, wide_shl(wide_from_u64(bytes.getbyte(idx)), acc_bits))
+            idx += 1
+            acc_bits += 8
+          end
+          out << wide_and(acc, mask).lo
+          acc = wide_shr(acc, width)
+          acc_bits -= width
+        end
+        raise Errors.invalid_data("fixed-width u64 trailing bytes") unless wide_zero?(acc)
+        while idx < bytes.bytesize
+          raise Errors.invalid_data("fixed-width u64 trailing bytes") unless bytes.getbyte(idx).zero?
+
+          idx += 1
+        end
+        out
+      end
+
+      def wide_zero?(w)
+        w.lo.zero? && w.hi.zero?
+      end
+
+      def wide_and(a, m)
+        WideU128.new(lo: a.lo & m.lo, hi: a.hi & m.hi)
+      end
+
+      def wide_or(a, b)
+        WideU128.new(lo: a.lo | b.lo, hi: a.hi | b.hi)
+      end
+
+      def wide_shl(w, n)
+        return w if n.zero?
+        return WideU128.new(lo: 0, hi: 0) if n >= 128
+
+        if n < 64
+          hi = ((w.hi << n) | (w.lo >> (64 - n))) & 0xFFFFFFFFFFFFFFFF
+          lo = (w.lo << n) & 0xFFFFFFFFFFFFFFFF
+          WideU128.new(lo: lo, hi: hi)
+        else
+          n -= 64
+          WideU128.new(lo: 0, hi: (w.lo << n) & 0xFFFFFFFFFFFFFFFF)
+        end
+      end
+
+      def wide_shr(w, n)
+        return w if n.zero?
+        return WideU128.new(lo: 0, hi: 0) if n >= 128
+
+        if n < 64
+          lo = ((w.lo >> n) | (w.hi << (64 - n))) & 0xFFFFFFFFFFFFFFFF
+          hi = w.hi >> n
+          WideU128.new(lo: lo, hi: hi)
+        else
+          n -= 64
+          WideU128.new(lo: w.hi >> n, hi: 0)
+        end
+      end
+
+      def apply_dictionary_references(state, columns)
+        columns.each_with_index do |column, i|
+          next unless column.values.kind == Model::ElementType::STRING
+
+          values = column.values.strings
+          next if values.length < 16
+
+          unique = values.uniq
+          next if unique.length.to_f / values.length > 0.5
+
+          codec = column.codec
+          next unless codec == Model::VectorCodec::DICTIONARY || codec == Model::VectorCodec::STRING_REF
+
+          dict_id = state.allocate_dictionary_id
+          payload = +""
+          keys = unique.sort
+          Wire.encode_varuint(keys.length, payload)
+          keys.each { |item| Wire.encode_string(item, payload) }
+          profile = Session::DictionaryProfile.new(
+            version: 1,
+            hash: dictionary_payload_hash(payload),
+            expires_at: 0,
+            fallback: state.options.unknown_reference_policy == Session::UnknownReferencePolicy::STATELESS_RETRY ?
+              Session::DictionaryFallback::STATELESS_RETRY : Session::DictionaryFallback::FAIL_FAST
+          )
+          state.dictionaries[dict_id] = payload
+          state.dictionary_profiles[dict_id] = profile
+          columns[i] = column.with(dictionary_id: dict_id)
+        end
+      end
+
+      def dictionary_payload_hash(payload)
+        h = 0xCBF29CE484222325
+        payload.each_byte do |b|
+          h ^= b
+          h = (h * 0x00000100000001B1) & 0xFFFFFFFFFFFFFFFF
+        end
+        h
+      end
+    end
+  end
+end

data/lib/twilic/core/errors.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Twilic
+  module Core
+    module Errors
+      UNEXPECTED_EOF = :unexpected_eof
+      INVALID_KIND = :invalid_kind
+      INVALID_TAG = :invalid_tag
+      INVALID_DATA = :invalid_data
+      UTF8 = :utf8
+      UNKNOWN_REFERENCE = :unknown_reference
+      STATELESS_RETRY_REQUIRED = :stateless_retry_required
+
+      class TwilicError < StandardError
+        attr_reader :kind, :byte, :msg, :ref_kind, :ref_id
+
+        def initialize(kind, byte: nil, msg: nil, ref_kind: nil, ref_id: nil)
+          @kind = kind
+          @byte = byte
+          @msg = msg
+          @ref_kind = ref_kind
+          @ref_id = ref_id
+          super(message)
+        end
+
+        def message
+          case kind
+          when UNEXPECTED_EOF
+            "unexpected end of input"
+          when INVALID_KIND
+            format("invalid message kind: 0x%02x", byte)
+          when INVALID_TAG
+            format("invalid value tag: 0x%02x", byte)
+          when INVALID_DATA
+            "invalid data: #{msg}"
+          when UTF8
+            "utf8 decode error"
+          when UNKNOWN_REFERENCE
+            "unknown reference: #{ref_kind}=#{ref_id}"
+          when STATELESS_RETRY_REQUIRED
+            "stateless retry required for reference: #{ref_kind}=#{ref_id}"
+          else
+            "twilic error"
+          end
+        end
+      end
+
+      module_function
+
+      def unexpected_eof
+        TwilicError.new(UNEXPECTED_EOF)
+      end
+
+      def invalid_kind(byte)
+        TwilicError.new(INVALID_KIND, byte: byte)
+      end
+
+      def invalid_tag(byte)
+        TwilicError.new(INVALID_TAG, byte: byte)
+      end
+
+      def invalid_data(msg)
+        TwilicError.new(INVALID_DATA, msg: msg)
+      end
+
+      def utf8_error
+        TwilicError.new(UTF8)
+      end
+
+      def unknown_reference(kind, id)
+        TwilicError.new(UNKNOWN_REFERENCE, ref_kind: kind, ref_id: id)
+      end
+
+      def stateless_retry_required(kind, id)
+        TwilicError.new(STATELESS_RETRY_REQUIRED, ref_kind: kind, ref_id: id)
+      end
+
+      def stateless_retry?(err)
+        err.is_a?(TwilicError) && err.kind == STATELESS_RETRY_REQUIRED
+      end
+
+      def unknown_reference?(err)
+        err.is_a?(TwilicError) && err.kind == UNKNOWN_REFERENCE
+      end
+    end
+  end
+end

data/lib/twilic/core/interop_fixtures.rb

@@ -0,0 +1,340 @@
+# frozen_string_literal: true
+
+require "twilic/core/model"
+require "twilic/core/protocol"
+
+module Twilic
+  module Core
+    module InteropFixtures
+      InteropFrame = Data.define(:stream, :label, :hex, :bytes)
+
+      module_function
+
+      def interop_id_name_map(id, name)
+        Model.map_value("id" => Model.u64_value(id), "name" => Model.string_value(name))
+      end
+
+      def interop_id_name_role_map(id, name, role)
+        Model.map_value(
+          "id" => Model.u64_value(id),
+          "name" => Model.string_value(name),
+          "role" => Model.string_value(role)
+        )
+      end
+
+      def interop_make_i64_array(length, start)
+        Array.new(length) { |i| Model.i64_value(start + i) }
+      end
+
+      def interop_make_user_rows(names)
+        names.each_with_index.map do |name, i|
+          Model.map_value("id" => Model.u64_value(i + 1), "name" => Model.string_value(name))
+        end
+      end
+
+      def interop_bitpack_control_payload
+        Array.new(512) { |i| i.even? ? 0 : 1 }.pack("C*")
+      end
+
+      def interop_huffman_control_payload
+        Array.new(512, 7).pack("C*")
+      end
+
+      def interop_fse_control_payload
+        Array.new(512) { |i| i % 4 }.pack("C*")
+      end
+
+      def reset_encode_shape_observation(codec, keys)
+        key = codec.shape_key(keys)
+        codec.state.encode_shape_observations.delete(key)
+      end
+
+      def emit_interop_fixtures(out)
+        codec = Protocol::TwilicCodec.new
+
+        alpha = Model.string_value("alpha")
+        emit_interop_value(out, "codec", "scalar_string", codec, alpha)
+
+        map_two = interop_id_name_map(1, "alice")
+        emit_interop_value(out, "codec", "map_two_fields_first", codec, map_two)
+        reset_encode_shape_observation(codec, %w[id name])
+        emit_interop_value(out, "codec", "map_two_fields_second", codec, map_two)
+
+        map_three = interop_id_name_role_map(1, "alice", "admin")
+        emit_interop_value(out, "codec", "map_three_fields_first", codec, map_three)
+        reset_encode_shape_observation(codec, %w[id name role])
+        emit_interop_value(out, "codec", "map_three_fields_second", codec, map_three)
+
+        8.times do |i|
+          dynamic = interop_id_name_map(10 + i, "user-#{i}")
+          emit_interop_value(out, "codec", "bulk_map_#{i}", codec, dynamic)
+        end
+
+        scalar = Model.i64_value(42)
+        base_snapshot = Model.message(
+          kind: Model::MessageKind::BASE_SNAPSHOT,
+          base_snapshot: Model::BaseSnapshotMessage.new(
+            base_id: 77,
+            schema_or_shape_ref: 0,
+            payload: Model.message(kind: Model::MessageKind::SCALAR, scalar: scalar)
+          )
+        )
+        emit_interop_message(out, "codec", "base_snapshot", codec, base_snapshot)
+
+        enc = Protocol::SessionEncoder.new(Session::SessionOptions.default)
+        base_array = Model.array_value(interop_make_i64_array(100, 0))
+        base_bytes = enc.encode(base_array)
+        emit_interop_frame(out, "session", "session_base_array", base_bytes)
+
+        one_change_arr = interop_make_i64_array(100, 0)
+        one_change_arr[0] = Model.i64_value(10_000)
+        one_change = Model.array_value(one_change_arr)
+        one_patch = enc.encode_patch(one_change)
+        emit_interop_frame(out, "session", "session_patch_one_change", one_patch)
+
+        4.times do |step|
+          iter_arr = interop_make_i64_array(100, 0)
+          iter_arr[step] = Model.i64_value(20_000 + step)
+          iterative = Model.array_value(iter_arr)
+          bytes = enc.encode_patch(iterative)
+          emit_interop_frame(out, "session", "session_patch_iter_#{step}", bytes)
+        end
+
+        many_arr = interop_make_i64_array(100, 0)
+        12.times { |idx| many_arr[idx] = Model.i64_value(10_000 + idx) }
+        many_change = Model.array_value(many_arr)
+        many_patch = enc.encode_patch(many_change)
+        emit_interop_frame(out, "session", "session_patch_many_changes", many_patch)
+
+        rows1 = interop_make_user_rows(%w[a b c d])
+        micro_first = enc.encode_micro_batch(rows1)
+        emit_interop_frame(out, "session", "session_micro_batch_first", micro_first)
+
+        rows2 = interop_make_user_rows(%w[aa bb cc dd])
+        micro_second = enc.encode_micro_batch(rows2)
+        emit_interop_frame(out, "session", "session_micro_batch_second", micro_second)
+      end
+
+      def emit_interop_value(out, stream, label, codec, value)
+        bytes = codec.encode_value(value)
+        emit_interop_frame(out, stream, label, bytes)
+      end
+
+      def emit_interop_message(out, stream, label, codec, message)
+        bytes = codec.encode_message(message)
+        emit_interop_frame(out, stream, label, bytes)
+      end
+
+      def emit_interop_frame(out, stream, label, bytes)
+        hex = "0123456789abcdef"
+        encoded = bytes.each_byte.map { |b| hex[b >> 4] + hex[b & 0x0f] }.join
+        frame = "#{stream}|#{label}|#{encoded}\n"
+        if out.respond_to?(:<<)
+          out << frame
+        else
+          out.write(frame)
+        end
+      end
+
+      def parse_interop_frames(input)
+        frames = []
+        input.each_line.with_index do |raw_line, line_no|
+          line = raw_line.strip
+          next if line.empty?
+
+          begin
+            stream, label, hex = parse_interop_frame_line(line)
+            bytes = decode_interop_hex(hex)
+            frames << InteropFrame.new(stream: stream, label: label, hex: hex, bytes: bytes)
+          rescue StandardError => e
+            raise "line #{line_no + 1}: #{e.message}"
+          end
+        end
+        raise "no fixture frames found" if frames.empty?
+
+        frames
+      end
+
+      def parse_interop_frame_line(line)
+        first = line.index("|")
+        raise "invalid frame" if first.nil? || first <= 0
+
+        rest = line[(first + 1)..]
+        second = rest.index("|")
+        raise "invalid frame" if second.nil? || second <= 0
+
+        [line[0...first], rest[0...second], rest[(second + 1)..]]
+      end
+
+      def decode_interop_hex(hex)
+        raise "invalid hex length" unless hex.length.even?
+
+        hex.chars.each_slice(2).map do |hi, lo|
+          (interop_hex_nibble(hi) << 4) | interop_hex_nibble(lo)
+        end.pack("C*")
+      end
+
+      def interop_hex_nibble(ch)
+        case ch
+        when "0".."9" then ch.ord - "0".ord
+        when "a".."f" then ch.ord - "a".ord + 10
+        when "A".."F" then ch.ord - "A".ord + 10
+        else
+          raise "invalid hex"
+        end
+      end
+
+      def interop_expect_codec_value(label)
+        case label
+        when "scalar_string"
+          [Model.string_value("alpha"), true]
+        else
+          if label.start_with?("map_two_fields_")
+            [interop_id_name_map(1, "alice"), true]
+          elsif label.start_with?("map_three_fields_")
+            [interop_id_name_role_map(1, "alice", "admin"), true]
+          elsif label.start_with?("bulk_map_")
+            idx = label.delete_prefix("bulk_map_").to_i
+            [interop_id_name_map(10 + idx, "user-#{idx}"), true]
+          else
+            [nil, false]
+          end
+        end
+      end
+
+      def interop_expect_codec_value?(label)
+        interop_expect_codec_value(label)[1]
+      end
+
+      def interop_expect_control_stream_codec(label)
+        case label
+        when "control_stream_bitpack" then [Model::ControlStreamCodec::BITPACK, true]
+        when "control_stream_huffman" then [Model::ControlStreamCodec::HUFFMAN, true]
+        when "control_stream_fse" then [Model::ControlStreamCodec::FSE, true]
+        else
+          [nil, false]
+        end
+      end
+
+      def interop_expect_control_payload(label)
+        case label
+        when "control_stream_bitpack" then [interop_bitpack_control_payload, true]
+        when "control_stream_huffman" then [interop_huffman_control_payload, true]
+        when "control_stream_fse" then [interop_fse_control_payload, true]
+        else
+          [nil, false]
+        end
+      end
+
+      def interop_expect_control_payload?(label)
+        interop_expect_control_payload(label)[1]
+      end
+
+      def assert_interop_codec_decode(codec, label, frame)
+        case label
+        when "base_snapshot"
+          msg = codec.decode_message(frame)
+          raise "expected base snapshot message" unless msg.kind == Model::MessageKind::BASE_SNAPSHOT && msg.base_snapshot
+          raise "base_id: got #{msg.base_snapshot.base_id} want 77" unless msg.base_snapshot.base_id == 77
+
+          payload = msg.base_snapshot.payload
+          unless payload.kind == Model::MessageKind::SCALAR &&
+                 payload.scalar&.kind == Model::ValueKind::I64 &&
+                 payload.scalar.i64 == 42
+            raise "base snapshot payload mismatch"
+          end
+          return
+        end
+
+        _payload, ok = interop_expect_control_payload(label)
+        if ok
+          msg = codec.decode_message(frame)
+          raise "expected control stream message" unless msg.kind == Model::MessageKind::CONTROL_STREAM && msg.control_stream
+          raise "control stream payload empty for #{label}" if msg.control_stream.payload.empty?
+
+          want_codec, codec_ok = interop_expect_control_stream_codec(label)
+          if codec_ok && msg.control_stream.codec != want_codec
+            raise "control stream codec mismatch for #{label}"
+          end
+          return
+        end
+
+        expected, ok = interop_expect_codec_value(label)
+        raise "no codec expectation for label #{label.inspect}" unless ok
+
+        got = codec.decode_value(frame)
+        raise "decoded value mismatch for #{label}" unless Model.equal(got, expected)
+      end
+
+      def assert_interop_session_decode(codec, label, frame)
+        case label
+        when "session_base_array"
+          got = codec.decode_value(frame)
+          want = Model.array_value(interop_make_i64_array(100, 0))
+          raise "session_base_array value mismatch" unless Model.equal(got, want)
+        when "session_patch_one_change"
+          msg = codec.decode_message(frame)
+          want_arr = interop_make_i64_array(100, 0)
+          want_arr[0] = Model.i64_value(10_000)
+          want = Model.array_value(want_arr)
+          case msg.kind
+          when Model::MessageKind::STATE_PATCH
+            return
+          when Model::MessageKind::TYPED_VECTOR
+            raise "session_patch_one_change: missing typed vector" unless msg.typed_vector
+
+            got = Protocol.send(:typed_vector_to_value, msg.typed_vector)
+            raise "session_patch_one_change typed vector mismatch" unless Model.equal(got, want)
+          when Model::MessageKind::ARRAY
+            got = Model.array_value(msg.array)
+            raise "session_patch_one_change array mismatch" unless Model.equal(got, want)
+          else
+            raise "session_patch_one_change: unexpected kind #{msg.kind}"
+          end
+        when "session_patch_many_changes", "session_micro_batch_first", "session_micro_batch_second"
+          msg = codec.decode_message(frame)
+          case label
+          when "session_patch_many_changes"
+            unless [Model::MessageKind::STATE_PATCH, Model::MessageKind::TYPED_VECTOR,
+                    Model::MessageKind::ARRAY].include?(msg.kind)
+              raise "expected patch or array message, got #{msg.kind}"
+            end
+          when "session_micro_batch_first", "session_micro_batch_second"
+            unless msg.kind == Model::MessageKind::TEMPLATE_BATCH && msg.template_batch
+              raise "expected template batch message, got #{msg.kind}"
+            end
+            raise "expected 4 rows, got #{msg.template_batch.count}" unless msg.template_batch.count == 4
+          end
+        else
+          if label.start_with?("session_patch_iter_")
+            msg = codec.decode_message(frame)
+            unless [Model::MessageKind::STATE_PATCH, Model::MessageKind::TYPED_VECTOR,
+                    Model::MessageKind::ARRAY].include?(msg.kind)
+              raise "#{label}: expected patch or array message, got #{msg.kind}"
+            end
+            return
+          end
+          raise "no session expectation for label #{label.inspect}"
+        end
+      end
+
+      def replay_codec_state(frames, stop_label)
+        iso = Protocol::TwilicCodec.new
+        frames.each do |prior|
+          next unless prior.stream == "codec"
+          break if prior.label == stop_label
+
+          _payload, ok = interop_expect_control_payload(prior.label)
+          if ok || prior.label == "base_snapshot"
+            iso.decode_message(prior.bytes)
+            next
+          end
+
+          _expected, value_ok = interop_expect_codec_value(prior.label)
+          iso.decode_value(prior.bytes) if value_ok
+        end
+        iso
+      end
+    end
+  end
+end