twilic 3.0.0

data/lib/twilic/core/v2.rb

@@ -0,0 +1,387 @@
+# frozen_string_literal: true
+
+require "twilic/core/errors"
+require "twilic/core/model"
+require "twilic/core/wire"
+
+module Twilic
+  module Core
+    module V2
+      NULL_TAG = 0xC0
+      FALSE_TAG = 0xC1
+      TRUE_TAG = 0xC2
+      F64_TAG = 0xC3
+      U8_TAG = 0xC4
+      U16_TAG = 0xC5
+      U32_TAG = 0xC6
+      U64_TAG = 0xC7
+      I8_TAG = 0xC8
+      I16_TAG = 0xC9
+      I32_TAG = 0xCA
+      I64_TAG = 0xCB
+      BIN8_TAG = 0xCC
+      BIN16_TAG = 0xCD
+      BIN32_TAG = 0xCE
+      STR8_TAG = 0xCF
+      STR16_TAG = 0xD0
+      STR32_TAG = 0xD1
+      ARRAY16_TAG = 0xD2
+      ARRAY32_TAG = 0xD3
+      MAP16_TAG = 0xD4
+      MAP32_TAG = 0xD5
+      SHAPE_DEF_TAG = 0xD6
+      KEY_REF_TAG = 0xD8
+      STR_REF_TAG = 0xD9
+
+      class EncodeState
+        attr_accessor :key_ids, :str_ids, :shape_ids, :next_key_id, :next_str_id, :next_shape_id
+
+        def initialize
+          @key_ids = {}
+          @str_ids = {}
+          @shape_ids = {}
+          @next_key_id = 0
+          @next_str_id = 0
+          @next_shape_id = 0
+        end
+      end
+
+      class DecodeState
+        attr_accessor :keys, :strings, :shapes
+
+        def initialize
+          @keys = []
+          @strings = []
+          @shapes = []
+        end
+      end
+
+      module_function
+
+      def encode_v2(value)
+        out = +""
+        state = EncodeState.new
+        encode_v2_value(value, out, state)
+        out
+      end
+
+      def decode_v2(bytes)
+        reader = Wire::Reader.new(bytes)
+        state = DecodeState.new
+        value = decode_v2_value(reader, state)
+        raise Errors.invalid_data("trailing bytes in v2 decode") unless reader.eof?
+
+        value
+      end
+
+      def encode_v2_value(value, out, state)
+        case value.kind
+        when Model::ValueKind::NULL
+          out << NULL_TAG.chr
+        when Model::ValueKind::BOOL
+          out << (value.bool ? TRUE_TAG : FALSE_TAG).chr
+        when Model::ValueKind::I64
+          encode_v2_i64(value.i64, out)
+        when Model::ValueKind::U64
+          encode_v2_u64(value.u64, out)
+        when Model::ValueKind::F64
+          out << F64_TAG.chr
+          Wire.append_f64_le(out, value.f64)
+        when Model::ValueKind::STRING
+          if state.str_ids.key?(value.str)
+            out << STR_REF_TAG.chr
+            Wire.encode_varuint(state.str_ids[value.str], out)
+          else
+            encode_v2_string_literal(value.str, out)
+            state.str_ids[value.str] = state.next_str_id
+            state.next_str_id += 1
+          end
+        when Model::ValueKind::BINARY
+          encode_v2_binary(value.bin, out)
+        when Model::ValueKind::ARRAY
+          encode_v2_array(value.arr, out, state)
+        when Model::ValueKind::MAP
+          encode_v2_map(value.map, out, state)
+        else
+          raise Errors.invalid_data("unsupported value kind")
+        end
+      end
+
+      def encode_v2_array(values, out, state)
+        shape_keys = detect_shape_keys(values)
+        if shape_keys
+          sk = shape_keys.join("\0")
+          shape_id = state.shape_ids[sk]
+          unless shape_id
+            shape_id = state.next_shape_id
+            state.next_shape_id += 1
+            state.shape_ids[sk] = shape_id
+          end
+          write_v2_array_header(values.length, out)
+          out << SHAPE_DEF_TAG.chr
+          Wire.encode_varuint(shape_id, out)
+          Wire.encode_varuint(shape_keys.length, out)
+          shape_keys.each { |key| encode_v2_key(key, out, state) }
+          values.each do |value|
+            raise Errors.invalid_data("shape array row must be map") unless value.kind == Model::ValueKind::MAP
+
+            value.map.each { |field| encode_v2_value(field.value, out, state) }
+          end
+          return
+        end
+        write_v2_array_header(values.length, out)
+        values.each { |value| encode_v2_value(value, out, state) }
+      end
+
+      def encode_v2_map(entries, out, state)
+        write_v2_map_header(entries.length, out)
+        entries.each do |entry|
+          encode_v2_key(entry.key, out, state)
+          encode_v2_value(entry.value, out, state)
+        end
+      end
+
+      def encode_v2_key(key, out, state)
+        if state.key_ids.key?(key)
+          out << KEY_REF_TAG.chr
+          Wire.encode_varuint(state.key_ids[key], out)
+          return
+        end
+        encode_v2_string_literal(key, out)
+        state.key_ids[key] = state.next_key_id
+        state.next_key_id += 1
+      end
+
+      def encode_v2_string_literal(value, out)
+        bytes = value.b
+        if bytes.bytesize <= 31
+          out << (0x80 | bytes.bytesize).chr
+        elsif bytes.bytesize <= 0xFF
+          out << STR8_TAG.chr << bytes.bytesize.chr
+        elsif bytes.bytesize <= 0xFFFF
+          out << STR16_TAG.chr << [bytes.bytesize].pack("v")
+        else
+          out << STR32_TAG.chr << [bytes.bytesize].pack("V")
+        end
+        out << bytes
+      end
+
+      def encode_v2_binary(value, out)
+        if value.bytesize <= 0xFF
+          out << BIN8_TAG.chr << value.bytesize.chr
+        elsif value.bytesize <= 0xFFFF
+          out << BIN16_TAG.chr << [value.bytesize].pack("v")
+        else
+          out << BIN32_TAG.chr << [value.bytesize].pack("V")
+        end
+        out << value
+      end
+
+      def encode_v2_u64(value, out)
+        if value <= 127
+          out << value.chr
+        elsif value <= 0xFF
+          out << U8_TAG.chr << value.chr
+        elsif value <= 0xFFFF
+          out << U16_TAG.chr << [value].pack("v")
+        elsif value <= 0xFFFFFFFF
+          out << U32_TAG.chr << [value].pack("V")
+        else
+          out << U64_TAG.chr
+          Wire.append_u64_le(out, value)
+        end
+      end
+
+      def encode_v2_i64(value, out)
+        if value >= -32 && value <= -1
+          out << (value & 0xFF).chr
+        elsif value >= 0 && value <= 127
+          out << value.chr
+        elsif value >= -128 && value <= 127
+          out << I8_TAG.chr << [value].pack("c")
+        elsif value >= -32_768 && value <= 32_767
+          out << I16_TAG.chr << [value].pack("s<")
+        elsif value >= -2_147_483_648 && value <= 2_147_483_647
+          out << I32_TAG.chr << [value].pack("l<")
+        else
+          out << I64_TAG.chr
+          Wire.append_u64_le(out, value & 0xFFFFFFFFFFFFFFFF)
+        end
+      end
+
+      def write_v2_array_header(length, out)
+        if length <= 15
+          out << (0xA0 | length).chr
+        elsif length <= 0xFFFF
+          out << ARRAY16_TAG.chr << [length].pack("v")
+        else
+          out << ARRAY32_TAG.chr << [length].pack("V")
+        end
+      end
+
+      def write_v2_map_header(length, out)
+        if length <= 15
+          out << (0xB0 | length).chr
+        elsif length <= 0xFFFF
+          out << MAP16_TAG.chr << [length].pack("v")
+        else
+          out << MAP32_TAG.chr << [length].pack("V")
+        end
+      end
+
+      def detect_shape_keys(values)
+        return nil if values.length < 2
+        return nil unless values[0].kind == Model::ValueKind::MAP && !values[0].map.empty?
+
+        keys = values[0].map.map(&:key)
+        values[1..].each do |value|
+          return nil unless value.kind == Model::ValueKind::MAP && value.map.length == keys.length
+
+          value.map.each_with_index { |e, i| return nil unless e.key == keys[i] }
+        end
+        keys
+      end
+
+      def decode_v2_value(reader, state)
+        tag = reader.read_u8
+        decode_v2_value_from_tag(reader, state, tag)
+      end
+
+      def decode_v2_value_from_tag(reader, state, tag)
+        case tag
+        when 0..0x7F
+          Model.u64_value(tag)
+        when 0x80..0x9F
+          length = tag & 0x1F
+          s = reader.read_exact(length).force_encoding(Encoding::UTF_8)
+          state.strings << s
+          Model.string_value(s)
+        when 0xA0..0xAF
+          decode_v2_array_body(reader, state, tag & 0x0F)
+        when 0xB0..0xBF
+          decode_v2_map_body(reader, state, tag & 0x0F)
+        when 0xE0..0xFF
+          Model.i64_value(tag - 256)
+        when NULL_TAG
+          Model.null_value
+        when FALSE_TAG
+          Model.bool_value(false)
+        when TRUE_TAG
+          Model.bool_value(true)
+        when F64_TAG
+          Model.f64_value(reader.read_f64_le)
+        when U8_TAG
+          Model.u64_value(reader.read_u8)
+        when U16_TAG
+          Model.u64_value(reader.read_exact(2).unpack1("v"))
+        when U32_TAG
+          Model.u64_value(reader.read_exact(4).unpack1("V"))
+        when U64_TAG
+          Model.u64_value(reader.read_u64_le)
+        when I8_TAG
+          Model.i64_value(reader.read_exact(1).unpack1("c"))
+        when I16_TAG
+          Model.i64_value(reader.read_exact(2).unpack1("s<"))
+        when I32_TAG
+          Model.i64_value(reader.read_exact(4).unpack1("l<"))
+        when I64_TAG
+          Model.i64_value(reader.read_exact(8).unpack1("q<"))
+        when BIN8_TAG
+          Model.binary_value(reader.read_exact(reader.read_u8))
+        when BIN16_TAG
+          Model.binary_value(reader.read_exact(reader.read_exact(2).unpack1("v")))
+        when BIN32_TAG
+          Model.binary_value(reader.read_exact(reader.read_exact(4).unpack1("V")))
+        when STR8_TAG, STR16_TAG, STR32_TAG
+          decode_v2_string_tag(reader, state, tag)
+        when ARRAY16_TAG
+          decode_v2_array_body(reader, state, reader.read_exact(2).unpack1("v"))
+        when ARRAY32_TAG
+          decode_v2_array_body(reader, state, reader.read_exact(4).unpack1("V"))
+        when MAP16_TAG
+          decode_v2_map_body(reader, state, reader.read_exact(2).unpack1("v"))
+        when MAP32_TAG
+          decode_v2_map_body(reader, state, reader.read_exact(4).unpack1("V"))
+        when STR_REF_TAG
+          id = reader.read_varuint
+          raise Errors.invalid_data("unknown str_ref id") if id >= state.strings.length
+
+          Model.string_value(state.strings[id])
+        else
+          raise Errors.invalid_tag(tag)
+        end
+      end
+
+      def decode_v2_string_tag(reader, state, tag)
+        length = case tag
+                 when STR8_TAG then reader.read_u8
+                 when STR16_TAG then reader.read_exact(2).unpack1("v")
+                 when STR32_TAG then reader.read_exact(4).unpack1("V")
+                 else raise Errors.invalid_data("invalid string tag")
+                 end
+        s = reader.read_exact(length).force_encoding(Encoding::UTF_8)
+        state.strings << s
+        Model.string_value(s)
+      end
+
+      def decode_v2_array_body(reader, state, length)
+        return Model.array_value([]) if length.zero?
+
+        first_tag = reader.read_u8
+        if first_tag == SHAPE_DEF_TAG
+          shape_id = reader.read_varuint
+          key_count = reader.read_varuint
+          keys = Array.new(key_count) { decode_v2_key(reader, state) }
+          while state.shapes.length <= shape_id
+            state.shapes << nil
+          end
+          state.shapes[shape_id] = keys
+          values = Array.new(length) do
+            row = keys.map do |key|
+              val = decode_v2_value(reader, state)
+              Model.entry(key, val)
+            end
+            Model.map_value(row)
+          end
+          return Model.array_value(values)
+        end
+        values = Array.new(length)
+        values[0] = decode_v2_value_from_tag(reader, state, first_tag)
+        (1...length).each { |i| values[i] = decode_v2_value(reader, state) }
+        Model.array_value(values)
+      end
+
+      def decode_v2_map_body(reader, state, length)
+        entries = Array.new(length) do
+          key = decode_v2_key(reader, state)
+          value = decode_v2_value(reader, state)
+          Model.entry(key, value)
+        end
+        Model.map_value(entries)
+      end
+
+      def decode_v2_key(reader, state)
+        tag = reader.read_u8
+        if tag == KEY_REF_TAG
+          id = reader.read_varuint
+          raise Errors.invalid_data("unknown key_ref id") if id >= state.keys.length
+
+          return state.keys[id]
+        end
+        if (0x80..0x9F).cover?(tag)
+          key = reader.read_exact(tag & 0x1F).force_encoding(Encoding::UTF_8)
+          state.keys << key
+          return key
+        end
+        if [STR8_TAG, STR16_TAG, STR32_TAG].include?(tag)
+          v = decode_v2_value_from_tag(reader, state, tag)
+          raise Errors.invalid_data("expected string key") unless v.kind == Model::ValueKind::STRING
+
+          state.keys << v.str
+          return v.str
+        end
+        raise Errors.invalid_data("map key must be key_ref or string")
+      end
+    end
+  end
+end

data/lib/twilic/core/wire.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+require "twilic/core/errors"
+
+module Twilic
+  module Core
+    module Wire
+      module_function
+
+      def encode_varuint(value, out)
+        if value < 0x80
+          out << value.chr
+          return
+        end
+        loop do
+          b = value & 0x7F
+          value >>= 7
+          b |= 0x80 unless value.zero?
+          out << b.chr
+          break if value.zero?
+        end
+      end
+
+      def encode_zigzag(value)
+        ((value << 1) ^ (value >> 63)) & 0xFFFFFFFFFFFFFFFF
+      end
+
+      def decode_zigzag(value)
+        ((value >> 1) ^ (-(value & 1))) & 0xFFFFFFFFFFFFFFFF
+        v = (value >> 1) ^ (-(value & 1))
+        v >= 0x8000000000000000 ? v - 0x10000000000000000 : v
+      end
+
+      def encode_bytes(bytes, out)
+        encode_varuint(bytes.bytesize, out)
+        out << bytes
+      end
+
+      def encode_string(value, out)
+        encode_bytes(value.b, out)
+      end
+
+      def encode_bitmap(bits, out)
+        encode_varuint(bits.length, out)
+        current = 0
+        bits.each_with_index do |bit, i|
+          current |= (1 << (i % 8)) if bit
+          if (i % 8) == 7
+            out << current.chr
+            current = 0
+          end
+        end
+        out << current.chr unless bits.empty? || (bits.length % 8).zero?
+      end
+
+      class Reader
+        attr_reader :offset
+
+        def initialize(input)
+          @input = input.b
+          @offset = 0
+        end
+
+        def position
+          @offset
+        end
+
+        def eof?
+          @offset >= @input.bytesize
+        end
+
+        def read_u8
+          raise Errors.unexpected_eof if @offset >= @input.bytesize
+
+          b = @input.getbyte(@offset)
+          @offset += 1
+          b
+        end
+
+        def read_exact(n)
+          raise Errors.unexpected_eof if @offset + n > @input.bytesize
+
+          slice = @input.byteslice(@offset, n)
+          @offset += n
+          slice
+        end
+
+        def read_varuint
+          shift = 0
+          result = 0
+          loop do
+            raise Errors.invalid_data("varuint too large") if shift >= 64
+
+            b = read_u8
+            result |= (b & 0x7F) << shift
+            return result if (b & 0x80).zero?
+
+            shift += 7
+          end
+        end
+
+        def read_i64_zigzag
+          encoded = read_varuint
+          Wire.decode_zigzag(encoded)
+        end
+
+        def read_bytes
+          n = read_varuint
+          read_exact(n)
+        end
+
+        def read_string
+          n = read_varuint
+          bytes = read_exact(n)
+          raise Errors.utf8_error unless bytes.valid_encoding? && bytes.force_encoding(Encoding::UTF_8).valid_encoding?
+
+          bytes.force_encoding(Encoding::UTF_8)
+        end
+
+        def read_bitmap
+          bit_count = read_varuint
+          byte_count = (bit_count + 7) / 8
+          bytes = read_exact(byte_count)
+          bits = Array.new(bit_count)
+          bit_count.times do |i|
+            bits[i] = ((bytes.getbyte(i / 8) >> (i % 8)) & 1) == 1
+          end
+          bits
+        end
+
+        def read_u64_le
+          b = read_exact(8)
+          b.unpack1("Q<")
+        end
+
+        def read_f64_le
+          [read_u64_le].pack("Q<").unpack1("E")
+        end
+      end
+
+      def read_u64_le(reader)
+        reader.read_u64_le
+      end
+
+      def read_f64_le(reader)
+        reader.read_f64_le
+      end
+
+      def append_u64_le(out, v)
+        out << [v].pack("Q<")
+      end
+
+      def append_f64_le(out, v)
+        append_u64_le(out, [v].pack("E").unpack1("Q<"))
+      end
+    end
+  end
+end

data/lib/twilic/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Twilic
+  VERSION = "3.0.0"
+end

data/lib/twilic.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+require "twilic/version"
+require "twilic/core/errors"
+require "twilic/core/model"
+require "twilic/core/wire"
+require "twilic/core/codec"
+require "twilic/core/session"
+require "twilic/core/dictionary"
+require "twilic/core/v2"
+require "twilic/core/protocol"
+require "twilic/core/api"
+
+module Twilic
+  # Types
+  MessageKind = Core::Model::MessageKind
+  ValueKind = Core::Model::ValueKind
+  Value = Core::Model::Value
+  MapEntry = Core::Model::MapEntry
+  MessageMapEntry = Core::Model::MessageMapEntry
+  KeyRef = Core::Model::KeyRef
+  StringMode = Core::Model::StringMode
+  StringValue = Core::Model::StringValue
+  ElementType = Core::Model::ElementType
+  VectorCodec = Core::Model::VectorCodec
+  TypedVectorData = Core::Model::TypedVectorData
+  TypedVector = Core::Model::TypedVector
+  SchemaField = Core::Model::SchemaField
+  Schema = Core::Model::Schema
+  NullStrategy = Core::Model::NullStrategy
+  Column = Core::Model::Column
+  ControlOpcode = Core::Model::ControlOpcode
+  ControlMessage = Core::Model::ControlMessage
+  RegisterShapeControl = Core::Model::RegisterShapeControl
+  PromoteEnumControl = Core::Model::PromoteEnumControl
+  PatchOpcode = Core::Model::PatchOpcode
+  BaseRef = Core::Model::BaseRef
+  PatchOperation = Core::Model::PatchOperation
+  ControlStreamCodec = Core::Model::ControlStreamCodec
+  Message = Core::Model::Message
+  ShapedObjectMessage = Core::Model::ShapedObjectMessage
+  SchemaObjectMessage = Core::Model::SchemaObjectMessage
+  RowBatchMessage = Core::Model::RowBatchMessage
+  ColumnBatchMessage = Core::Model::ColumnBatchMessage
+  ExtMessage = Core::Model::ExtMessage
+  StatePatchMessage = Core::Model::StatePatchMessage
+  TemplateBatchMessage = Core::Model::TemplateBatchMessage
+  ControlStreamMessage = Core::Model::ControlStreamMessage
+  BaseSnapshotMessage = Core::Model::BaseSnapshotMessage
+  TemplateDescriptor = Core::Model::TemplateDescriptor
+  TwilicError = Core::Errors::TwilicError
+  UnknownReferencePolicy = Core::Session::UnknownReferencePolicy
+  DictionaryFallback = Core::Session::DictionaryFallback
+  DictionaryProfile = Core::Session::DictionaryProfile
+  SessionOptions = Core::Session::SessionOptions
+  SessionState = Core::Session::MutableSessionState
+  TwilicCodec = Core::Protocol::TwilicCodec
+  SessionEncoder = Core::Protocol::SessionEncoder
+
+  # Error constants
+  ERR_UNEXPECTED_EOF = Core::Errors::UNEXPECTED_EOF
+  ERR_INVALID_KIND = Core::Errors::INVALID_KIND
+  ERR_INVALID_TAG = Core::Errors::INVALID_TAG
+  ERR_INVALID_DATA = Core::Errors::INVALID_DATA
+  ERR_UTF8 = Core::Errors::UTF8
+  ERR_UNKNOWN_REFERENCE = Core::Errors::UNKNOWN_REFERENCE
+  ERR_STATELESS_RETRY_REQUIRED = Core::Errors::STATELESS_RETRY_REQUIRED
+
+  class << self
+    def encode(value)
+      Core::API.encode(value)
+    end
+
+    def decode(bytes)
+      Core::API.decode(bytes)
+    end
+
+    def encode_with_schema(schema, value)
+      Core::API.encode_with_schema(schema, value)
+    end
+
+    def encode_batch(values)
+      Core::API.encode_batch(values)
+    end
+
+    def null
+      Core::Model.null_value
+    end
+
+    def bool(b)
+      Core::Model.bool_value(b)
+    end
+
+    def i64(n)
+      Core::Model.i64_value(n)
+    end
+
+    def u64(n)
+      Core::Model.u64_value(n)
+    end
+
+    def f64(n)
+      Core::Model.f64_value(n)
+    end
+
+    def string(s)
+      Core::Model.string_value(s)
+    end
+
+    def binary(b)
+      Core::Model.binary_value(b)
+    end
+
+    def array(items)
+      Core::Model.array_value(items)
+    end
+
+    def map(**kwargs)
+      Core::Model.map_value(kwargs)
+    end
+
+    def equal(a, b)
+      Core::Model.equal(a, b)
+    end
+
+    def default_session_options
+      Core::Session::SessionOptions.default
+    end
+
+    def new_twilic_codec
+      TwilicCodec.new
+    end
+
+    def twilic_codec_with_options(options)
+      TwilicCodec.new(options)
+    end
+
+    def new_session_encoder(options = default_session_options)
+      SessionEncoder.new(options)
+    end
+
+    def reset_encode_shape_observation(codec, keys)
+      key = codec.shape_key(keys)
+      codec.state.encode_shape_observations.delete(key)
+    end
+  end
+end

data/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "twilic-ruby",
+  "private": true,
+  "packageManager": "pnpm@10.33.2",
+  "scripts": {
+    "format": "prettier --write \"{README.md,docs/**/*.md,.github/**/*.md}\"",
+    "format:check": "prettier --check \"{README.md,docs/**/*.md,.github/**/*.md}\"",
+    "lint": "markdownlint-cli2 \"{README.md,docs/**/*.md}\""
+  },
+  "devDependencies": {
+    "markdownlint-cli2": "^0.22.1",
+    "prettier": "^3.8.3"
+  }
+}