kiwi-schema 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 319c18010050ea7b3ed08e12c7e5935cedd7b7679a594a18e08fe40fcd7a566e
-  data.tar.gz: f749389795cb200fb7b2f4872a7c6d335a1c928e324a1665f0a2aba72d332a23
+  metadata.gz: 20a267de32ce89b5857fba834f24c9533ee7490b73a890a2ca0cb4c7c1c62f6c
+  data.tar.gz: afdc08f234d21b4cf628ccc84a7b74a45f6c9c43af875846161e36e57ce4cb06
 SHA512:
-  metadata.gz: 165a2f2fbece25ae8b42f35cc5047ee907d55c062e6010b9067176dc9788b8b64ac5865f1518eb7d8c47161a320e554b58529f4f1e5bf23127e7ed4f0ce96a58
-  data.tar.gz: 4ca3004904bb0cec021c227cb1b02ad3663a2d01d25a71410a84e7ac9755efd61291a34949d7d1eb5e5386341a8c3801d5112da9d74f2b541599b79062df6b21
+  metadata.gz: d2f279968376713444644ac41467d53d8c443ce26a61cfbab97da334dd32ed63a22e92b2adfe29ecab0ccf9ef88191b468729c53c5dd5e4d8e7e8e424472b834
+  data.tar.gz: 448ff682553625986f2724acfa2eea836a3c863b5acc4a3558547d4fba9b3b40d4ad4e7a31ad72052f33cde7ef40a5ddb60bc0280a79baccbb995e3a09b072ac

data/Gemfile.lock

@@ -1,12 +1,12 @@
 PATH
   remote: .
   specs:
-    kiwi-schema (0.1.1)
+    kiwi-schema (0.2.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    minitest (5.14.1)
+    minitest (5.15.0)
     rake (12.3.3)
 
 PLATFORMS
@@ -18,4 +18,4 @@ DEPENDENCIES
   rake (~> 12.0)
 
 BUNDLED WITH
-   2.1.4
+   2.2.3

data/README.md

@@ -4,11 +4,6 @@ Kiwi is a schema-based binary format for efficiently encoding trees of data.
 
 This is a ruby implementation of the kiwi message format, see [evanw/kiwi](https://github.com/evanw/kiwi/).
 
-Note:
-
-This gem is a just for fun work-in-progress, probably slow, and not yet proven to work in all cases, so be warned.
-You definitely want to take a look at the original C++, Rust, and JS implementations from Evan first.
-
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -33,8 +28,8 @@ require "kiwi"
 # This is the encoding of the Kiwi schema "message ABC { int[] xyz = 1; }"
 schema_bytes = [1, 65, 66, 67, 0, 2, 1, 120, 121, 122, 0, 5, 1, 1]
 schema = Kiwi::Schema.from_binary(schema_bytes)
-schema.encode_abc({ "xyz" => [99, -12] }) # => [1, 2, 198, 1, 23, 0]
-schema.decode_abc([1, 2, 198, 1, 23, 0])  # => {"xyz"=>[99, -12]}
+schema.encode_abc({ "xyz" => [99, -12] }).bytes # => [1, 2, 198, 1, 23, 0]
+schema.decode_abc(Kiwi::ByteBuffer.new([1, 2, 198, 1, 23, 0]))  # => {"xyz"=>[99, -12]}
 ```
 
 ## Contributing

data/lib/kiwi/byte_buffer.rb

@@ -5,6 +5,8 @@ module Kiwi
     attr_reader :data
     attr_reader :index
 
+    alias_method :bytes, :data
+
     def initialize(data = [])
       @data = data
       @index = 0
@@ -20,6 +22,10 @@ module Kiwi
       end
     end
 
+    def read_byte_array
+      read_bytes(read_var_uint)
+    end
+
     def read_bytes(len)
       if index + len > data.length
         raise RangeError, "Read bytes out of bounds"
@@ -90,6 +96,11 @@ module Kiwi
       data.push(value & 255)
     end
 
+    def write_byte_array(value)
+      write_var_uint(value.length)
+      data.push(*value)
+    end
+
     # Write a variable-length signed 32-bit integer to the end of the buffer.
     def write_var_int(value)
       write_var_uint((value << 1) ^ (value >> 31))

data/lib/kiwi/compiler.rb

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+
+module Kiwi
+  module Compiler
+    def self.compile(schema)
+      definitions = {}
+      rb = []
+
+      schema.definitions.each_with_index do |definition, index|
+        definitions[index] = definition
+      end
+
+      schema.definitions.each do |definition|
+        case definition.kind
+        when Definition::KIND_ENUM
+          rb << "#{definition.name} = {"
+          definition.fields.each do |field|
+            rb << "  '#{field.name}' => #{field.value},"
+            rb << "  #{field.value} => '#{field.name}',"
+          end
+          rb << "}.freeze"
+          rb << ""
+        when Definition::KIND_STRUCT, Definition::KIND_MESSAGE
+          rb << "def decode_#{definition.name.downcase}(bb)"
+          rb << compile_decode(definition, definitions)
+          rb << "end"
+          rb << ""
+          rb << "def encode_#{definition.name.downcase}(message, bb = Kiwi::ByteBuffer.new)"
+          rb << compile_encode(definition, definitions)
+          rb << "end"
+          rb << ""
+        else
+          raise "Invalid definition kind: #{definition.kind}"
+        end
+      end
+
+      rb.join("\n")
+    end
+
+    def self.compile_decode(definition, definitions)
+      lines = []
+      indent = "  "
+
+      lines << "  result = {}"
+
+      if definition.kind == Definition::KIND_MESSAGE
+        lines << "  loop do"
+        lines << "    case bb.read_var_uint"
+        lines << "    when 0"
+        lines << "      return result"
+        indent = "    "
+      end
+
+      definition.fields.each do |field|
+        code = ""
+
+        case field.type_id
+        when Field::TYPE_BOOL
+          code = "bb.read_bool"
+        when Field::TYPE_BYTE
+          code = "bb.read_byte"
+        when Field::TYPE_INT
+          code = "bb.read_var_int"
+        when Field::TYPE_UINT
+          code = "bb.read_var_uint"
+        when Field::TYPE_FLOAT
+          code = "bb.read_var_float"
+        when Field::TYPE_STRING
+          code = "bb.read_string"
+        else
+          type = definitions[field.type]
+ 
+          if (!type)
+            raise "Invalid field type: #{field.type} for field #{field.name}"
+          elsif type.kind == Definition::KIND_ENUM
+            code = "#{type.name}[bb.read_var_uint]"
+          else
+            code = "decode_#{type.name.downcase}(bb)"
+          end
+        end
+
+        if definition.kind == Definition::KIND_MESSAGE
+          lines << "    when #{field.value}"
+        end
+
+        if field.is_array
+          if field.type_id == Field::TYPE_BYTE
+            lines << indent + "  result['#{field.name}'] = bb.read_byte_array"
+          else
+            lines << indent + "  length = bb.read_var_uint"
+            lines << indent + "  values = result['#{field.name}'] = Array.new(length)"
+            lines << indent + "  length.times { |i| values[i] = #{code} }"
+          end
+        else
+          lines << indent + "  result['#{field.name}'] = #{code}"
+        end
+      end
+
+      if definition.kind == Definition::KIND_MESSAGE
+        lines << "    else"
+        lines << "      raise RuntimeError, 'Attempted to parse invalid message'"
+        lines << "    end"
+        lines << "  end"
+      else
+        lines << "  return result"
+      end
+
+      lines.join("\n")
+    end
+
+    def self.compile_encode(definition, definitions)
+      lines = []
+
+      definition.fields.each do |field|
+        code = ""
+
+        case field.type_id
+        when Field::TYPE_BOOL
+          code = "bb.write_bool(value)"
+        when Field::TYPE_BYTE
+          code = "bb.write_byte(value)"
+        when Field::TYPE_INT
+          code = "bb.write_var_int(value)"
+        when Field::TYPE_UINT
+          code = "bb.write_var_uint(value)"
+        when Field::TYPE_FLOAT
+          code = "bb.write_var_float(value)"
+        when Field::TYPE_STRING
+          code = "bb.write_string(value)"
+        else
+          type = definitions[field.type]
+ 
+          if (!type)
+            raise "Invalid field type: #{field.type} for field #{field.name}"
+          elsif type.kind == Definition::KIND_ENUM
+            code = <<~CODE
+              encoded = #{type.name}[value]
+              raise "Invalid value for enum #{type.name}" if !encoded
+              bb.write_var_uint(encoded)
+            CODE
+          else
+            code = "encode_#{type.name.downcase}(value, bb)"
+          end
+        end
+
+        lines << "  value = message['#{field.name}']"
+        lines << "  if !value.nil?"
+
+        if definition.kind == Definition::KIND_MESSAGE
+          lines << "    bb.write_var_uint(#{field.value})"
+        end
+
+        if field.is_array
+          if field.type_id == Field::TYPE_BYTE
+            lines << "    bb.write_byte_array(value)"
+          else
+            lines << "    bb.write_var_uint(value.length)"
+            lines << "    value.each do |value|"
+            lines << "      #{code}"
+            lines << "    end"
+          end
+        else
+          lines << "    #{code}"
+        end
+
+        if definition.kind == Definition::KIND_STRUCT
+          lines << "  else"
+          lines << "    raise \"Missing required field: #{field.name}\""
+        end
+
+        lines << "  end"
+        lines << ""
+      end
+
+      if definition.kind == Definition::KIND_MESSAGE
+        lines << "  bb.write_var_uint(0)"
+      end
+
+      lines << ""
+      lines << "  return bb"
+
+      lines.join("\n")
+    end
+  end
+end

data/lib/kiwi/definition.rb

@@ -26,8 +26,6 @@ module Kiwi
     def field(name)
       if idx = field_name_to_index[name]
         fields[idx]
-      else
-        nil
       end
     end
   end

data/lib/kiwi/field.rb

@@ -17,5 +17,9 @@ module Kiwi
       @is_array = is_array
       @value = value
     end
+
+    def type
+      @type_id
+    end
   end
 end

data/lib/kiwi/schema.rb

@@ -8,7 +8,7 @@ module Kiwi
       definition_count = bb.read_var_uint
 
       (0...definition_count).each do
-        definition_name = bb.read_string
+        type_name = bb.read_string
         kind = bb.read_byte
         field_count = bb.read_var_uint
         fields = []
@@ -21,159 +21,19 @@ module Kiwi
           fields << Field.new(name: name, type_id: type_id, is_array: is_array, value: value)
         end
 
-        defs << Definition.new(name: definition_name, kind: kind, fields: fields)
+        defs << Definition.new(name: type_name, kind: kind, fields: fields)
       end
 
-      Kiwi::Schema.new(defs: defs)
+      Kiwi::Schema.new(defs)
     end
 
-    def initialize(defs: [])
-      @defs = defs
-      @def_name_to_index = {}
-
-      @defs.each_with_index do |definition, i|
-        definition.index = i
-        @def_name_to_index[definition.name] = i
-
-        if definition.kind == Definition::KIND_MESSAGE
-          define_singleton_method(:"encode_#{definition.name.downcase}") do |message|
-            encode(definition.index, message)
-          end
-
-          define_singleton_method(:"decode_#{definition.name.downcase}") do |bytes|
-            decode(definition.index, bytes)
-          end
-        end
-      end
-    end
-
-    def decode(type_id, bytes)
-      decode_bb(type_id, ByteBuffer.new(bytes))
-    end
-
-    def encode(type_id, value)
-      bb = ByteBuffer.new
-      encode_bb(type_id, value, bb)
-      bb.data
+    def initialize(definitions)
+      @definitions = definitions
+      instance_eval Compiler.compile(self)
     end
 
     def definitions
-      @defs
+      @definitions
     end
-
-    private
-
-      attr_reader :defs, :def_name_to_index
-
-      def decode_bb(type_id, byte_buffer)
-        case type_id
-        when Field::TYPE_BOOL
-          byte_buffer.read_bool
-        when Field::TYPE_BYTE
-          byte_buffer.read_byte
-        when Field::TYPE_INT
-          byte_buffer.read_var_int
-        when Field::TYPE_UINT
-          byte_buffer.read_var_uint
-        when Field::TYPE_FLOAT
-          byte_buffer.read_var_float
-        when Field::TYPE_STRING
-          byte_buffer.read_string
-        else
-          definition = defs[type_id]
-
-          case definition.kind
-          when Definition::KIND_ENUM
-            if index = definition.field_value_to_index[byte_buffer.read_var_uint]
-              definition.fields[index].name
-            else
-              raise RuntimeError
-            end
-          when Definition::KIND_STRUCT
-            fields = {}
-            definition.fields.each do |field|
-              fields[field.name] = decode_field(field, byte_buffer)
-            end
-            fields
-          when Definition::KIND_MESSAGE
-            fields = {}
-            loop do
-              value = byte_buffer.read_var_uint
-              return fields if value == 0
-
-              if index = definition.field_value_to_index[value]
-                field = definition.fields[index]
-                fields[field.name] = decode_field(field, byte_buffer)
-              else
-                raise RuntimeError
-              end
-            end
-          end
-        end
-      end
-
-      def decode_field(field, byte_buffer)
-        if field.is_array
-          len = byte_buffer.read_var_uint
-          array = []
-          (0...len).each do
-            array << decode_bb(field.type_id, byte_buffer)
-          end
-          array
-        else
-          decode_bb(field.type_id, byte_buffer)
-        end
-      end
-
-      def encode_bb(type_id, value, byte_buffer)
-        case type_id
-        when Field::TYPE_BOOL
-          byte_buffer.write_bool(value)
-        when Field::TYPE_BYTE
-          byte_buffer.write_byte(value)
-        when Field::TYPE_INT
-          byte_buffer.write_var_int(value)
-        when Field::TYPE_UINT
-          byte_buffer.write_var_uint(value)
-        when Field::TYPE_FLOAT
-          byte_buffer.write_var_float(value)
-        when Field::TYPE_STRING
-          byte_buffer.write_string(value)
-        else
-          definition = defs[type_id]
-
-          case definition.kind
-          when Definition::KIND_ENUM
-            enum = definition.field(value)
-            byte_buffer.write_var_uint(enum.value)
-          when Definition::KIND_STRUCT
-            definition.fields.each do |field|
-              if !(field_value = value[field.name]).nil?
-                encode_value(field, field_value, byte_buffer)
-              else
-                raise ArgumentError, "missing required field #{field.name}"
-              end
-            end
-          when Definition::KIND_MESSAGE
-            value.each do |field_name, field_value|
-              field = definition.field(field_name)
-              if !field.nil?
-                byte_buffer.write_var_uint(field.value)
-                encode_value(field, field_value, byte_buffer)
-              end
-            end
-            byte_buffer.write_byte(0)
-          end
-        end
-      end
-
-      def encode_value(field, value, byte_buffer)
-        if field.is_array
-          byte_buffer.write_var_uint(value.length)
-          value.each { |v| encode_bb(field.type_id, v, byte_buffer) }
-        else
-          encode_bb(field.type_id, value, byte_buffer)
-        end
-      end
   end
 end

data/lib/kiwi.rb

@@ -1,10 +1,11 @@
 # frozen_string_literal: true
 
 module Kiwi
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
 
   autoload :ByteBuffer, "kiwi/byte_buffer"
   autoload :Definition, "kiwi/definition"
   autoload :Field, "kiwi/field"
+  autoload :Compiler, "kiwi/compiler"
   autoload :Schema, "kiwi/schema"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kiwi-schema
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Jan Habermann
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-06-22 00:00:00.000000000 Z
+date: 2022-01-30 00:00:00.000000000 Z
 dependencies: []
 description: Kiwi is a schema-based binary format for efficiently encoding trees of
   data.
@@ -28,6 +28,7 @@ files:
 - kiwi-schema.gemspec
 - lib/kiwi.rb
 - lib/kiwi/byte_buffer.rb
+- lib/kiwi/compiler.rb
 - lib/kiwi/definition.rb
 - lib/kiwi/field.rb
 - lib/kiwi/schema.rb
@@ -50,7 +51,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.2.3
 signing_key:
 specification_version: 4
 summary: Kiwi message format implementation for ruby