kiwi-schema 0.1.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 319c18010050ea7b3ed08e12c7e5935cedd7b7679a594a18e08fe40fcd7a566e
4
- data.tar.gz: f749389795cb200fb7b2f4872a7c6d335a1c928e324a1665f0a2aba72d332a23
3
+ metadata.gz: 20a267de32ce89b5857fba834f24c9533ee7490b73a890a2ca0cb4c7c1c62f6c
4
+ data.tar.gz: afdc08f234d21b4cf628ccc84a7b74a45f6c9c43af875846161e36e57ce4cb06
5
5
  SHA512:
6
- metadata.gz: 165a2f2fbece25ae8b42f35cc5047ee907d55c062e6010b9067176dc9788b8b64ac5865f1518eb7d8c47161a320e554b58529f4f1e5bf23127e7ed4f0ce96a58
7
- data.tar.gz: 4ca3004904bb0cec021c227cb1b02ad3663a2d01d25a71410a84e7ac9755efd61291a34949d7d1eb5e5386341a8c3801d5112da9d74f2b541599b79062df6b21
6
+ metadata.gz: d2f279968376713444644ac41467d53d8c443ce26a61cfbab97da334dd32ed63a22e92b2adfe29ecab0ccf9ef88191b468729c53c5dd5e4d8e7e8e424472b834
7
+ data.tar.gz: 448ff682553625986f2724acfa2eea836a3c863b5acc4a3558547d4fba9b3b40d4ad4e7a31ad72052f33cde7ef40a5ddb60bc0280a79baccbb995e3a09b072ac
data/Gemfile.lock CHANGED
@@ -1,12 +1,12 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- kiwi-schema (0.1.1)
4
+ kiwi-schema (0.2.0)
5
5
 
6
6
  GEM
7
7
  remote: https://rubygems.org/
8
8
  specs:
9
- minitest (5.14.1)
9
+ minitest (5.15.0)
10
10
  rake (12.3.3)
11
11
 
12
12
  PLATFORMS
@@ -18,4 +18,4 @@ DEPENDENCIES
18
18
  rake (~> 12.0)
19
19
 
20
20
  BUNDLED WITH
21
- 2.1.4
21
+ 2.2.3
data/README.md CHANGED
@@ -4,11 +4,6 @@ Kiwi is a schema-based binary format for efficiently encoding trees of data.
4
4
 
5
5
  This is a ruby implementation of the kiwi message format, see [evanw/kiwi](https://github.com/evanw/kiwi/).
6
6
 
7
- Note:
8
-
9
- This gem is a just for fun work-in-progress, probably slow, and not yet proven to work in all cases, so be warned.
10
- You definitely want to take a look at the original C++, Rust, and JS implementations from Evan first.
11
-
12
7
  ## Installation
13
8
 
14
9
  Add this line to your application's Gemfile:
@@ -33,8 +28,8 @@ require "kiwi"
33
28
  # This is the encoding of the Kiwi schema "message ABC { int[] xyz = 1; }"
34
29
  schema_bytes = [1, 65, 66, 67, 0, 2, 1, 120, 121, 122, 0, 5, 1, 1]
35
30
  schema = Kiwi::Schema.from_binary(schema_bytes)
36
- schema.encode_abc({ "xyz" => [99, -12] }) # => [1, 2, 198, 1, 23, 0]
37
- schema.decode_abc([1, 2, 198, 1, 23, 0]) # => {"xyz"=>[99, -12]}
31
+ schema.encode_abc({ "xyz" => [99, -12] }).bytes # => [1, 2, 198, 1, 23, 0]
32
+ schema.decode_abc(Kiwi::ByteBuffer.new([1, 2, 198, 1, 23, 0])) # => {"xyz"=>[99, -12]}
38
33
  ```
39
34
 
40
35
  ## Contributing
@@ -5,6 +5,8 @@ module Kiwi
5
5
  attr_reader :data
6
6
  attr_reader :index
7
7
 
8
+ alias_method :bytes, :data
9
+
8
10
  def initialize(data = [])
9
11
  @data = data
10
12
  @index = 0
@@ -20,6 +22,10 @@ module Kiwi
20
22
  end
21
23
  end
22
24
 
25
+ def read_byte_array
26
+ read_bytes(read_var_uint)
27
+ end
28
+
23
29
  def read_bytes(len)
24
30
  if index + len > data.length
25
31
  raise RangeError, "Read bytes out of bounds"
@@ -90,6 +96,11 @@ module Kiwi
90
96
  data.push(value & 255)
91
97
  end
92
98
 
99
+ def write_byte_array(value)
100
+ write_var_uint(value.length)
101
+ data.push(*value)
102
+ end
103
+
93
104
  # Write a variable-length signed 32-bit integer to the end of the buffer.
94
105
  def write_var_int(value)
95
106
  write_var_uint((value << 1) ^ (value >> 31))
@@ -0,0 +1,185 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kiwi
4
+ module Compiler
5
+ def self.compile(schema)
6
+ definitions = {}
7
+ rb = []
8
+
9
+ schema.definitions.each_with_index do |definition, index|
10
+ definitions[index] = definition
11
+ end
12
+
13
+ schema.definitions.each do |definition|
14
+ case definition.kind
15
+ when Definition::KIND_ENUM
16
+ rb << "#{definition.name} = {"
17
+ definition.fields.each do |field|
18
+ rb << " '#{field.name}' => #{field.value},"
19
+ rb << " #{field.value} => '#{field.name}',"
20
+ end
21
+ rb << "}.freeze"
22
+ rb << ""
23
+ when Definition::KIND_STRUCT, Definition::KIND_MESSAGE
24
+ rb << "def decode_#{definition.name.downcase}(bb)"
25
+ rb << compile_decode(definition, definitions)
26
+ rb << "end"
27
+ rb << ""
28
+ rb << "def encode_#{definition.name.downcase}(message, bb = Kiwi::ByteBuffer.new)"
29
+ rb << compile_encode(definition, definitions)
30
+ rb << "end"
31
+ rb << ""
32
+ else
33
+ raise "Invalid definition kind: #{definition.kind}"
34
+ end
35
+ end
36
+
37
+ rb.join("\n")
38
+ end
39
+
40
+ def self.compile_decode(definition, definitions)
41
+ lines = []
42
+ indent = " "
43
+
44
+ lines << " result = {}"
45
+
46
+ if definition.kind == Definition::KIND_MESSAGE
47
+ lines << " loop do"
48
+ lines << " case bb.read_var_uint"
49
+ lines << " when 0"
50
+ lines << " return result"
51
+ indent = " "
52
+ end
53
+
54
+ definition.fields.each do |field|
55
+ code = ""
56
+
57
+ case field.type_id
58
+ when Field::TYPE_BOOL
59
+ code = "bb.read_bool"
60
+ when Field::TYPE_BYTE
61
+ code = "bb.read_byte"
62
+ when Field::TYPE_INT
63
+ code = "bb.read_var_int"
64
+ when Field::TYPE_UINT
65
+ code = "bb.read_var_uint"
66
+ when Field::TYPE_FLOAT
67
+ code = "bb.read_var_float"
68
+ when Field::TYPE_STRING
69
+ code = "bb.read_string"
70
+ else
71
+ type = definitions[field.type]
72
+
73
+ if (!type)
74
+ raise "Invalid field type: #{field.type} for field #{field.name}"
75
+ elsif type.kind == Definition::KIND_ENUM
76
+ code = "#{type.name}[bb.read_var_uint]"
77
+ else
78
+ code = "decode_#{type.name.downcase}(bb)"
79
+ end
80
+ end
81
+
82
+ if definition.kind == Definition::KIND_MESSAGE
83
+ lines << " when #{field.value}"
84
+ end
85
+
86
+ if field.is_array
87
+ if field.type_id == Field::TYPE_BYTE
88
+ lines << indent + " result['#{field.name}'] = bb.read_byte_array"
89
+ else
90
+ lines << indent + " length = bb.read_var_uint"
91
+ lines << indent + " values = result['#{field.name}'] = Array.new(length)"
92
+ lines << indent + " length.times { |i| values[i] = #{code} }"
93
+ end
94
+ else
95
+ lines << indent + " result['#{field.name}'] = #{code}"
96
+ end
97
+ end
98
+
99
+ if definition.kind == Definition::KIND_MESSAGE
100
+ lines << " else"
101
+ lines << " raise RuntimeError, 'Attempted to parse invalid message'"
102
+ lines << " end"
103
+ lines << " end"
104
+ else
105
+ lines << " return result"
106
+ end
107
+
108
+ lines.join("\n")
109
+ end
110
+
111
+ def self.compile_encode(definition, definitions)
112
+ lines = []
113
+
114
+ definition.fields.each do |field|
115
+ code = ""
116
+
117
+ case field.type_id
118
+ when Field::TYPE_BOOL
119
+ code = "bb.write_bool(value)"
120
+ when Field::TYPE_BYTE
121
+ code = "bb.write_byte(value)"
122
+ when Field::TYPE_INT
123
+ code = "bb.write_var_int(value)"
124
+ when Field::TYPE_UINT
125
+ code = "bb.write_var_uint(value)"
126
+ when Field::TYPE_FLOAT
127
+ code = "bb.write_var_float(value)"
128
+ when Field::TYPE_STRING
129
+ code = "bb.write_string(value)"
130
+ else
131
+ type = definitions[field.type]
132
+
133
+ if (!type)
134
+ raise "Invalid field type: #{field.type} for field #{field.name}"
135
+ elsif type.kind == Definition::KIND_ENUM
136
+ code = <<~CODE
137
+ encoded = #{type.name}[value]
138
+ raise "Invalid value for enum #{type.name}" if !encoded
139
+ bb.write_var_uint(encoded)
140
+ CODE
141
+ else
142
+ code = "encode_#{type.name.downcase}(value, bb)"
143
+ end
144
+ end
145
+
146
+ lines << " value = message['#{field.name}']"
147
+ lines << " if !value.nil?"
148
+
149
+ if definition.kind == Definition::KIND_MESSAGE
150
+ lines << " bb.write_var_uint(#{field.value})"
151
+ end
152
+
153
+ if field.is_array
154
+ if field.type_id == Field::TYPE_BYTE
155
+ lines << " bb.write_byte_array(value)"
156
+ else
157
+ lines << " bb.write_var_uint(value.length)"
158
+ lines << " value.each do |value|"
159
+ lines << " #{code}"
160
+ lines << " end"
161
+ end
162
+ else
163
+ lines << " #{code}"
164
+ end
165
+
166
+ if definition.kind == Definition::KIND_STRUCT
167
+ lines << " else"
168
+ lines << " raise \"Missing required field: #{field.name}\""
169
+ end
170
+
171
+ lines << " end"
172
+ lines << ""
173
+ end
174
+
175
+ if definition.kind == Definition::KIND_MESSAGE
176
+ lines << " bb.write_var_uint(0)"
177
+ end
178
+
179
+ lines << ""
180
+ lines << " return bb"
181
+
182
+ lines.join("\n")
183
+ end
184
+ end
185
+ end
@@ -26,8 +26,6 @@ module Kiwi
26
26
  def field(name)
27
27
  if idx = field_name_to_index[name]
28
28
  fields[idx]
29
- else
30
- nil
31
29
  end
32
30
  end
33
31
  end
data/lib/kiwi/field.rb CHANGED
@@ -17,5 +17,9 @@ module Kiwi
17
17
  @is_array = is_array
18
18
  @value = value
19
19
  end
20
+
21
+ def type
22
+ @type_id
23
+ end
20
24
  end
21
25
  end
data/lib/kiwi/schema.rb CHANGED
@@ -8,7 +8,7 @@ module Kiwi
8
8
  definition_count = bb.read_var_uint
9
9
 
10
10
  (0...definition_count).each do
11
- definition_name = bb.read_string
11
+ type_name = bb.read_string
12
12
  kind = bb.read_byte
13
13
  field_count = bb.read_var_uint
14
14
  fields = []
@@ -21,159 +21,19 @@ module Kiwi
21
21
  fields << Field.new(name: name, type_id: type_id, is_array: is_array, value: value)
22
22
  end
23
23
 
24
- defs << Definition.new(name: definition_name, kind: kind, fields: fields)
24
+ defs << Definition.new(name: type_name, kind: kind, fields: fields)
25
25
  end
26
26
 
27
- Kiwi::Schema.new(defs: defs)
27
+ Kiwi::Schema.new(defs)
28
28
  end
29
29
 
30
- def initialize(defs: [])
31
- @defs = defs
32
- @def_name_to_index = {}
33
-
34
- @defs.each_with_index do |definition, i|
35
- definition.index = i
36
- @def_name_to_index[definition.name] = i
37
-
38
- if definition.kind == Definition::KIND_MESSAGE
39
- define_singleton_method(:"encode_#{definition.name.downcase}") do |message|
40
- encode(definition.index, message)
41
- end
42
-
43
- define_singleton_method(:"decode_#{definition.name.downcase}") do |bytes|
44
- decode(definition.index, bytes)
45
- end
46
- end
47
- end
48
- end
49
-
50
- def decode(type_id, bytes)
51
- decode_bb(type_id, ByteBuffer.new(bytes))
52
- end
53
-
54
- def encode(type_id, value)
55
- bb = ByteBuffer.new
56
- encode_bb(type_id, value, bb)
57
- bb.data
30
+ def initialize(definitions)
31
+ @definitions = definitions
32
+ instance_eval Compiler.compile(self)
58
33
  end
59
34
 
60
35
  def definitions
61
- @defs
36
+ @definitions
62
37
  end
63
-
64
- private
65
-
66
- attr_reader :defs, :def_name_to_index
67
-
68
- def decode_bb(type_id, byte_buffer)
69
- case type_id
70
- when Field::TYPE_BOOL
71
- byte_buffer.read_bool
72
- when Field::TYPE_BYTE
73
- byte_buffer.read_byte
74
- when Field::TYPE_INT
75
- byte_buffer.read_var_int
76
- when Field::TYPE_UINT
77
- byte_buffer.read_var_uint
78
- when Field::TYPE_FLOAT
79
- byte_buffer.read_var_float
80
- when Field::TYPE_STRING
81
- byte_buffer.read_string
82
- else
83
- definition = defs[type_id]
84
-
85
- case definition.kind
86
- when Definition::KIND_ENUM
87
- if index = definition.field_value_to_index[byte_buffer.read_var_uint]
88
- definition.fields[index].name
89
- else
90
- raise RuntimeError
91
- end
92
- when Definition::KIND_STRUCT
93
- fields = {}
94
- definition.fields.each do |field|
95
- fields[field.name] = decode_field(field, byte_buffer)
96
- end
97
- fields
98
- when Definition::KIND_MESSAGE
99
- fields = {}
100
- loop do
101
- value = byte_buffer.read_var_uint
102
- return fields if value == 0
103
-
104
- if index = definition.field_value_to_index[value]
105
- field = definition.fields[index]
106
- fields[field.name] = decode_field(field, byte_buffer)
107
- else
108
- raise RuntimeError
109
- end
110
- end
111
- end
112
- end
113
- end
114
-
115
- def decode_field(field, byte_buffer)
116
- if field.is_array
117
- len = byte_buffer.read_var_uint
118
- array = []
119
- (0...len).each do
120
- array << decode_bb(field.type_id, byte_buffer)
121
- end
122
- array
123
- else
124
- decode_bb(field.type_id, byte_buffer)
125
- end
126
- end
127
-
128
- def encode_bb(type_id, value, byte_buffer)
129
- case type_id
130
- when Field::TYPE_BOOL
131
- byte_buffer.write_bool(value)
132
- when Field::TYPE_BYTE
133
- byte_buffer.write_byte(value)
134
- when Field::TYPE_INT
135
- byte_buffer.write_var_int(value)
136
- when Field::TYPE_UINT
137
- byte_buffer.write_var_uint(value)
138
- when Field::TYPE_FLOAT
139
- byte_buffer.write_var_float(value)
140
- when Field::TYPE_STRING
141
- byte_buffer.write_string(value)
142
- else
143
- definition = defs[type_id]
144
-
145
- case definition.kind
146
- when Definition::KIND_ENUM
147
- enum = definition.field(value)
148
- byte_buffer.write_var_uint(enum.value)
149
- when Definition::KIND_STRUCT
150
- definition.fields.each do |field|
151
- if !(field_value = value[field.name]).nil?
152
- encode_value(field, field_value, byte_buffer)
153
- else
154
- raise ArgumentError, "missing required field #{field.name}"
155
- end
156
- end
157
- when Definition::KIND_MESSAGE
158
- value.each do |field_name, field_value|
159
- field = definition.field(field_name)
160
- if !field.nil?
161
- byte_buffer.write_var_uint(field.value)
162
- encode_value(field, field_value, byte_buffer)
163
- end
164
- end
165
- byte_buffer.write_byte(0)
166
- end
167
- end
168
- end
169
-
170
- def encode_value(field, value, byte_buffer)
171
- if field.is_array
172
- byte_buffer.write_var_uint(value.length)
173
- value.each { |v| encode_bb(field.type_id, v, byte_buffer) }
174
- else
175
- encode_bb(field.type_id, value, byte_buffer)
176
- end
177
- end
178
38
  end
179
39
  end
data/lib/kiwi.rb CHANGED
@@ -1,10 +1,11 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Kiwi
4
- VERSION = "0.1.1"
4
+ VERSION = "0.2.0"
5
5
 
6
6
  autoload :ByteBuffer, "kiwi/byte_buffer"
7
7
  autoload :Definition, "kiwi/definition"
8
8
  autoload :Field, "kiwi/field"
9
+ autoload :Compiler, "kiwi/compiler"
9
10
  autoload :Schema, "kiwi/schema"
10
11
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: kiwi-schema
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.1
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jan Habermann
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-06-22 00:00:00.000000000 Z
11
+ date: 2022-01-30 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: Kiwi is a schema-based binary format for efficiently encoding trees of
14
14
  data.
@@ -28,6 +28,7 @@ files:
28
28
  - kiwi-schema.gemspec
29
29
  - lib/kiwi.rb
30
30
  - lib/kiwi/byte_buffer.rb
31
+ - lib/kiwi/compiler.rb
31
32
  - lib/kiwi/definition.rb
32
33
  - lib/kiwi/field.rb
33
34
  - lib/kiwi/schema.rb
@@ -50,7 +51,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
50
51
  - !ruby/object:Gem::Version
51
52
  version: '0'
52
53
  requirements: []
53
- rubygems_version: 3.1.2
54
+ rubygems_version: 3.2.3
54
55
  signing_key:
55
56
  specification_version: 4
56
57
  summary: Kiwi message format implementation for ruby