kiwi-schema 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +3 -3
- data/README.md +2 -7
- data/lib/kiwi/byte_buffer.rb +11 -0
- data/lib/kiwi/compiler.rb +185 -0
- data/lib/kiwi/definition.rb +0 -2
- data/lib/kiwi/field.rb +4 -0
- data/lib/kiwi/schema.rb +7 -147
- data/lib/kiwi.rb +2 -1
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 20a267de32ce89b5857fba834f24c9533ee7490b73a890a2ca0cb4c7c1c62f6c
|
4
|
+
data.tar.gz: afdc08f234d21b4cf628ccc84a7b74a45f6c9c43af875846161e36e57ce4cb06
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d2f279968376713444644ac41467d53d8c443ce26a61cfbab97da334dd32ed63a22e92b2adfe29ecab0ccf9ef88191b468729c53c5dd5e4d8e7e8e424472b834
|
7
|
+
data.tar.gz: 448ff682553625986f2724acfa2eea836a3c863b5acc4a3558547d4fba9b3b40d4ad4e7a31ad72052f33cde7ef40a5ddb60bc0280a79baccbb995e3a09b072ac
|
data/Gemfile.lock
CHANGED
@@ -1,12 +1,12 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
kiwi-schema (0.
|
4
|
+
kiwi-schema (0.2.0)
|
5
5
|
|
6
6
|
GEM
|
7
7
|
remote: https://rubygems.org/
|
8
8
|
specs:
|
9
|
-
minitest (5.
|
9
|
+
minitest (5.15.0)
|
10
10
|
rake (12.3.3)
|
11
11
|
|
12
12
|
PLATFORMS
|
@@ -18,4 +18,4 @@ DEPENDENCIES
|
|
18
18
|
rake (~> 12.0)
|
19
19
|
|
20
20
|
BUNDLED WITH
|
21
|
-
2.
|
21
|
+
2.2.3
|
data/README.md
CHANGED
@@ -4,11 +4,6 @@ Kiwi is a schema-based binary format for efficiently encoding trees of data.
|
|
4
4
|
|
5
5
|
This is a ruby implementation of the kiwi message format, see [evanw/kiwi](https://github.com/evanw/kiwi/).
|
6
6
|
|
7
|
-
Note:
|
8
|
-
|
9
|
-
This gem is a just for fun work-in-progress, probably slow, and not yet proven to work in all cases, so be warned.
|
10
|
-
You definitely want to take a look at the original C++, Rust, and JS implementations from Evan first.
|
11
|
-
|
12
7
|
## Installation
|
13
8
|
|
14
9
|
Add this line to your application's Gemfile:
|
@@ -33,8 +28,8 @@ require "kiwi"
|
|
33
28
|
# This is the encoding of the Kiwi schema "message ABC { int[] xyz = 1; }"
|
34
29
|
schema_bytes = [1, 65, 66, 67, 0, 2, 1, 120, 121, 122, 0, 5, 1, 1]
|
35
30
|
schema = Kiwi::Schema.from_binary(schema_bytes)
|
36
|
-
schema.encode_abc({ "xyz" => [99, -12] }) # => [1, 2, 198, 1, 23, 0]
|
37
|
-
schema.decode_abc([1, 2, 198, 1, 23, 0]) # => {"xyz"=>[99, -12]}
|
31
|
+
schema.encode_abc({ "xyz" => [99, -12] }).bytes # => [1, 2, 198, 1, 23, 0]
|
32
|
+
schema.decode_abc(Kiwi::ByteBuffer.new([1, 2, 198, 1, 23, 0])) # => {"xyz"=>[99, -12]}
|
38
33
|
```
|
39
34
|
|
40
35
|
## Contributing
|
data/lib/kiwi/byte_buffer.rb
CHANGED
@@ -5,6 +5,8 @@ module Kiwi
|
|
5
5
|
attr_reader :data
|
6
6
|
attr_reader :index
|
7
7
|
|
8
|
+
alias_method :bytes, :data
|
9
|
+
|
8
10
|
def initialize(data = [])
|
9
11
|
@data = data
|
10
12
|
@index = 0
|
@@ -20,6 +22,10 @@ module Kiwi
|
|
20
22
|
end
|
21
23
|
end
|
22
24
|
|
25
|
+
def read_byte_array
|
26
|
+
read_bytes(read_var_uint)
|
27
|
+
end
|
28
|
+
|
23
29
|
def read_bytes(len)
|
24
30
|
if index + len > data.length
|
25
31
|
raise RangeError, "Read bytes out of bounds"
|
@@ -90,6 +96,11 @@ module Kiwi
|
|
90
96
|
data.push(value & 255)
|
91
97
|
end
|
92
98
|
|
99
|
+
def write_byte_array(value)
|
100
|
+
write_var_uint(value.length)
|
101
|
+
data.push(*value)
|
102
|
+
end
|
103
|
+
|
93
104
|
# Write a variable-length signed 32-bit integer to the end of the buffer.
|
94
105
|
def write_var_int(value)
|
95
106
|
write_var_uint((value << 1) ^ (value >> 31))
|
@@ -0,0 +1,185 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Kiwi
|
4
|
+
module Compiler
|
5
|
+
def self.compile(schema)
|
6
|
+
definitions = {}
|
7
|
+
rb = []
|
8
|
+
|
9
|
+
schema.definitions.each_with_index do |definition, index|
|
10
|
+
definitions[index] = definition
|
11
|
+
end
|
12
|
+
|
13
|
+
schema.definitions.each do |definition|
|
14
|
+
case definition.kind
|
15
|
+
when Definition::KIND_ENUM
|
16
|
+
rb << "#{definition.name} = {"
|
17
|
+
definition.fields.each do |field|
|
18
|
+
rb << " '#{field.name}' => #{field.value},"
|
19
|
+
rb << " #{field.value} => '#{field.name}',"
|
20
|
+
end
|
21
|
+
rb << "}.freeze"
|
22
|
+
rb << ""
|
23
|
+
when Definition::KIND_STRUCT, Definition::KIND_MESSAGE
|
24
|
+
rb << "def decode_#{definition.name.downcase}(bb)"
|
25
|
+
rb << compile_decode(definition, definitions)
|
26
|
+
rb << "end"
|
27
|
+
rb << ""
|
28
|
+
rb << "def encode_#{definition.name.downcase}(message, bb = Kiwi::ByteBuffer.new)"
|
29
|
+
rb << compile_encode(definition, definitions)
|
30
|
+
rb << "end"
|
31
|
+
rb << ""
|
32
|
+
else
|
33
|
+
raise "Invalid definition kind: #{definition.kind}"
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
rb.join("\n")
|
38
|
+
end
|
39
|
+
|
40
|
+
def self.compile_decode(definition, definitions)
|
41
|
+
lines = []
|
42
|
+
indent = " "
|
43
|
+
|
44
|
+
lines << " result = {}"
|
45
|
+
|
46
|
+
if definition.kind == Definition::KIND_MESSAGE
|
47
|
+
lines << " loop do"
|
48
|
+
lines << " case bb.read_var_uint"
|
49
|
+
lines << " when 0"
|
50
|
+
lines << " return result"
|
51
|
+
indent = " "
|
52
|
+
end
|
53
|
+
|
54
|
+
definition.fields.each do |field|
|
55
|
+
code = ""
|
56
|
+
|
57
|
+
case field.type_id
|
58
|
+
when Field::TYPE_BOOL
|
59
|
+
code = "bb.read_bool"
|
60
|
+
when Field::TYPE_BYTE
|
61
|
+
code = "bb.read_byte"
|
62
|
+
when Field::TYPE_INT
|
63
|
+
code = "bb.read_var_int"
|
64
|
+
when Field::TYPE_UINT
|
65
|
+
code = "bb.read_var_uint"
|
66
|
+
when Field::TYPE_FLOAT
|
67
|
+
code = "bb.read_var_float"
|
68
|
+
when Field::TYPE_STRING
|
69
|
+
code = "bb.read_string"
|
70
|
+
else
|
71
|
+
type = definitions[field.type]
|
72
|
+
|
73
|
+
if (!type)
|
74
|
+
raise "Invalid field type: #{field.type} for field #{field.name}"
|
75
|
+
elsif type.kind == Definition::KIND_ENUM
|
76
|
+
code = "#{type.name}[bb.read_var_uint]"
|
77
|
+
else
|
78
|
+
code = "decode_#{type.name.downcase}(bb)"
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
if definition.kind == Definition::KIND_MESSAGE
|
83
|
+
lines << " when #{field.value}"
|
84
|
+
end
|
85
|
+
|
86
|
+
if field.is_array
|
87
|
+
if field.type_id == Field::TYPE_BYTE
|
88
|
+
lines << indent + " result['#{field.name}'] = bb.read_byte_array"
|
89
|
+
else
|
90
|
+
lines << indent + " length = bb.read_var_uint"
|
91
|
+
lines << indent + " values = result['#{field.name}'] = Array.new(length)"
|
92
|
+
lines << indent + " length.times { |i| values[i] = #{code} }"
|
93
|
+
end
|
94
|
+
else
|
95
|
+
lines << indent + " result['#{field.name}'] = #{code}"
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
if definition.kind == Definition::KIND_MESSAGE
|
100
|
+
lines << " else"
|
101
|
+
lines << " raise RuntimeError, 'Attempted to parse invalid message'"
|
102
|
+
lines << " end"
|
103
|
+
lines << " end"
|
104
|
+
else
|
105
|
+
lines << " return result"
|
106
|
+
end
|
107
|
+
|
108
|
+
lines.join("\n")
|
109
|
+
end
|
110
|
+
|
111
|
+
def self.compile_encode(definition, definitions)
|
112
|
+
lines = []
|
113
|
+
|
114
|
+
definition.fields.each do |field|
|
115
|
+
code = ""
|
116
|
+
|
117
|
+
case field.type_id
|
118
|
+
when Field::TYPE_BOOL
|
119
|
+
code = "bb.write_bool(value)"
|
120
|
+
when Field::TYPE_BYTE
|
121
|
+
code = "bb.write_byte(value)"
|
122
|
+
when Field::TYPE_INT
|
123
|
+
code = "bb.write_var_int(value)"
|
124
|
+
when Field::TYPE_UINT
|
125
|
+
code = "bb.write_var_uint(value)"
|
126
|
+
when Field::TYPE_FLOAT
|
127
|
+
code = "bb.write_var_float(value)"
|
128
|
+
when Field::TYPE_STRING
|
129
|
+
code = "bb.write_string(value)"
|
130
|
+
else
|
131
|
+
type = definitions[field.type]
|
132
|
+
|
133
|
+
if (!type)
|
134
|
+
raise "Invalid field type: #{field.type} for field #{field.name}"
|
135
|
+
elsif type.kind == Definition::KIND_ENUM
|
136
|
+
code = <<~CODE
|
137
|
+
encoded = #{type.name}[value]
|
138
|
+
raise "Invalid value for enum #{type.name}" if !encoded
|
139
|
+
bb.write_var_uint(encoded)
|
140
|
+
CODE
|
141
|
+
else
|
142
|
+
code = "encode_#{type.name.downcase}(value, bb)"
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
146
|
+
lines << " value = message['#{field.name}']"
|
147
|
+
lines << " if !value.nil?"
|
148
|
+
|
149
|
+
if definition.kind == Definition::KIND_MESSAGE
|
150
|
+
lines << " bb.write_var_uint(#{field.value})"
|
151
|
+
end
|
152
|
+
|
153
|
+
if field.is_array
|
154
|
+
if field.type_id == Field::TYPE_BYTE
|
155
|
+
lines << " bb.write_byte_array(value)"
|
156
|
+
else
|
157
|
+
lines << " bb.write_var_uint(value.length)"
|
158
|
+
lines << " value.each do |value|"
|
159
|
+
lines << " #{code}"
|
160
|
+
lines << " end"
|
161
|
+
end
|
162
|
+
else
|
163
|
+
lines << " #{code}"
|
164
|
+
end
|
165
|
+
|
166
|
+
if definition.kind == Definition::KIND_STRUCT
|
167
|
+
lines << " else"
|
168
|
+
lines << " raise \"Missing required field: #{field.name}\""
|
169
|
+
end
|
170
|
+
|
171
|
+
lines << " end"
|
172
|
+
lines << ""
|
173
|
+
end
|
174
|
+
|
175
|
+
if definition.kind == Definition::KIND_MESSAGE
|
176
|
+
lines << " bb.write_var_uint(0)"
|
177
|
+
end
|
178
|
+
|
179
|
+
lines << ""
|
180
|
+
lines << " return bb"
|
181
|
+
|
182
|
+
lines.join("\n")
|
183
|
+
end
|
184
|
+
end
|
185
|
+
end
|
data/lib/kiwi/definition.rb
CHANGED
data/lib/kiwi/field.rb
CHANGED
data/lib/kiwi/schema.rb
CHANGED
@@ -8,7 +8,7 @@ module Kiwi
|
|
8
8
|
definition_count = bb.read_var_uint
|
9
9
|
|
10
10
|
(0...definition_count).each do
|
11
|
-
|
11
|
+
type_name = bb.read_string
|
12
12
|
kind = bb.read_byte
|
13
13
|
field_count = bb.read_var_uint
|
14
14
|
fields = []
|
@@ -21,159 +21,19 @@ module Kiwi
|
|
21
21
|
fields << Field.new(name: name, type_id: type_id, is_array: is_array, value: value)
|
22
22
|
end
|
23
23
|
|
24
|
-
defs << Definition.new(name:
|
24
|
+
defs << Definition.new(name: type_name, kind: kind, fields: fields)
|
25
25
|
end
|
26
26
|
|
27
|
-
Kiwi::Schema.new(defs
|
27
|
+
Kiwi::Schema.new(defs)
|
28
28
|
end
|
29
29
|
|
30
|
-
def initialize(
|
31
|
-
@
|
32
|
-
|
33
|
-
|
34
|
-
@defs.each_with_index do |definition, i|
|
35
|
-
definition.index = i
|
36
|
-
@def_name_to_index[definition.name] = i
|
37
|
-
|
38
|
-
if definition.kind == Definition::KIND_MESSAGE
|
39
|
-
define_singleton_method(:"encode_#{definition.name.downcase}") do |message|
|
40
|
-
encode(definition.index, message)
|
41
|
-
end
|
42
|
-
|
43
|
-
define_singleton_method(:"decode_#{definition.name.downcase}") do |bytes|
|
44
|
-
decode(definition.index, bytes)
|
45
|
-
end
|
46
|
-
end
|
47
|
-
end
|
48
|
-
end
|
49
|
-
|
50
|
-
def decode(type_id, bytes)
|
51
|
-
decode_bb(type_id, ByteBuffer.new(bytes))
|
52
|
-
end
|
53
|
-
|
54
|
-
def encode(type_id, value)
|
55
|
-
bb = ByteBuffer.new
|
56
|
-
encode_bb(type_id, value, bb)
|
57
|
-
bb.data
|
30
|
+
def initialize(definitions)
|
31
|
+
@definitions = definitions
|
32
|
+
instance_eval Compiler.compile(self)
|
58
33
|
end
|
59
34
|
|
60
35
|
def definitions
|
61
|
-
@
|
36
|
+
@definitions
|
62
37
|
end
|
63
|
-
|
64
|
-
private
|
65
|
-
|
66
|
-
attr_reader :defs, :def_name_to_index
|
67
|
-
|
68
|
-
def decode_bb(type_id, byte_buffer)
|
69
|
-
case type_id
|
70
|
-
when Field::TYPE_BOOL
|
71
|
-
byte_buffer.read_bool
|
72
|
-
when Field::TYPE_BYTE
|
73
|
-
byte_buffer.read_byte
|
74
|
-
when Field::TYPE_INT
|
75
|
-
byte_buffer.read_var_int
|
76
|
-
when Field::TYPE_UINT
|
77
|
-
byte_buffer.read_var_uint
|
78
|
-
when Field::TYPE_FLOAT
|
79
|
-
byte_buffer.read_var_float
|
80
|
-
when Field::TYPE_STRING
|
81
|
-
byte_buffer.read_string
|
82
|
-
else
|
83
|
-
definition = defs[type_id]
|
84
|
-
|
85
|
-
case definition.kind
|
86
|
-
when Definition::KIND_ENUM
|
87
|
-
if index = definition.field_value_to_index[byte_buffer.read_var_uint]
|
88
|
-
definition.fields[index].name
|
89
|
-
else
|
90
|
-
raise RuntimeError
|
91
|
-
end
|
92
|
-
when Definition::KIND_STRUCT
|
93
|
-
fields = {}
|
94
|
-
definition.fields.each do |field|
|
95
|
-
fields[field.name] = decode_field(field, byte_buffer)
|
96
|
-
end
|
97
|
-
fields
|
98
|
-
when Definition::KIND_MESSAGE
|
99
|
-
fields = {}
|
100
|
-
loop do
|
101
|
-
value = byte_buffer.read_var_uint
|
102
|
-
return fields if value == 0
|
103
|
-
|
104
|
-
if index = definition.field_value_to_index[value]
|
105
|
-
field = definition.fields[index]
|
106
|
-
fields[field.name] = decode_field(field, byte_buffer)
|
107
|
-
else
|
108
|
-
raise RuntimeError
|
109
|
-
end
|
110
|
-
end
|
111
|
-
end
|
112
|
-
end
|
113
|
-
end
|
114
|
-
|
115
|
-
def decode_field(field, byte_buffer)
|
116
|
-
if field.is_array
|
117
|
-
len = byte_buffer.read_var_uint
|
118
|
-
array = []
|
119
|
-
(0...len).each do
|
120
|
-
array << decode_bb(field.type_id, byte_buffer)
|
121
|
-
end
|
122
|
-
array
|
123
|
-
else
|
124
|
-
decode_bb(field.type_id, byte_buffer)
|
125
|
-
end
|
126
|
-
end
|
127
|
-
|
128
|
-
def encode_bb(type_id, value, byte_buffer)
|
129
|
-
case type_id
|
130
|
-
when Field::TYPE_BOOL
|
131
|
-
byte_buffer.write_bool(value)
|
132
|
-
when Field::TYPE_BYTE
|
133
|
-
byte_buffer.write_byte(value)
|
134
|
-
when Field::TYPE_INT
|
135
|
-
byte_buffer.write_var_int(value)
|
136
|
-
when Field::TYPE_UINT
|
137
|
-
byte_buffer.write_var_uint(value)
|
138
|
-
when Field::TYPE_FLOAT
|
139
|
-
byte_buffer.write_var_float(value)
|
140
|
-
when Field::TYPE_STRING
|
141
|
-
byte_buffer.write_string(value)
|
142
|
-
else
|
143
|
-
definition = defs[type_id]
|
144
|
-
|
145
|
-
case definition.kind
|
146
|
-
when Definition::KIND_ENUM
|
147
|
-
enum = definition.field(value)
|
148
|
-
byte_buffer.write_var_uint(enum.value)
|
149
|
-
when Definition::KIND_STRUCT
|
150
|
-
definition.fields.each do |field|
|
151
|
-
if !(field_value = value[field.name]).nil?
|
152
|
-
encode_value(field, field_value, byte_buffer)
|
153
|
-
else
|
154
|
-
raise ArgumentError, "missing required field #{field.name}"
|
155
|
-
end
|
156
|
-
end
|
157
|
-
when Definition::KIND_MESSAGE
|
158
|
-
value.each do |field_name, field_value|
|
159
|
-
field = definition.field(field_name)
|
160
|
-
if !field.nil?
|
161
|
-
byte_buffer.write_var_uint(field.value)
|
162
|
-
encode_value(field, field_value, byte_buffer)
|
163
|
-
end
|
164
|
-
end
|
165
|
-
byte_buffer.write_byte(0)
|
166
|
-
end
|
167
|
-
end
|
168
|
-
end
|
169
|
-
|
170
|
-
def encode_value(field, value, byte_buffer)
|
171
|
-
if field.is_array
|
172
|
-
byte_buffer.write_var_uint(value.length)
|
173
|
-
value.each { |v| encode_bb(field.type_id, v, byte_buffer) }
|
174
|
-
else
|
175
|
-
encode_bb(field.type_id, value, byte_buffer)
|
176
|
-
end
|
177
|
-
end
|
178
38
|
end
|
179
39
|
end
|
data/lib/kiwi.rb
CHANGED
@@ -1,10 +1,11 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module Kiwi
|
4
|
-
VERSION = "0.
|
4
|
+
VERSION = "0.2.0"
|
5
5
|
|
6
6
|
autoload :ByteBuffer, "kiwi/byte_buffer"
|
7
7
|
autoload :Definition, "kiwi/definition"
|
8
8
|
autoload :Field, "kiwi/field"
|
9
|
+
autoload :Compiler, "kiwi/compiler"
|
9
10
|
autoload :Schema, "kiwi/schema"
|
10
11
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: kiwi-schema
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jan Habermann
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-01-30 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Kiwi is a schema-based binary format for efficiently encoding trees of
|
14
14
|
data.
|
@@ -28,6 +28,7 @@ files:
|
|
28
28
|
- kiwi-schema.gemspec
|
29
29
|
- lib/kiwi.rb
|
30
30
|
- lib/kiwi/byte_buffer.rb
|
31
|
+
- lib/kiwi/compiler.rb
|
31
32
|
- lib/kiwi/definition.rb
|
32
33
|
- lib/kiwi/field.rb
|
33
34
|
- lib/kiwi/schema.rb
|
@@ -50,7 +51,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
50
51
|
- !ruby/object:Gem::Version
|
51
52
|
version: '0'
|
52
53
|
requirements: []
|
53
|
-
rubygems_version: 3.
|
54
|
+
rubygems_version: 3.2.3
|
54
55
|
signing_key:
|
55
56
|
specification_version: 4
|
56
57
|
summary: Kiwi message format implementation for ruby
|