kiwi-schema 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of kiwi-schema might be problematic. Click here for more details.
- checksums.yaml +7 -0
- data/.gitignore +8 -0
- data/.travis.yml +6 -0
- data/Gemfile +7 -0
- data/Gemfile.lock +21 -0
- data/README.md +43 -0
- data/Rakefile +10 -0
- data/bin/console +7 -0
- data/kiwi-schema.gemspec +17 -0
- data/lib/kiwi.rb +10 -0
- data/lib/kiwi/byte_buffer.rb +129 -0
- data/lib/kiwi/definition.rb +34 -0
- data/lib/kiwi/field.rb +21 -0
- data/lib/kiwi/schema.rb +179 -0
- metadata +56 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 152e96108e00da08616725a10209ae3416c6f9ec611fe4502206d27f33545a7e
|
4
|
+
data.tar.gz: 04d75522e8b59fd2135d2e477bc1e2220127abb422e80ed7e31ad9e8a17f8751
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 94d70eaa4a840b0c25d5468fd1bcdca0bc880eed42e31efda43f742c397cf9ed3e1e7ce9948e76681fc8eaa50d96a70335435d179bd18ac064e56472d9e89bd6
|
7
|
+
data.tar.gz: 48a71dc215afa11eeb16ba94c3aaaff95477964269700c2d2e7b49e430ac3c1b7466e3b720693c8f694b09184a38c8fa31ad069ee9f2bd2ca2a97ef4ea5ba804
|
data/.gitignore
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
kiwi-schema (0.1.0)
|
5
|
+
|
6
|
+
GEM
|
7
|
+
remote: https://rubygems.org/
|
8
|
+
specs:
|
9
|
+
minitest (5.14.1)
|
10
|
+
rake (12.3.3)
|
11
|
+
|
12
|
+
PLATFORMS
|
13
|
+
ruby
|
14
|
+
|
15
|
+
DEPENDENCIES
|
16
|
+
kiwi-schema!
|
17
|
+
minitest (~> 5.0)
|
18
|
+
rake (~> 12.0)
|
19
|
+
|
20
|
+
BUNDLED WITH
|
21
|
+
2.1.4
|
data/README.md
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
# kiwi-schema GEM
|
2
|
+
|
3
|
+
Kiwi is a schema-based binary format for efficiently encoding trees of data.
|
4
|
+
|
5
|
+
This is a ruby implementation of the kiwi message format, see [evanw/kiwi](https://github.com/evanw/kiwi/).
|
6
|
+
|
7
|
+
Note:
|
8
|
+
|
9
|
+
This gem is a just for fun work-in-progress, probably slow, and not yet proven to work in all cases, so be warned.
|
10
|
+
You definitely want to take a look at the original C++, Rust, and JS implementations from Evan first.
|
11
|
+
|
12
|
+
## Installation
|
13
|
+
|
14
|
+
Add this line to your application's Gemfile:
|
15
|
+
|
16
|
+
```ruby
|
17
|
+
gem 'kiwi-schema'
|
18
|
+
```
|
19
|
+
|
20
|
+
And then execute:
|
21
|
+
|
22
|
+
$ bundle install
|
23
|
+
|
24
|
+
Or install it yourself as:
|
25
|
+
|
26
|
+
$ gem install kiwi-schema
|
27
|
+
|
28
|
+
## Usage
|
29
|
+
|
30
|
+
```ruby
|
31
|
+
require "kiwi"
|
32
|
+
|
33
|
+
# This is the encoding of the Kiwi schema "message ABC { int[] xyz = 1; }"
|
34
|
+
schema_bytes = [1, 65, 66, 67, 0, 2, 1, 120, 121, 122, 0, 5, 1, 1]
|
35
|
+
schema = Kiwi::Schema.from_binary(schema_bytes)
|
36
|
+
schema.encode_abc({ "xyz" => [99, -12] }) # => [1, 2, 198, 1, 23, 0]
|
37
|
+
schema.decode_abc([1, 2, 198, 1, 23, 0]) # => {"xyz"=>[99, -12]}
|
38
|
+
```
|
39
|
+
|
40
|
+
## Contributing
|
41
|
+
|
42
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/haberbyte/kiwi-schema.
|
43
|
+
|
data/Rakefile
ADDED
data/bin/console
ADDED
data/kiwi-schema.gemspec
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
require_relative 'lib/kiwi'
|
2
|
+
|
3
|
+
Gem::Specification.new do |spec|
|
4
|
+
spec.name = "kiwi-schema"
|
5
|
+
spec.version = Kiwi::VERSION
|
6
|
+
spec.authors = ["Jan Habermann"]
|
7
|
+
spec.email = ["jan@habermann.io"]
|
8
|
+
spec.summary = %q{Kiwi encoding/decoding in ruby}
|
9
|
+
spec.description = %q{Kiwi encoding/decoding in ruby}
|
10
|
+
spec.homepage = "https://www.github/com/haberbyte/kiwi-schema"
|
11
|
+
spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
|
12
|
+
spec.metadata["homepage_uri"] = spec.homepage
|
13
|
+
spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
|
14
|
+
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
15
|
+
end
|
16
|
+
spec.require_paths = ["lib"]
|
17
|
+
end
|
data/lib/kiwi.rb
ADDED
@@ -0,0 +1,129 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Kiwi
|
4
|
+
class ByteBuffer
|
5
|
+
attr_reader :data
|
6
|
+
attr_reader :index
|
7
|
+
|
8
|
+
def initialize(data = [])
|
9
|
+
@data = data
|
10
|
+
@index = 0
|
11
|
+
end
|
12
|
+
|
13
|
+
def read_byte
|
14
|
+
if index >= data.length
|
15
|
+
raise RangeError, "Index out of bounds"
|
16
|
+
else
|
17
|
+
value = data[index]
|
18
|
+
@index += 1
|
19
|
+
value
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def read_bytes(len)
|
24
|
+
if index + len > data.length
|
25
|
+
raise RangeError, "Read bytes out of bounds"
|
26
|
+
else
|
27
|
+
value = data[index..(index + len)]
|
28
|
+
@index += len
|
29
|
+
value
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def read_bool
|
34
|
+
read_byte == 0x1
|
35
|
+
end
|
36
|
+
|
37
|
+
def read_var_int
|
38
|
+
value = read_var_uint
|
39
|
+
|
40
|
+
if (value & 1) != 0
|
41
|
+
~(value >> 1)
|
42
|
+
else
|
43
|
+
value >> 1
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def read_var_uint
|
48
|
+
shift = 0
|
49
|
+
result = 0
|
50
|
+
|
51
|
+
loop do
|
52
|
+
byte = read_byte
|
53
|
+
result |= (byte & 127) << shift
|
54
|
+
shift += 7
|
55
|
+
|
56
|
+
break if (byte & 128) == 0 || shift >= 35
|
57
|
+
end
|
58
|
+
|
59
|
+
result
|
60
|
+
end
|
61
|
+
|
62
|
+
def read_var_float
|
63
|
+
first = read_byte
|
64
|
+
return 0.0 if first == 0
|
65
|
+
raise RangeError, "Index out of bounds" if index + 3 > data.length
|
66
|
+
bits = first | (data[index] << 8) | (data[index + 1] << 16) | (data[index + 2] << 24)
|
67
|
+
bits = (bits << 23) | (bits >> 9) # Move the exponent back into place
|
68
|
+
value = [bits].pack("L").unpack("F*")[0]
|
69
|
+
@index += 3
|
70
|
+
value
|
71
|
+
end
|
72
|
+
|
73
|
+
def read_string
|
74
|
+
result = []
|
75
|
+
|
76
|
+
loop do
|
77
|
+
char = read_byte
|
78
|
+
break if char == 0
|
79
|
+
result << char
|
80
|
+
end
|
81
|
+
|
82
|
+
result.pack("C*").force_encoding("UTF-8")
|
83
|
+
end
|
84
|
+
|
85
|
+
def write_bool(value)
|
86
|
+
data.push(value ? 0x1 : 0x0)
|
87
|
+
end
|
88
|
+
|
89
|
+
def write_byte(value)
|
90
|
+
data.push(value & 255)
|
91
|
+
end
|
92
|
+
|
93
|
+
# Write a variable-length signed 32-bit integer to the end of the buffer.
|
94
|
+
def write_var_int(value)
|
95
|
+
write_var_uint((value << 1) ^ (value >> 31))
|
96
|
+
end
|
97
|
+
|
98
|
+
def write_var_uint(value)
|
99
|
+
loop do
|
100
|
+
byte = value & 127
|
101
|
+
value = value >> 7
|
102
|
+
|
103
|
+
if value == 0
|
104
|
+
write_byte(byte)
|
105
|
+
return
|
106
|
+
end
|
107
|
+
|
108
|
+
write_byte(byte | 128)
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
def write_var_float(value)
|
113
|
+
bits = [value].pack("F*").unpack("L")[0]
|
114
|
+
bits = (bits >> 23) | (bits << 9)
|
115
|
+
bytes = [bits].pack("L").unpack("C*")
|
116
|
+
|
117
|
+
if (bytes[0] & 255) == 0
|
118
|
+
data.push(0)
|
119
|
+
else
|
120
|
+
data.push(*bytes)
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
def write_string(value)
|
125
|
+
data.push(*value.bytes)
|
126
|
+
data.push(0)
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Kiwi
|
4
|
+
class Definition
|
5
|
+
KIND_ENUM = 0
|
6
|
+
KIND_STRUCT = 1
|
7
|
+
KIND_MESSAGE = 2
|
8
|
+
|
9
|
+
attr_accessor :name, :index, :kind, :fields
|
10
|
+
attr_reader :field_value_to_index, :field_name_to_index
|
11
|
+
|
12
|
+
def initialize(name:, kind:, fields: [])
|
13
|
+
@name = name
|
14
|
+
@kind = kind
|
15
|
+
@fields = fields
|
16
|
+
@index = 0
|
17
|
+
@field_name_to_index = {}
|
18
|
+
@field_value_to_index = {}
|
19
|
+
|
20
|
+
@fields.each_with_index do |field, i|
|
21
|
+
field_value_to_index[field.value] = i
|
22
|
+
field_name_to_index[field.name] = i
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def field(name)
|
27
|
+
if idx = field_name_to_index[name]
|
28
|
+
fields[idx]
|
29
|
+
else
|
30
|
+
nil
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
data/lib/kiwi/field.rb
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Kiwi
|
4
|
+
class Field
|
5
|
+
TYPE_BOOL = -1
|
6
|
+
TYPE_BYTE = -2
|
7
|
+
TYPE_INT = -3
|
8
|
+
TYPE_UINT = -4
|
9
|
+
TYPE_FLOAT = -5
|
10
|
+
TYPE_STRING = -6
|
11
|
+
|
12
|
+
attr_reader :name, :type_id, :is_array, :value
|
13
|
+
|
14
|
+
def initialize(name:, type_id:, is_array:, value:)
|
15
|
+
@name = name
|
16
|
+
@type_id = type_id
|
17
|
+
@is_array = is_array
|
18
|
+
@value = value
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
data/lib/kiwi/schema.rb
ADDED
@@ -0,0 +1,179 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Kiwi
|
4
|
+
class Schema
|
5
|
+
def self.from_binary(bytes)
|
6
|
+
defs = []
|
7
|
+
bb = ByteBuffer.new(bytes)
|
8
|
+
definition_count = bb.read_var_uint
|
9
|
+
|
10
|
+
(0...definition_count).each do
|
11
|
+
definition_name = bb.read_string
|
12
|
+
kind = bb.read_byte
|
13
|
+
field_count = bb.read_var_uint
|
14
|
+
fields = []
|
15
|
+
|
16
|
+
(0...(field_count)).each do |field|
|
17
|
+
name = bb.read_string
|
18
|
+
type_id = bb.read_var_int
|
19
|
+
is_array = bb.read_bool
|
20
|
+
value = bb.read_var_uint
|
21
|
+
fields << Field.new(name: name, type_id: type_id, is_array: is_array, value: value)
|
22
|
+
end
|
23
|
+
|
24
|
+
defs << Definition.new(name: definition_name, kind: kind, fields: fields)
|
25
|
+
end
|
26
|
+
|
27
|
+
Kiwi::Schema.new(defs: defs)
|
28
|
+
end
|
29
|
+
|
30
|
+
def initialize(defs: [])
|
31
|
+
@defs = defs
|
32
|
+
@def_name_to_index = {}
|
33
|
+
|
34
|
+
@defs.each_with_index do |definition, i|
|
35
|
+
definition.index = i
|
36
|
+
@def_name_to_index[definition.name] = i
|
37
|
+
|
38
|
+
if definition.kind == Definition::KIND_MESSAGE
|
39
|
+
define_singleton_method(:"encode_#{definition.name.downcase}") do |message|
|
40
|
+
encode(definition.index, message)
|
41
|
+
end
|
42
|
+
|
43
|
+
define_singleton_method(:"decode_#{definition.name.downcase}") do |bytes|
|
44
|
+
decode(definition.index, bytes)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def decode(type_id, bytes)
|
51
|
+
decode_bb(type_id, ByteBuffer.new(bytes))
|
52
|
+
end
|
53
|
+
|
54
|
+
def encode(type_id, value)
|
55
|
+
bb = ByteBuffer.new
|
56
|
+
encode_bb(type_id, value, bb)
|
57
|
+
bb.data
|
58
|
+
end
|
59
|
+
|
60
|
+
def definitions
|
61
|
+
@defs
|
62
|
+
end
|
63
|
+
|
64
|
+
private
|
65
|
+
|
66
|
+
attr_reader :defs, :def_name_to_index
|
67
|
+
|
68
|
+
def decode_bb(type_id, byte_buffer)
|
69
|
+
case type_id
|
70
|
+
when Field::TYPE_BOOL
|
71
|
+
byte_buffer.read_bool
|
72
|
+
when Field::TYPE_BYTE
|
73
|
+
byte_buffer.read_byte
|
74
|
+
when Field::TYPE_INT
|
75
|
+
byte_buffer.read_var_int
|
76
|
+
when Field::TYPE_UINT
|
77
|
+
byte_buffer.read_var_uint
|
78
|
+
when Field::TYPE_FLOAT
|
79
|
+
byte_buffer.read_var_float
|
80
|
+
when Field::TYPE_STRING
|
81
|
+
byte_buffer.read_string
|
82
|
+
else
|
83
|
+
definition = defs[type_id]
|
84
|
+
|
85
|
+
case definition.kind
|
86
|
+
when Definition::KIND_ENUM
|
87
|
+
if index = definition.field_value_to_index[byte_buffer.read_var_uint]
|
88
|
+
definition.fields[index].name
|
89
|
+
else
|
90
|
+
raise RuntimeError
|
91
|
+
end
|
92
|
+
when Definition::KIND_STRUCT
|
93
|
+
fields = {}
|
94
|
+
definition.fields.each do |field|
|
95
|
+
fields[field.name] = decode_field(field, byte_buffer)
|
96
|
+
end
|
97
|
+
fields
|
98
|
+
when Definition::KIND_MESSAGE
|
99
|
+
fields = {}
|
100
|
+
loop do
|
101
|
+
value = byte_buffer.read_var_uint
|
102
|
+
return fields if value == 0
|
103
|
+
|
104
|
+
if index = definition.field_value_to_index[value]
|
105
|
+
field = definition.fields[index]
|
106
|
+
fields[field.name] = decode_field(field, byte_buffer)
|
107
|
+
else
|
108
|
+
raise RuntimeError
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
def decode_field(field, byte_buffer)
|
116
|
+
if field.is_array
|
117
|
+
len = byte_buffer.read_var_uint
|
118
|
+
array = []
|
119
|
+
(0...len).each do
|
120
|
+
array << decode_bb(field.type_id, byte_buffer)
|
121
|
+
end
|
122
|
+
array
|
123
|
+
else
|
124
|
+
decode_bb(field.type_id, byte_buffer)
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
def encode_bb(type_id, value, byte_buffer)
|
129
|
+
case type_id
|
130
|
+
when Field::TYPE_BOOL
|
131
|
+
byte_buffer.write_bool(value)
|
132
|
+
when Field::TYPE_BYTE
|
133
|
+
byte_buffer.write_byte(value)
|
134
|
+
when Field::TYPE_INT
|
135
|
+
byte_buffer.write_var_int(value)
|
136
|
+
when Field::TYPE_UINT
|
137
|
+
byte_buffer.write_var_uint(value)
|
138
|
+
when Field::TYPE_FLOAT
|
139
|
+
byte_buffer.write_var_float(value)
|
140
|
+
when Field::TYPE_STRING
|
141
|
+
byte_buffer.write_string(value)
|
142
|
+
else
|
143
|
+
definition = defs[type_id]
|
144
|
+
|
145
|
+
case definition.kind
|
146
|
+
when Definition::KIND_ENUM
|
147
|
+
enum = definition.field(value)
|
148
|
+
byte_buffer.write_var_uint(enum.value)
|
149
|
+
when Definition::KIND_STRUCT
|
150
|
+
definition.fields.each do |field|
|
151
|
+
if !(field_value = value[field.name]).nil?
|
152
|
+
encode_value(field, field_value, byte_buffer)
|
153
|
+
else
|
154
|
+
raise ArgumentError, "missing required field #{field.name}"
|
155
|
+
end
|
156
|
+
end
|
157
|
+
when Definition::KIND_MESSAGE
|
158
|
+
value.each do |field_name, field_value|
|
159
|
+
field = definition.field(field_name)
|
160
|
+
if !field.nil?
|
161
|
+
byte_buffer.write_var_uint(field.value)
|
162
|
+
encode_value(field, field_value, byte_buffer)
|
163
|
+
end
|
164
|
+
end
|
165
|
+
byte_buffer.write_byte(0)
|
166
|
+
end
|
167
|
+
end
|
168
|
+
end
|
169
|
+
|
170
|
+
def encode_value(field, value, byte_buffer)
|
171
|
+
if field.is_array
|
172
|
+
byte_buffer.write_var_uint(value.length)
|
173
|
+
value.each { |v| encode_bb(field.type_id, v, byte_buffer) }
|
174
|
+
else
|
175
|
+
encode_bb(field.type_id, value, byte_buffer)
|
176
|
+
end
|
177
|
+
end
|
178
|
+
end
|
179
|
+
end
|
metadata
ADDED
@@ -0,0 +1,56 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: kiwi-schema
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Jan Habermann
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2020-06-22 00:00:00.000000000 Z
|
12
|
+
dependencies: []
|
13
|
+
description: Kiwi encoding/decoding in ruby
|
14
|
+
email:
|
15
|
+
- jan@habermann.io
|
16
|
+
executables: []
|
17
|
+
extensions: []
|
18
|
+
extra_rdoc_files: []
|
19
|
+
files:
|
20
|
+
- ".gitignore"
|
21
|
+
- ".travis.yml"
|
22
|
+
- Gemfile
|
23
|
+
- Gemfile.lock
|
24
|
+
- README.md
|
25
|
+
- Rakefile
|
26
|
+
- bin/console
|
27
|
+
- kiwi-schema.gemspec
|
28
|
+
- lib/kiwi.rb
|
29
|
+
- lib/kiwi/byte_buffer.rb
|
30
|
+
- lib/kiwi/definition.rb
|
31
|
+
- lib/kiwi/field.rb
|
32
|
+
- lib/kiwi/schema.rb
|
33
|
+
homepage: https://www.github/com/haberbyte/kiwi-schema
|
34
|
+
licenses: []
|
35
|
+
metadata:
|
36
|
+
homepage_uri: https://www.github/com/haberbyte/kiwi-schema
|
37
|
+
post_install_message:
|
38
|
+
rdoc_options: []
|
39
|
+
require_paths:
|
40
|
+
- lib
|
41
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
42
|
+
requirements:
|
43
|
+
- - ">="
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: 2.3.0
|
46
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
47
|
+
requirements:
|
48
|
+
- - ">="
|
49
|
+
- !ruby/object:Gem::Version
|
50
|
+
version: '0'
|
51
|
+
requirements: []
|
52
|
+
rubygems_version: 3.1.2
|
53
|
+
signing_key:
|
54
|
+
specification_version: 4
|
55
|
+
summary: Kiwi encoding/decoding in ruby
|
56
|
+
test_files: []
|