kiwi-schema 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of kiwi-schema might be problematic. Click here for more details.

@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 152e96108e00da08616725a10209ae3416c6f9ec611fe4502206d27f33545a7e
4
+ data.tar.gz: 04d75522e8b59fd2135d2e477bc1e2220127abb422e80ed7e31ad9e8a17f8751
5
+ SHA512:
6
+ metadata.gz: 94d70eaa4a840b0c25d5468fd1bcdca0bc880eed42e31efda43f742c397cf9ed3e1e7ce9948e76681fc8eaa50d96a70335435d179bd18ac064e56472d9e89bd6
7
+ data.tar.gz: 48a71dc215afa11eeb16ba94c3aaaff95477964269700c2d2e7b49e430ac3c1b7466e3b720693c8f694b09184a38c8fa31ad069ee9f2bd2ca2a97ef4ea5ba804
@@ -0,0 +1,8 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /_yardoc/
4
+ /coverage/
5
+ /doc/
6
+ /pkg/
7
+ /spec/reports/
8
+ /tmp/
@@ -0,0 +1,6 @@
1
+ ---
2
+ language: ruby
3
+ cache: bundler
4
+ rvm:
5
+ - 2.7.1
6
+ before_install: gem install bundler -v 2.1.4
data/Gemfile ADDED
@@ -0,0 +1,7 @@
1
+ source "https://rubygems.org"
2
+
3
+ # Specify your gem's dependencies in kiwi-schema.gemspec
4
+ gemspec
5
+
6
+ gem "rake", "~> 12.0"
7
+ gem "minitest", "~> 5.0"
@@ -0,0 +1,21 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ kiwi-schema (0.1.0)
5
+
6
+ GEM
7
+ remote: https://rubygems.org/
8
+ specs:
9
+ minitest (5.14.1)
10
+ rake (12.3.3)
11
+
12
+ PLATFORMS
13
+ ruby
14
+
15
+ DEPENDENCIES
16
+ kiwi-schema!
17
+ minitest (~> 5.0)
18
+ rake (~> 12.0)
19
+
20
+ BUNDLED WITH
21
+ 2.1.4
@@ -0,0 +1,43 @@
1
+ # kiwi-schema GEM
2
+
3
+ Kiwi is a schema-based binary format for efficiently encoding trees of data.
4
+
5
+ This is a ruby implementation of the kiwi message format, see [evanw/kiwi](https://github.com/evanw/kiwi/).
6
+
7
+ Note:
8
+
9
+ This gem is a just for fun work-in-progress, probably slow, and not yet proven to work in all cases, so be warned.
10
+ You definitely want to take a look at the original C++, Rust, and JS implementations from Evan first.
11
+
12
+ ## Installation
13
+
14
+ Add this line to your application's Gemfile:
15
+
16
+ ```ruby
17
+ gem 'kiwi-schema'
18
+ ```
19
+
20
+ And then execute:
21
+
22
+ $ bundle install
23
+
24
+ Or install it yourself as:
25
+
26
+ $ gem install kiwi-schema
27
+
28
+ ## Usage
29
+
30
+ ```ruby
31
+ require "kiwi"
32
+
33
+ # This is the encoding of the Kiwi schema "message ABC { int[] xyz = 1; }"
34
+ schema_bytes = [1, 65, 66, 67, 0, 2, 1, 120, 121, 122, 0, 5, 1, 1]
35
+ schema = Kiwi::Schema.from_binary(schema_bytes)
36
+ schema.encode_abc({ "xyz" => [99, -12] }) # => [1, 2, 198, 1, 23, 0]
37
+ schema.decode_abc([1, 2, 198, 1, 23, 0]) # => {"xyz"=>[99, -12]}
38
+ ```
39
+
40
+ ## Contributing
41
+
42
+ Bug reports and pull requests are welcome on GitHub at https://github.com/haberbyte/kiwi-schema.
43
+
@@ -0,0 +1,10 @@
1
+ require "bundler/gem_tasks"
2
+ require "rake/testtask"
3
+
4
+ Rake::TestTask.new(:test) do |t|
5
+ t.libs << "test"
6
+ t.libs << "lib"
7
+ t.test_files = FileList["test/**/*_test.rb"]
8
+ end
9
+
10
+ task :default => :test
@@ -0,0 +1,7 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "kiwi/schema"
5
+
6
+ require "irb"
7
+ IRB.start(__FILE__)
@@ -0,0 +1,17 @@
1
+ require_relative 'lib/kiwi'
2
+
3
+ Gem::Specification.new do |spec|
4
+ spec.name = "kiwi-schema"
5
+ spec.version = Kiwi::VERSION
6
+ spec.authors = ["Jan Habermann"]
7
+ spec.email = ["jan@habermann.io"]
8
+ spec.summary = %q{Kiwi encoding/decoding in ruby}
9
+ spec.description = %q{Kiwi encoding/decoding in ruby}
10
+ spec.homepage = "https://www.github/com/haberbyte/kiwi-schema"
11
+ spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
12
+ spec.metadata["homepage_uri"] = spec.homepage
13
+ spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
14
+ `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
15
+ end
16
+ spec.require_paths = ["lib"]
17
+ end
@@ -0,0 +1,10 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kiwi
4
+ VERSION = "0.1.0"
5
+
6
+ autoload :ByteBuffer, "kiwi/byte_buffer"
7
+ autoload :Definition, "kiwi/definition"
8
+ autoload :Field, "kiwi/field"
9
+ autoload :Schema, "kiwi/schema"
10
+ end
@@ -0,0 +1,129 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kiwi
4
+ class ByteBuffer
5
+ attr_reader :data
6
+ attr_reader :index
7
+
8
+ def initialize(data = [])
9
+ @data = data
10
+ @index = 0
11
+ end
12
+
13
+ def read_byte
14
+ if index >= data.length
15
+ raise RangeError, "Index out of bounds"
16
+ else
17
+ value = data[index]
18
+ @index += 1
19
+ value
20
+ end
21
+ end
22
+
23
+ def read_bytes(len)
24
+ if index + len > data.length
25
+ raise RangeError, "Read bytes out of bounds"
26
+ else
27
+ value = data[index..(index + len)]
28
+ @index += len
29
+ value
30
+ end
31
+ end
32
+
33
+ def read_bool
34
+ read_byte == 0x1
35
+ end
36
+
37
+ def read_var_int
38
+ value = read_var_uint
39
+
40
+ if (value & 1) != 0
41
+ ~(value >> 1)
42
+ else
43
+ value >> 1
44
+ end
45
+ end
46
+
47
+ def read_var_uint
48
+ shift = 0
49
+ result = 0
50
+
51
+ loop do
52
+ byte = read_byte
53
+ result |= (byte & 127) << shift
54
+ shift += 7
55
+
56
+ break if (byte & 128) == 0 || shift >= 35
57
+ end
58
+
59
+ result
60
+ end
61
+
62
+ def read_var_float
63
+ first = read_byte
64
+ return 0.0 if first == 0
65
+ raise RangeError, "Index out of bounds" if index + 3 > data.length
66
+ bits = first | (data[index] << 8) | (data[index + 1] << 16) | (data[index + 2] << 24)
67
+ bits = (bits << 23) | (bits >> 9) # Move the exponent back into place
68
+ value = [bits].pack("L").unpack("F*")[0]
69
+ @index += 3
70
+ value
71
+ end
72
+
73
+ def read_string
74
+ result = []
75
+
76
+ loop do
77
+ char = read_byte
78
+ break if char == 0
79
+ result << char
80
+ end
81
+
82
+ result.pack("C*").force_encoding("UTF-8")
83
+ end
84
+
85
+ def write_bool(value)
86
+ data.push(value ? 0x1 : 0x0)
87
+ end
88
+
89
+ def write_byte(value)
90
+ data.push(value & 255)
91
+ end
92
+
93
+ # Write a variable-length signed 32-bit integer to the end of the buffer.
94
+ def write_var_int(value)
95
+ write_var_uint((value << 1) ^ (value >> 31))
96
+ end
97
+
98
+ def write_var_uint(value)
99
+ loop do
100
+ byte = value & 127
101
+ value = value >> 7
102
+
103
+ if value == 0
104
+ write_byte(byte)
105
+ return
106
+ end
107
+
108
+ write_byte(byte | 128)
109
+ end
110
+ end
111
+
112
+ def write_var_float(value)
113
+ bits = [value].pack("F*").unpack("L")[0]
114
+ bits = (bits >> 23) | (bits << 9)
115
+ bytes = [bits].pack("L").unpack("C*")
116
+
117
+ if (bytes[0] & 255) == 0
118
+ data.push(0)
119
+ else
120
+ data.push(*bytes)
121
+ end
122
+ end
123
+
124
+ def write_string(value)
125
+ data.push(*value.bytes)
126
+ data.push(0)
127
+ end
128
+ end
129
+ end
@@ -0,0 +1,34 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kiwi
4
+ class Definition
5
+ KIND_ENUM = 0
6
+ KIND_STRUCT = 1
7
+ KIND_MESSAGE = 2
8
+
9
+ attr_accessor :name, :index, :kind, :fields
10
+ attr_reader :field_value_to_index, :field_name_to_index
11
+
12
+ def initialize(name:, kind:, fields: [])
13
+ @name = name
14
+ @kind = kind
15
+ @fields = fields
16
+ @index = 0
17
+ @field_name_to_index = {}
18
+ @field_value_to_index = {}
19
+
20
+ @fields.each_with_index do |field, i|
21
+ field_value_to_index[field.value] = i
22
+ field_name_to_index[field.name] = i
23
+ end
24
+ end
25
+
26
+ def field(name)
27
+ if idx = field_name_to_index[name]
28
+ fields[idx]
29
+ else
30
+ nil
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,21 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kiwi
4
+ class Field
5
+ TYPE_BOOL = -1
6
+ TYPE_BYTE = -2
7
+ TYPE_INT = -3
8
+ TYPE_UINT = -4
9
+ TYPE_FLOAT = -5
10
+ TYPE_STRING = -6
11
+
12
+ attr_reader :name, :type_id, :is_array, :value
13
+
14
+ def initialize(name:, type_id:, is_array:, value:)
15
+ @name = name
16
+ @type_id = type_id
17
+ @is_array = is_array
18
+ @value = value
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,179 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kiwi
4
+ class Schema
5
+ def self.from_binary(bytes)
6
+ defs = []
7
+ bb = ByteBuffer.new(bytes)
8
+ definition_count = bb.read_var_uint
9
+
10
+ (0...definition_count).each do
11
+ definition_name = bb.read_string
12
+ kind = bb.read_byte
13
+ field_count = bb.read_var_uint
14
+ fields = []
15
+
16
+ (0...(field_count)).each do |field|
17
+ name = bb.read_string
18
+ type_id = bb.read_var_int
19
+ is_array = bb.read_bool
20
+ value = bb.read_var_uint
21
+ fields << Field.new(name: name, type_id: type_id, is_array: is_array, value: value)
22
+ end
23
+
24
+ defs << Definition.new(name: definition_name, kind: kind, fields: fields)
25
+ end
26
+
27
+ Kiwi::Schema.new(defs: defs)
28
+ end
29
+
30
+ def initialize(defs: [])
31
+ @defs = defs
32
+ @def_name_to_index = {}
33
+
34
+ @defs.each_with_index do |definition, i|
35
+ definition.index = i
36
+ @def_name_to_index[definition.name] = i
37
+
38
+ if definition.kind == Definition::KIND_MESSAGE
39
+ define_singleton_method(:"encode_#{definition.name.downcase}") do |message|
40
+ encode(definition.index, message)
41
+ end
42
+
43
+ define_singleton_method(:"decode_#{definition.name.downcase}") do |bytes|
44
+ decode(definition.index, bytes)
45
+ end
46
+ end
47
+ end
48
+ end
49
+
50
+ def decode(type_id, bytes)
51
+ decode_bb(type_id, ByteBuffer.new(bytes))
52
+ end
53
+
54
+ def encode(type_id, value)
55
+ bb = ByteBuffer.new
56
+ encode_bb(type_id, value, bb)
57
+ bb.data
58
+ end
59
+
60
+ def definitions
61
+ @defs
62
+ end
63
+
64
+ private
65
+
66
+ attr_reader :defs, :def_name_to_index
67
+
68
+ def decode_bb(type_id, byte_buffer)
69
+ case type_id
70
+ when Field::TYPE_BOOL
71
+ byte_buffer.read_bool
72
+ when Field::TYPE_BYTE
73
+ byte_buffer.read_byte
74
+ when Field::TYPE_INT
75
+ byte_buffer.read_var_int
76
+ when Field::TYPE_UINT
77
+ byte_buffer.read_var_uint
78
+ when Field::TYPE_FLOAT
79
+ byte_buffer.read_var_float
80
+ when Field::TYPE_STRING
81
+ byte_buffer.read_string
82
+ else
83
+ definition = defs[type_id]
84
+
85
+ case definition.kind
86
+ when Definition::KIND_ENUM
87
+ if index = definition.field_value_to_index[byte_buffer.read_var_uint]
88
+ definition.fields[index].name
89
+ else
90
+ raise RuntimeError
91
+ end
92
+ when Definition::KIND_STRUCT
93
+ fields = {}
94
+ definition.fields.each do |field|
95
+ fields[field.name] = decode_field(field, byte_buffer)
96
+ end
97
+ fields
98
+ when Definition::KIND_MESSAGE
99
+ fields = {}
100
+ loop do
101
+ value = byte_buffer.read_var_uint
102
+ return fields if value == 0
103
+
104
+ if index = definition.field_value_to_index[value]
105
+ field = definition.fields[index]
106
+ fields[field.name] = decode_field(field, byte_buffer)
107
+ else
108
+ raise RuntimeError
109
+ end
110
+ end
111
+ end
112
+ end
113
+ end
114
+
115
+ def decode_field(field, byte_buffer)
116
+ if field.is_array
117
+ len = byte_buffer.read_var_uint
118
+ array = []
119
+ (0...len).each do
120
+ array << decode_bb(field.type_id, byte_buffer)
121
+ end
122
+ array
123
+ else
124
+ decode_bb(field.type_id, byte_buffer)
125
+ end
126
+ end
127
+
128
+ def encode_bb(type_id, value, byte_buffer)
129
+ case type_id
130
+ when Field::TYPE_BOOL
131
+ byte_buffer.write_bool(value)
132
+ when Field::TYPE_BYTE
133
+ byte_buffer.write_byte(value)
134
+ when Field::TYPE_INT
135
+ byte_buffer.write_var_int(value)
136
+ when Field::TYPE_UINT
137
+ byte_buffer.write_var_uint(value)
138
+ when Field::TYPE_FLOAT
139
+ byte_buffer.write_var_float(value)
140
+ when Field::TYPE_STRING
141
+ byte_buffer.write_string(value)
142
+ else
143
+ definition = defs[type_id]
144
+
145
+ case definition.kind
146
+ when Definition::KIND_ENUM
147
+ enum = definition.field(value)
148
+ byte_buffer.write_var_uint(enum.value)
149
+ when Definition::KIND_STRUCT
150
+ definition.fields.each do |field|
151
+ if !(field_value = value[field.name]).nil?
152
+ encode_value(field, field_value, byte_buffer)
153
+ else
154
+ raise ArgumentError, "missing required field #{field.name}"
155
+ end
156
+ end
157
+ when Definition::KIND_MESSAGE
158
+ value.each do |field_name, field_value|
159
+ field = definition.field(field_name)
160
+ if !field.nil?
161
+ byte_buffer.write_var_uint(field.value)
162
+ encode_value(field, field_value, byte_buffer)
163
+ end
164
+ end
165
+ byte_buffer.write_byte(0)
166
+ end
167
+ end
168
+ end
169
+
170
+ def encode_value(field, value, byte_buffer)
171
+ if field.is_array
172
+ byte_buffer.write_var_uint(value.length)
173
+ value.each { |v| encode_bb(field.type_id, v, byte_buffer) }
174
+ else
175
+ encode_bb(field.type_id, value, byte_buffer)
176
+ end
177
+ end
178
+ end
179
+ end
metadata ADDED
@@ -0,0 +1,56 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: kiwi-schema
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Jan Habermann
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2020-06-22 00:00:00.000000000 Z
12
+ dependencies: []
13
+ description: Kiwi encoding/decoding in ruby
14
+ email:
15
+ - jan@habermann.io
16
+ executables: []
17
+ extensions: []
18
+ extra_rdoc_files: []
19
+ files:
20
+ - ".gitignore"
21
+ - ".travis.yml"
22
+ - Gemfile
23
+ - Gemfile.lock
24
+ - README.md
25
+ - Rakefile
26
+ - bin/console
27
+ - kiwi-schema.gemspec
28
+ - lib/kiwi.rb
29
+ - lib/kiwi/byte_buffer.rb
30
+ - lib/kiwi/definition.rb
31
+ - lib/kiwi/field.rb
32
+ - lib/kiwi/schema.rb
33
+ homepage: https://www.github/com/haberbyte/kiwi-schema
34
+ licenses: []
35
+ metadata:
36
+ homepage_uri: https://www.github/com/haberbyte/kiwi-schema
37
+ post_install_message:
38
+ rdoc_options: []
39
+ require_paths:
40
+ - lib
41
+ required_ruby_version: !ruby/object:Gem::Requirement
42
+ requirements:
43
+ - - ">="
44
+ - !ruby/object:Gem::Version
45
+ version: 2.3.0
46
+ required_rubygems_version: !ruby/object:Gem::Requirement
47
+ requirements:
48
+ - - ">="
49
+ - !ruby/object:Gem::Version
50
+ version: '0'
51
+ requirements: []
52
+ rubygems_version: 3.1.2
53
+ signing_key:
54
+ specification_version: 4
55
+ summary: Kiwi encoding/decoding in ruby
56
+ test_files: []