kiwi-schema 0.1.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of kiwi-schema might be problematic. Click here for more details.

@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 152e96108e00da08616725a10209ae3416c6f9ec611fe4502206d27f33545a7e
4
+ data.tar.gz: 04d75522e8b59fd2135d2e477bc1e2220127abb422e80ed7e31ad9e8a17f8751
5
+ SHA512:
6
+ metadata.gz: 94d70eaa4a840b0c25d5468fd1bcdca0bc880eed42e31efda43f742c397cf9ed3e1e7ce9948e76681fc8eaa50d96a70335435d179bd18ac064e56472d9e89bd6
7
+ data.tar.gz: 48a71dc215afa11eeb16ba94c3aaaff95477964269700c2d2e7b49e430ac3c1b7466e3b720693c8f694b09184a38c8fa31ad069ee9f2bd2ca2a97ef4ea5ba804
@@ -0,0 +1,8 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /_yardoc/
4
+ /coverage/
5
+ /doc/
6
+ /pkg/
7
+ /spec/reports/
8
+ /tmp/
@@ -0,0 +1,6 @@
1
+ ---
2
+ language: ruby
3
+ cache: bundler
4
+ rvm:
5
+ - 2.7.1
6
+ before_install: gem install bundler -v 2.1.4
data/Gemfile ADDED
@@ -0,0 +1,7 @@
1
+ source "https://rubygems.org"
2
+
3
+ # Specify your gem's dependencies in kiwi-schema.gemspec
4
+ gemspec
5
+
6
+ gem "rake", "~> 12.0"
7
+ gem "minitest", "~> 5.0"
@@ -0,0 +1,21 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ kiwi-schema (0.1.0)
5
+
6
+ GEM
7
+ remote: https://rubygems.org/
8
+ specs:
9
+ minitest (5.14.1)
10
+ rake (12.3.3)
11
+
12
+ PLATFORMS
13
+ ruby
14
+
15
+ DEPENDENCIES
16
+ kiwi-schema!
17
+ minitest (~> 5.0)
18
+ rake (~> 12.0)
19
+
20
+ BUNDLED WITH
21
+ 2.1.4
@@ -0,0 +1,43 @@
1
+ # kiwi-schema GEM
2
+
3
+ Kiwi is a schema-based binary format for efficiently encoding trees of data.
4
+
5
+ This is a ruby implementation of the kiwi message format, see [evanw/kiwi](https://github.com/evanw/kiwi/).
6
+
7
+ Note:
8
+
9
+ This gem is a just for fun work-in-progress, probably slow, and not yet proven to work in all cases, so be warned.
10
+ You definitely want to take a look at the original C++, Rust, and JS implementations from Evan first.
11
+
12
+ ## Installation
13
+
14
+ Add this line to your application's Gemfile:
15
+
16
+ ```ruby
17
+ gem 'kiwi-schema'
18
+ ```
19
+
20
+ And then execute:
21
+
22
+ $ bundle install
23
+
24
+ Or install it yourself as:
25
+
26
+ $ gem install kiwi-schema
27
+
28
+ ## Usage
29
+
30
+ ```ruby
31
+ require "kiwi"
32
+
33
+ # This is the encoding of the Kiwi schema "message ABC { int[] xyz = 1; }"
34
+ schema_bytes = [1, 65, 66, 67, 0, 2, 1, 120, 121, 122, 0, 5, 1, 1]
35
+ schema = Kiwi::Schema.from_binary(schema_bytes)
36
+ schema.encode_abc({ "xyz" => [99, -12] }) # => [1, 2, 198, 1, 23, 0]
37
+ schema.decode_abc([1, 2, 198, 1, 23, 0]) # => {"xyz"=>[99, -12]}
38
+ ```
39
+
40
+ ## Contributing
41
+
42
+ Bug reports and pull requests are welcome on GitHub at https://github.com/haberbyte/kiwi-schema.
43
+
@@ -0,0 +1,10 @@
1
+ require "bundler/gem_tasks"
2
+ require "rake/testtask"
3
+
4
+ Rake::TestTask.new(:test) do |t|
5
+ t.libs << "test"
6
+ t.libs << "lib"
7
+ t.test_files = FileList["test/**/*_test.rb"]
8
+ end
9
+
10
+ task :default => :test
@@ -0,0 +1,7 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "kiwi/schema"
5
+
6
+ require "irb"
7
+ IRB.start(__FILE__)
@@ -0,0 +1,17 @@
1
+ require_relative 'lib/kiwi'
2
+
3
+ Gem::Specification.new do |spec|
4
+ spec.name = "kiwi-schema"
5
+ spec.version = Kiwi::VERSION
6
+ spec.authors = ["Jan Habermann"]
7
+ spec.email = ["jan@habermann.io"]
8
+ spec.summary = %q{Kiwi encoding/decoding in ruby}
9
+ spec.description = %q{Kiwi encoding/decoding in ruby}
10
+ spec.homepage = "https://www.github/com/haberbyte/kiwi-schema"
11
+ spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
12
+ spec.metadata["homepage_uri"] = spec.homepage
13
+ spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
14
+ `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
15
+ end
16
+ spec.require_paths = ["lib"]
17
+ end
@@ -0,0 +1,10 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kiwi
4
+ VERSION = "0.1.0"
5
+
6
+ autoload :ByteBuffer, "kiwi/byte_buffer"
7
+ autoload :Definition, "kiwi/definition"
8
+ autoload :Field, "kiwi/field"
9
+ autoload :Schema, "kiwi/schema"
10
+ end
@@ -0,0 +1,129 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kiwi
4
+ class ByteBuffer
5
+ attr_reader :data
6
+ attr_reader :index
7
+
8
+ def initialize(data = [])
9
+ @data = data
10
+ @index = 0
11
+ end
12
+
13
+ def read_byte
14
+ if index >= data.length
15
+ raise RangeError, "Index out of bounds"
16
+ else
17
+ value = data[index]
18
+ @index += 1
19
+ value
20
+ end
21
+ end
22
+
23
+ def read_bytes(len)
24
+ if index + len > data.length
25
+ raise RangeError, "Read bytes out of bounds"
26
+ else
27
+ value = data[index..(index + len)]
28
+ @index += len
29
+ value
30
+ end
31
+ end
32
+
33
+ def read_bool
34
+ read_byte == 0x1
35
+ end
36
+
37
+ def read_var_int
38
+ value = read_var_uint
39
+
40
+ if (value & 1) != 0
41
+ ~(value >> 1)
42
+ else
43
+ value >> 1
44
+ end
45
+ end
46
+
47
+ def read_var_uint
48
+ shift = 0
49
+ result = 0
50
+
51
+ loop do
52
+ byte = read_byte
53
+ result |= (byte & 127) << shift
54
+ shift += 7
55
+
56
+ break if (byte & 128) == 0 || shift >= 35
57
+ end
58
+
59
+ result
60
+ end
61
+
62
+ def read_var_float
63
+ first = read_byte
64
+ return 0.0 if first == 0
65
+ raise RangeError, "Index out of bounds" if index + 3 > data.length
66
+ bits = first | (data[index] << 8) | (data[index + 1] << 16) | (data[index + 2] << 24)
67
+ bits = (bits << 23) | (bits >> 9) # Move the exponent back into place
68
+ value = [bits].pack("L").unpack("F*")[0]
69
+ @index += 3
70
+ value
71
+ end
72
+
73
+ def read_string
74
+ result = []
75
+
76
+ loop do
77
+ char = read_byte
78
+ break if char == 0
79
+ result << char
80
+ end
81
+
82
+ result.pack("C*").force_encoding("UTF-8")
83
+ end
84
+
85
+ def write_bool(value)
86
+ data.push(value ? 0x1 : 0x0)
87
+ end
88
+
89
+ def write_byte(value)
90
+ data.push(value & 255)
91
+ end
92
+
93
+ # Write a variable-length signed 32-bit integer to the end of the buffer.
94
+ def write_var_int(value)
95
+ write_var_uint((value << 1) ^ (value >> 31))
96
+ end
97
+
98
+ def write_var_uint(value)
99
+ loop do
100
+ byte = value & 127
101
+ value = value >> 7
102
+
103
+ if value == 0
104
+ write_byte(byte)
105
+ return
106
+ end
107
+
108
+ write_byte(byte | 128)
109
+ end
110
+ end
111
+
112
+ def write_var_float(value)
113
+ bits = [value].pack("F*").unpack("L")[0]
114
+ bits = (bits >> 23) | (bits << 9)
115
+ bytes = [bits].pack("L").unpack("C*")
116
+
117
+ if (bytes[0] & 255) == 0
118
+ data.push(0)
119
+ else
120
+ data.push(*bytes)
121
+ end
122
+ end
123
+
124
+ def write_string(value)
125
+ data.push(*value.bytes)
126
+ data.push(0)
127
+ end
128
+ end
129
+ end
@@ -0,0 +1,34 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kiwi
4
+ class Definition
5
+ KIND_ENUM = 0
6
+ KIND_STRUCT = 1
7
+ KIND_MESSAGE = 2
8
+
9
+ attr_accessor :name, :index, :kind, :fields
10
+ attr_reader :field_value_to_index, :field_name_to_index
11
+
12
+ def initialize(name:, kind:, fields: [])
13
+ @name = name
14
+ @kind = kind
15
+ @fields = fields
16
+ @index = 0
17
+ @field_name_to_index = {}
18
+ @field_value_to_index = {}
19
+
20
+ @fields.each_with_index do |field, i|
21
+ field_value_to_index[field.value] = i
22
+ field_name_to_index[field.name] = i
23
+ end
24
+ end
25
+
26
+ def field(name)
27
+ if idx = field_name_to_index[name]
28
+ fields[idx]
29
+ else
30
+ nil
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,21 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kiwi
4
+ class Field
5
+ TYPE_BOOL = -1
6
+ TYPE_BYTE = -2
7
+ TYPE_INT = -3
8
+ TYPE_UINT = -4
9
+ TYPE_FLOAT = -5
10
+ TYPE_STRING = -6
11
+
12
+ attr_reader :name, :type_id, :is_array, :value
13
+
14
+ def initialize(name:, type_id:, is_array:, value:)
15
+ @name = name
16
+ @type_id = type_id
17
+ @is_array = is_array
18
+ @value = value
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,179 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kiwi
4
+ class Schema
5
+ def self.from_binary(bytes)
6
+ defs = []
7
+ bb = ByteBuffer.new(bytes)
8
+ definition_count = bb.read_var_uint
9
+
10
+ (0...definition_count).each do
11
+ definition_name = bb.read_string
12
+ kind = bb.read_byte
13
+ field_count = bb.read_var_uint
14
+ fields = []
15
+
16
+ (0...(field_count)).each do |field|
17
+ name = bb.read_string
18
+ type_id = bb.read_var_int
19
+ is_array = bb.read_bool
20
+ value = bb.read_var_uint
21
+ fields << Field.new(name: name, type_id: type_id, is_array: is_array, value: value)
22
+ end
23
+
24
+ defs << Definition.new(name: definition_name, kind: kind, fields: fields)
25
+ end
26
+
27
+ Kiwi::Schema.new(defs: defs)
28
+ end
29
+
30
+ def initialize(defs: [])
31
+ @defs = defs
32
+ @def_name_to_index = {}
33
+
34
+ @defs.each_with_index do |definition, i|
35
+ definition.index = i
36
+ @def_name_to_index[definition.name] = i
37
+
38
+ if definition.kind == Definition::KIND_MESSAGE
39
+ define_singleton_method(:"encode_#{definition.name.downcase}") do |message|
40
+ encode(definition.index, message)
41
+ end
42
+
43
+ define_singleton_method(:"decode_#{definition.name.downcase}") do |bytes|
44
+ decode(definition.index, bytes)
45
+ end
46
+ end
47
+ end
48
+ end
49
+
50
+ def decode(type_id, bytes)
51
+ decode_bb(type_id, ByteBuffer.new(bytes))
52
+ end
53
+
54
+ def encode(type_id, value)
55
+ bb = ByteBuffer.new
56
+ encode_bb(type_id, value, bb)
57
+ bb.data
58
+ end
59
+
60
+ def definitions
61
+ @defs
62
+ end
63
+
64
+ private
65
+
66
+ attr_reader :defs, :def_name_to_index
67
+
68
+ def decode_bb(type_id, byte_buffer)
69
+ case type_id
70
+ when Field::TYPE_BOOL
71
+ byte_buffer.read_bool
72
+ when Field::TYPE_BYTE
73
+ byte_buffer.read_byte
74
+ when Field::TYPE_INT
75
+ byte_buffer.read_var_int
76
+ when Field::TYPE_UINT
77
+ byte_buffer.read_var_uint
78
+ when Field::TYPE_FLOAT
79
+ byte_buffer.read_var_float
80
+ when Field::TYPE_STRING
81
+ byte_buffer.read_string
82
+ else
83
+ definition = defs[type_id]
84
+
85
+ case definition.kind
86
+ when Definition::KIND_ENUM
87
+ if index = definition.field_value_to_index[byte_buffer.read_var_uint]
88
+ definition.fields[index].name
89
+ else
90
+ raise RuntimeError
91
+ end
92
+ when Definition::KIND_STRUCT
93
+ fields = {}
94
+ definition.fields.each do |field|
95
+ fields[field.name] = decode_field(field, byte_buffer)
96
+ end
97
+ fields
98
+ when Definition::KIND_MESSAGE
99
+ fields = {}
100
+ loop do
101
+ value = byte_buffer.read_var_uint
102
+ return fields if value == 0
103
+
104
+ if index = definition.field_value_to_index[value]
105
+ field = definition.fields[index]
106
+ fields[field.name] = decode_field(field, byte_buffer)
107
+ else
108
+ raise RuntimeError
109
+ end
110
+ end
111
+ end
112
+ end
113
+ end
114
+
115
+ def decode_field(field, byte_buffer)
116
+ if field.is_array
117
+ len = byte_buffer.read_var_uint
118
+ array = []
119
+ (0...len).each do
120
+ array << decode_bb(field.type_id, byte_buffer)
121
+ end
122
+ array
123
+ else
124
+ decode_bb(field.type_id, byte_buffer)
125
+ end
126
+ end
127
+
128
+ def encode_bb(type_id, value, byte_buffer)
129
+ case type_id
130
+ when Field::TYPE_BOOL
131
+ byte_buffer.write_bool(value)
132
+ when Field::TYPE_BYTE
133
+ byte_buffer.write_byte(value)
134
+ when Field::TYPE_INT
135
+ byte_buffer.write_var_int(value)
136
+ when Field::TYPE_UINT
137
+ byte_buffer.write_var_uint(value)
138
+ when Field::TYPE_FLOAT
139
+ byte_buffer.write_var_float(value)
140
+ when Field::TYPE_STRING
141
+ byte_buffer.write_string(value)
142
+ else
143
+ definition = defs[type_id]
144
+
145
+ case definition.kind
146
+ when Definition::KIND_ENUM
147
+ enum = definition.field(value)
148
+ byte_buffer.write_var_uint(enum.value)
149
+ when Definition::KIND_STRUCT
150
+ definition.fields.each do |field|
151
+ if !(field_value = value[field.name]).nil?
152
+ encode_value(field, field_value, byte_buffer)
153
+ else
154
+ raise ArgumentError, "missing required field #{field.name}"
155
+ end
156
+ end
157
+ when Definition::KIND_MESSAGE
158
+ value.each do |field_name, field_value|
159
+ field = definition.field(field_name)
160
+ if !field.nil?
161
+ byte_buffer.write_var_uint(field.value)
162
+ encode_value(field, field_value, byte_buffer)
163
+ end
164
+ end
165
+ byte_buffer.write_byte(0)
166
+ end
167
+ end
168
+ end
169
+
170
+ def encode_value(field, value, byte_buffer)
171
+ if field.is_array
172
+ byte_buffer.write_var_uint(value.length)
173
+ value.each { |v| encode_bb(field.type_id, v, byte_buffer) }
174
+ else
175
+ encode_bb(field.type_id, value, byte_buffer)
176
+ end
177
+ end
178
+ end
179
+ end
metadata ADDED
@@ -0,0 +1,56 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: kiwi-schema
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Jan Habermann
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2020-06-22 00:00:00.000000000 Z
12
+ dependencies: []
13
+ description: Kiwi encoding/decoding in ruby
14
+ email:
15
+ - jan@habermann.io
16
+ executables: []
17
+ extensions: []
18
+ extra_rdoc_files: []
19
+ files:
20
+ - ".gitignore"
21
+ - ".travis.yml"
22
+ - Gemfile
23
+ - Gemfile.lock
24
+ - README.md
25
+ - Rakefile
26
+ - bin/console
27
+ - kiwi-schema.gemspec
28
+ - lib/kiwi.rb
29
+ - lib/kiwi/byte_buffer.rb
30
+ - lib/kiwi/definition.rb
31
+ - lib/kiwi/field.rb
32
+ - lib/kiwi/schema.rb
33
+ homepage: https://www.github/com/haberbyte/kiwi-schema
34
+ licenses: []
35
+ metadata:
36
+ homepage_uri: https://www.github/com/haberbyte/kiwi-schema
37
+ post_install_message:
38
+ rdoc_options: []
39
+ require_paths:
40
+ - lib
41
+ required_ruby_version: !ruby/object:Gem::Requirement
42
+ requirements:
43
+ - - ">="
44
+ - !ruby/object:Gem::Version
45
+ version: 2.3.0
46
+ required_rubygems_version: !ruby/object:Gem::Requirement
47
+ requirements:
48
+ - - ">="
49
+ - !ruby/object:Gem::Version
50
+ version: '0'
51
+ requirements: []
52
+ rubygems_version: 3.1.2
53
+ signing_key:
54
+ specification_version: 4
55
+ summary: Kiwi encoding/decoding in ruby
56
+ test_files: []