field_struct-avro-native 1.0.0.pre1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +5 -0
- data/Cargo.lock +286 -0
- data/Cargo.toml +7 -0
- data/LICENSE.txt +21 -0
- data/README.md +122 -0
- data/ext/field_struct_avro_kernel/Cargo.toml +9 -0
- data/ext/field_struct_avro_kernel/src/decimal.rs +150 -0
- data/ext/field_struct_avro_kernel/src/lib.rs +48 -0
- data/ext/field_struct_avro_native/Cargo.toml +13 -0
- data/ext/field_struct_avro_native/extconf.rb +6 -0
- data/ext/field_struct_avro_native/src/lib.rs +47 -0
- data/ext/field_struct_avro_native/src/plan.rs +157 -0
- data/ext/field_struct_avro_native/src/write.rs +435 -0
- data/lib/field_struct/avro/native/compiler.rb +124 -0
- data/lib/field_struct/avro/native/version.rb +12 -0
- data/lib/field_struct/avro/native.rb +76 -0
- data/sig/field_struct-avro-native.rbs +61 -0
- metadata +98 -0
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FieldStruct
|
|
4
|
+
module Avro
|
|
5
|
+
module Native
|
|
6
|
+
# Turns a parsed +Avro::Schema+ into the flat node list +Plan.new+ takes, or +nil+ for a schema with a shape the
|
|
7
|
+
# native encoder does not cover (then every encode with it runs the pure path).
|
|
8
|
+
#
|
|
9
|
+
# Each node is an Array starting with a tag; child nodes are referenced by index, so a recursive record is a
|
|
10
|
+
# node that points back at itself.
|
|
11
|
+
class Compiler
|
|
12
|
+
# @return [Hash{Symbol => Integer}] the tag number of each node kind
|
|
13
|
+
TAGS = {
|
|
14
|
+
null: 0, boolean: 1, int: 2, long: 3, float: 4, double: 5, bytes: 6, string: 7,
|
|
15
|
+
decimal: 8, date: 9, timestamp: 10, enum: 11, array: 12, map: 13, record: 14, union: 15
|
|
16
|
+
}.freeze
|
|
17
|
+
|
|
18
|
+
# Ticks per second of each timestamp logical type.
|
|
19
|
+
#
|
|
20
|
+
# @return [Hash{String => Integer}]
|
|
21
|
+
TICKS = { 'timestamp-millis' => 1_000, 'timestamp-micros' => 1_000_000, 'timestamp-nanos' => 1_000_000_000 }.freeze
|
|
22
|
+
|
|
23
|
+
# The logical types written with a conversion; any other logical type must use the identity adapter.
|
|
24
|
+
#
|
|
25
|
+
# @return [Hash{String => Array(Symbol, String)}] base type and adapter name
|
|
26
|
+
CONVERTED = {
|
|
27
|
+
'decimal' => [:bytes, 'BytesDecimal'], 'date' => [:int, 'IntDate'],
|
|
28
|
+
'timestamp-millis' => [:long, 'TimestampMillis'], 'timestamp-micros' => [:long, 'TimestampMicros'],
|
|
29
|
+
'timestamp-nanos' => [:long, 'TimestampNanos']
|
|
30
|
+
}.freeze
|
|
31
|
+
|
|
32
|
+
# @return [Array<Symbol>]
|
|
33
|
+
PRIMITIVES = %i[null boolean int long float double bytes string].freeze
|
|
34
|
+
# @return [Array<Symbol>]
|
|
35
|
+
COMPOSITES = %i[record enum array map union].freeze
|
|
36
|
+
|
|
37
|
+
# Raised inside the walk for a shape with no native encoding.
|
|
38
|
+
class Unsupported < StandardError; end
|
|
39
|
+
|
|
40
|
+
# @param schema [::Object] a parsed +Avro::Schema+
|
|
41
|
+
# @return [Array(::Array<::Array<::Object>>, ::Integer), nil] the nodes and the root index, or nil when unsupported
|
|
42
|
+
def self.call(schema)
|
|
43
|
+
compiler = new
|
|
44
|
+
root = compiler.index(schema)
|
|
45
|
+
[compiler.nodes, root]
|
|
46
|
+
rescue Unsupported
|
|
47
|
+
nil
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# @return [::Array<::Array<::Object>>]
|
|
51
|
+
attr_reader :nodes
|
|
52
|
+
|
|
53
|
+
def initialize
|
|
54
|
+
@nodes = []
|
|
55
|
+
@seen = {}.compare_by_identity
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# @param schema [::Object] a parsed +Avro::Schema+
|
|
59
|
+
# @return [::Integer] the node index of the schema, compiling it on first sight
|
|
60
|
+
def index(schema)
|
|
61
|
+
found = @seen[schema]
|
|
62
|
+
return found if found
|
|
63
|
+
|
|
64
|
+
at = @nodes.size
|
|
65
|
+
@seen[schema] = at
|
|
66
|
+
@nodes << nil
|
|
67
|
+
@nodes[at] = node(schema)
|
|
68
|
+
at
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
private
|
|
72
|
+
|
|
73
|
+
def node(schema)
|
|
74
|
+
type = schema.type_sym
|
|
75
|
+
return primitive(schema, type) if PRIMITIVES.include?(type)
|
|
76
|
+
raise Unsupported unless COMPOSITES.include?(type)
|
|
77
|
+
|
|
78
|
+
[TAGS.fetch(type), send(type, schema)]
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def record(schema) = schema.fields.map { |field| [field.name.to_s, index(field.type)] }
|
|
82
|
+
def enum(schema) = schema.symbols.map(&:to_s)
|
|
83
|
+
def array(schema) = index(schema.items)
|
|
84
|
+
def map(schema) = index(schema.values)
|
|
85
|
+
def union(schema) = schema.schemas.map { |member| index(member) }
|
|
86
|
+
|
|
87
|
+
def primitive(schema, type)
|
|
88
|
+
logical = schema.logical_type
|
|
89
|
+
return [TAGS.fetch(type)] if logical.nil? || identity?(schema)
|
|
90
|
+
|
|
91
|
+
base, adapter = CONVERTED[logical]
|
|
92
|
+
raise Unsupported unless base == type && adapter?(schema.type_adapter, ::Avro::LogicalTypes.const_get(adapter))
|
|
93
|
+
|
|
94
|
+
converted(schema, logical)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def converted(schema, logical)
|
|
98
|
+
case logical
|
|
99
|
+
when 'decimal' then decimal(schema)
|
|
100
|
+
when 'date' then [TAGS[:date]]
|
|
101
|
+
else [TAGS[:timestamp], TICKS.fetch(logical)]
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def decimal(schema)
|
|
106
|
+
precision = schema.precision
|
|
107
|
+
scale = schema.scale || 0
|
|
108
|
+
raise Unsupported unless precision.is_a?(::Integer) && precision.positive? && scale.is_a?(::Integer) && scale >= 0
|
|
109
|
+
|
|
110
|
+
[TAGS[:decimal], precision, scale]
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# The avro gem keeps some adapters as modules and builds others per schema (decimal).
|
|
114
|
+
def adapter?(adapter, expected)
|
|
115
|
+
adapter.equal?(expected) || adapter.instance_of?(expected)
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def identity?(schema)
|
|
119
|
+
schema.type_adapter.equal?(::Avro::LogicalTypes::Identity)
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
end
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'bigdecimal'
|
|
4
|
+
require 'date'
|
|
5
|
+
require 'field_struct/avro'
|
|
6
|
+
require_relative 'native/version'
|
|
7
|
+
require_relative 'native/compiler'
|
|
8
|
+
|
|
9
|
+
# A precompiled platform gem keeps one binary per Ruby minor version in a folder named for it; a source build
|
|
10
|
+
# (and `rake compile`) puts the single binary next to this file.
|
|
11
|
+
begin
|
|
12
|
+
require_relative "#{RUBY_VERSION[/\A\d+\.\d+/]}/field_struct_avro_native"
|
|
13
|
+
rescue LoadError
|
|
14
|
+
require_relative 'field_struct_avro_native'
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
# The field_struct namespace.
|
|
18
|
+
module FieldStruct
|
|
19
|
+
# Apache Avro for field_struct classes.
|
|
20
|
+
module Avro
|
|
21
|
+
# An optional Rust encoder behind +FieldStruct::Avro.encode+. Requiring this file swaps the binary write: the
|
|
22
|
+
# datum +to_datum+ built is written by an encoding plan compiled once per parsed writer schema. A datum the plan is
|
|
23
|
+
# not sure about (a value of an unexpected class, out of range, needing rounding) is written by the pure-Ruby path
|
|
24
|
+
# instead, so the bytes and every error stay what the pure path gives.
|
|
25
|
+
module Native
|
|
26
|
+
# The instance variable a parsed schema keeps its compiled plan in, so the plan lives and dies with the
|
|
27
|
+
# schema the adapter's cache holds (a rebuilt or cleared entry gets a new schema and a new plan).
|
|
28
|
+
PLAN_IVAR = :@field_struct_avro_native_plan
|
|
29
|
+
private_constant :PLAN_IVAR
|
|
30
|
+
|
|
31
|
+
class << self
|
|
32
|
+
# @return [Boolean] whether +FieldStruct::Avro.encode+ goes through the native encoder
|
|
33
|
+
def enabled?
|
|
34
|
+
FieldStruct::Avro.singleton_class.ancestors.include?(Write)
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# The Avro binary of a datum, or nil when the native encoder leaves it to the pure path.
|
|
38
|
+
#
|
|
39
|
+
# @param schema [::Object] a parsed +Avro::Schema+
|
|
40
|
+
# @param datum [::Object] what +FieldStruct::Avro.to_datum+ returns for that schema
|
|
41
|
+
# @return [::String, nil] binary bytes, or nil
|
|
42
|
+
def encode(schema, datum)
|
|
43
|
+
plan = plan_for(schema)
|
|
44
|
+
plan&.encode(datum)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
private
|
|
48
|
+
|
|
49
|
+
def plan_for(schema)
|
|
50
|
+
plan = schema.instance_variable_get(PLAN_IVAR)
|
|
51
|
+
return plan || nil unless plan.nil?
|
|
52
|
+
|
|
53
|
+
plan = compile(schema)
|
|
54
|
+
schema.instance_variable_set(PLAN_IVAR, plan || false) unless schema.frozen?
|
|
55
|
+
plan
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def compile(schema)
|
|
59
|
+
nodes, root = Compiler.call(schema)
|
|
60
|
+
nodes && Plan.new(nodes, root)
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# Prepended to +FieldStruct::Avro+'s singleton class: the binary write tries the plan first.
|
|
65
|
+
module Write
|
|
66
|
+
private
|
|
67
|
+
|
|
68
|
+
def write(parsed, datum, struct)
|
|
69
|
+
Native.encode(parsed, datum) || super
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
FieldStruct::Avro.singleton_class.prepend(Write)
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# The field_struct namespace.
|
|
2
|
+
module FieldStruct
|
|
3
|
+
# Apache Avro for field_struct classes.
|
|
4
|
+
module Avro
|
|
5
|
+
# An optional Rust encoder behind +FieldStruct::Avro.encode+. Requiring this file swaps the binary write: the
|
|
6
|
+
# datum +to_datum+ built is written by an encoding plan compiled once per parsed writer schema. A datum the plan is
|
|
7
|
+
# not sure about (a value of an unexpected class, out of range, needing rounding) is written by the pure-Ruby path
|
|
8
|
+
# instead, so the bytes and every error stay what the pure path gives.
|
|
9
|
+
module Native
|
|
10
|
+
include FieldStruct::Avro::Native::Write
|
|
11
|
+
VERSION: ::String
|
|
12
|
+
|
|
13
|
+
# _@return_ — whether +FieldStruct::Avro.encode+ goes through the native encoder
|
|
14
|
+
def self.enabled?: () -> bool
|
|
15
|
+
|
|
16
|
+
# The Avro binary of a datum, or nil when the native encoder leaves it to the pure path.
|
|
17
|
+
#
|
|
18
|
+
# _@param_ `schema` — a parsed +Avro::Schema+
|
|
19
|
+
#
|
|
20
|
+
# _@param_ `datum` — what +FieldStruct::Avro.to_datum+ returns for that schema
|
|
21
|
+
#
|
|
22
|
+
# _@return_ — binary bytes, or nil
|
|
23
|
+
def self.encode: (::Object schema, ::Object datum) -> ::String?
|
|
24
|
+
|
|
25
|
+
# Prepended to +FieldStruct::Avro+'s singleton class: the binary write tries the plan first.
|
|
26
|
+
module Write
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Turns a parsed +Avro::Schema+ into the flat node list +Plan.new+ takes, or +nil+ for a schema with a shape the
|
|
30
|
+
# native encoder does not cover (then every encode with it runs the pure path).
|
|
31
|
+
#
|
|
32
|
+
# Each node is an Array starting with a tag; child nodes are referenced by index, so a recursive record is a
|
|
33
|
+
# node that points back at itself.
|
|
34
|
+
class Compiler
|
|
35
|
+
TAGS: ::Hash[Symbol, Integer]
|
|
36
|
+
TICKS: ::Hash[String, Integer]
|
|
37
|
+
CONVERTED: ::Hash[String, [Symbol, String]]
|
|
38
|
+
PRIMITIVES: ::Array[Symbol]
|
|
39
|
+
COMPOSITES: ::Array[Symbol]
|
|
40
|
+
|
|
41
|
+
# _@param_ `schema` — a parsed +Avro::Schema+
|
|
42
|
+
#
|
|
43
|
+
# _@return_ — the nodes and the root index, or nil when unsupported
|
|
44
|
+
def self.call: (::Object schema) -> [::Array[::Array[::Object]], ::Integer]?
|
|
45
|
+
|
|
46
|
+
def initialize: () -> void
|
|
47
|
+
|
|
48
|
+
# _@param_ `schema` — a parsed +Avro::Schema+
|
|
49
|
+
#
|
|
50
|
+
# _@return_ — the node index of the schema, compiling it on first sight
|
|
51
|
+
def index: (::Object schema) -> ::Integer
|
|
52
|
+
|
|
53
|
+
attr_reader nodes: ::Array[::Array[::Object]]
|
|
54
|
+
|
|
55
|
+
# Raised inside the walk for a shape with no native encoding.
|
|
56
|
+
class Unsupported < StandardError
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: field_struct-avro-native
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 1.0.0.pre1
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Adrian Madrid
|
|
8
|
+
bindir: bin
|
|
9
|
+
cert_chain: []
|
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
|
+
dependencies:
|
|
12
|
+
- !ruby/object:Gem::Dependency
|
|
13
|
+
name: field_struct-avro
|
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
|
15
|
+
requirements:
|
|
16
|
+
- - ">="
|
|
17
|
+
- !ruby/object:Gem::Version
|
|
18
|
+
version: 1.0.0.pre1
|
|
19
|
+
- - "<"
|
|
20
|
+
- !ruby/object:Gem::Version
|
|
21
|
+
version: '2'
|
|
22
|
+
type: :runtime
|
|
23
|
+
prerelease: false
|
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
25
|
+
requirements:
|
|
26
|
+
- - ">="
|
|
27
|
+
- !ruby/object:Gem::Version
|
|
28
|
+
version: 1.0.0.pre1
|
|
29
|
+
- - "<"
|
|
30
|
+
- !ruby/object:Gem::Version
|
|
31
|
+
version: '2'
|
|
32
|
+
- !ruby/object:Gem::Dependency
|
|
33
|
+
name: rb_sys
|
|
34
|
+
requirement: !ruby/object:Gem::Requirement
|
|
35
|
+
requirements:
|
|
36
|
+
- - "~>"
|
|
37
|
+
- !ruby/object:Gem::Version
|
|
38
|
+
version: '0.9'
|
|
39
|
+
type: :runtime
|
|
40
|
+
prerelease: false
|
|
41
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
42
|
+
requirements:
|
|
43
|
+
- - "~>"
|
|
44
|
+
- !ruby/object:Gem::Version
|
|
45
|
+
version: '0.9'
|
|
46
|
+
description: Compiles each writer schema into an encoding plan and writes field_struct-avro's
|
|
47
|
+
datums to Avro binary in Rust. Anything it is not sure about goes to the pure-Ruby
|
|
48
|
+
path, so output and errors stay the same.
|
|
49
|
+
email:
|
|
50
|
+
- adrian.madrid@timeworxs.com
|
|
51
|
+
executables: []
|
|
52
|
+
extensions:
|
|
53
|
+
- ext/field_struct_avro_native/extconf.rb
|
|
54
|
+
extra_rdoc_files: []
|
|
55
|
+
files:
|
|
56
|
+
- CHANGELOG.md
|
|
57
|
+
- Cargo.lock
|
|
58
|
+
- Cargo.toml
|
|
59
|
+
- LICENSE.txt
|
|
60
|
+
- README.md
|
|
61
|
+
- ext/field_struct_avro_kernel/Cargo.toml
|
|
62
|
+
- ext/field_struct_avro_kernel/src/decimal.rs
|
|
63
|
+
- ext/field_struct_avro_kernel/src/lib.rs
|
|
64
|
+
- ext/field_struct_avro_native/Cargo.toml
|
|
65
|
+
- ext/field_struct_avro_native/extconf.rb
|
|
66
|
+
- ext/field_struct_avro_native/src/lib.rs
|
|
67
|
+
- ext/field_struct_avro_native/src/plan.rs
|
|
68
|
+
- ext/field_struct_avro_native/src/write.rs
|
|
69
|
+
- lib/field_struct/avro/native.rb
|
|
70
|
+
- lib/field_struct/avro/native/compiler.rb
|
|
71
|
+
- lib/field_struct/avro/native/version.rb
|
|
72
|
+
- sig/field_struct-avro-native.rbs
|
|
73
|
+
homepage: https://github.com/gliders-rb/gems
|
|
74
|
+
licenses:
|
|
75
|
+
- MIT
|
|
76
|
+
metadata:
|
|
77
|
+
source_code_uri: https://github.com/gliders-rb/gems/tree/main/adapters/field_struct-avro-native
|
|
78
|
+
changelog_uri: https://github.com/gliders-rb/gems/blob/main/adapters/field_struct-avro-native/CHANGELOG.md
|
|
79
|
+
rubygems_mfa_required: 'true'
|
|
80
|
+
rdoc_options: []
|
|
81
|
+
require_paths:
|
|
82
|
+
- lib
|
|
83
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
84
|
+
requirements:
|
|
85
|
+
- - ">="
|
|
86
|
+
- !ruby/object:Gem::Version
|
|
87
|
+
version: '3.3'
|
|
88
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
89
|
+
requirements:
|
|
90
|
+
- - ">="
|
|
91
|
+
- !ruby/object:Gem::Version
|
|
92
|
+
version: '0'
|
|
93
|
+
requirements: []
|
|
94
|
+
rubygems_version: 3.6.9
|
|
95
|
+
specification_version: 4
|
|
96
|
+
summary: 'An optional Rust encoder for field_struct-avro: the same Avro bytes, written
|
|
97
|
+
faster.'
|
|
98
|
+
test_files: []
|