field_struct-avro-native 1.0.0.pre1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,124 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FieldStruct
4
+ module Avro
5
+ module Native
6
+ # Turns a parsed +Avro::Schema+ into the flat node list +Plan.new+ takes, or +nil+ for a schema with a shape the
7
+ # native encoder does not cover (then every encode with it runs the pure path).
8
+ #
9
+ # Each node is an Array starting with a tag; child nodes are referenced by index, so a recursive record is a
10
+ # node that points back at itself.
11
+ class Compiler
12
+ # @return [Hash{Symbol => Integer}] the tag number of each node kind
13
+ TAGS = {
14
+ null: 0, boolean: 1, int: 2, long: 3, float: 4, double: 5, bytes: 6, string: 7,
15
+ decimal: 8, date: 9, timestamp: 10, enum: 11, array: 12, map: 13, record: 14, union: 15
16
+ }.freeze
17
+
18
+ # Ticks per second of each timestamp logical type.
19
+ #
20
+ # @return [Hash{String => Integer}]
21
+ TICKS = { 'timestamp-millis' => 1_000, 'timestamp-micros' => 1_000_000, 'timestamp-nanos' => 1_000_000_000 }.freeze
22
+
23
+ # The logical types written with a conversion; any other logical type must use the identity adapter.
24
+ #
25
+ # @return [Hash{String => Array(Symbol, String)}] base type and adapter name
26
+ CONVERTED = {
27
+ 'decimal' => [:bytes, 'BytesDecimal'], 'date' => [:int, 'IntDate'],
28
+ 'timestamp-millis' => [:long, 'TimestampMillis'], 'timestamp-micros' => [:long, 'TimestampMicros'],
29
+ 'timestamp-nanos' => [:long, 'TimestampNanos']
30
+ }.freeze
31
+
32
+ # @return [Array<Symbol>]
33
+ PRIMITIVES = %i[null boolean int long float double bytes string].freeze
34
+ # @return [Array<Symbol>]
35
+ COMPOSITES = %i[record enum array map union].freeze
36
+
37
+ # Raised inside the walk for a shape with no native encoding.
38
+ class Unsupported < StandardError; end
39
+
40
+ # @param schema [::Object] a parsed +Avro::Schema+
41
+ # @return [Array(::Array<::Array<::Object>>, ::Integer), nil] the nodes and the root index, or nil when unsupported
42
+ def self.call(schema)
43
+ compiler = new
44
+ root = compiler.index(schema)
45
+ [compiler.nodes, root]
46
+ rescue Unsupported
47
+ nil
48
+ end
49
+
50
+ # @return [::Array<::Array<::Object>>]
51
+ attr_reader :nodes
52
+
53
+ def initialize
54
+ @nodes = []
55
+ @seen = {}.compare_by_identity
56
+ end
57
+
58
+ # @param schema [::Object] a parsed +Avro::Schema+
59
+ # @return [::Integer] the node index of the schema, compiling it on first sight
60
+ def index(schema)
61
+ found = @seen[schema]
62
+ return found if found
63
+
64
+ at = @nodes.size
65
+ @seen[schema] = at
66
+ @nodes << nil
67
+ @nodes[at] = node(schema)
68
+ at
69
+ end
70
+
71
+ private
72
+
73
+ def node(schema)
74
+ type = schema.type_sym
75
+ return primitive(schema, type) if PRIMITIVES.include?(type)
76
+ raise Unsupported unless COMPOSITES.include?(type)
77
+
78
+ [TAGS.fetch(type), send(type, schema)]
79
+ end
80
+
81
+ def record(schema) = schema.fields.map { |field| [field.name.to_s, index(field.type)] }
82
+ def enum(schema) = schema.symbols.map(&:to_s)
83
+ def array(schema) = index(schema.items)
84
+ def map(schema) = index(schema.values)
85
+ def union(schema) = schema.schemas.map { |member| index(member) }
86
+
87
+ def primitive(schema, type)
88
+ logical = schema.logical_type
89
+ return [TAGS.fetch(type)] if logical.nil? || identity?(schema)
90
+
91
+ base, adapter = CONVERTED[logical]
92
+ raise Unsupported unless base == type && adapter?(schema.type_adapter, ::Avro::LogicalTypes.const_get(adapter))
93
+
94
+ converted(schema, logical)
95
+ end
96
+
97
+ def converted(schema, logical)
98
+ case logical
99
+ when 'decimal' then decimal(schema)
100
+ when 'date' then [TAGS[:date]]
101
+ else [TAGS[:timestamp], TICKS.fetch(logical)]
102
+ end
103
+ end
104
+
105
+ def decimal(schema)
106
+ precision = schema.precision
107
+ scale = schema.scale || 0
108
+ raise Unsupported unless precision.is_a?(::Integer) && precision.positive? && scale.is_a?(::Integer) && scale >= 0
109
+
110
+ [TAGS[:decimal], precision, scale]
111
+ end
112
+
113
+ # The avro gem keeps some adapters as modules and builds others per schema (decimal).
114
+ def adapter?(adapter, expected)
115
+ adapter.equal?(expected) || adapter.instance_of?(expected)
116
+ end
117
+
118
+ def identity?(schema)
119
+ schema.type_adapter.equal?(::Avro::LogicalTypes::Identity)
120
+ end
121
+ end
122
+ end
123
+ end
124
+ end
@@ -0,0 +1,12 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FieldStruct
4
+ module Avro
5
+ module Native
6
+ # The gem version.
7
+ #
8
+ # @return [::String]
9
+ VERSION = '1.0.0.pre1'
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,76 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'bigdecimal'
4
+ require 'date'
5
+ require 'field_struct/avro'
6
+ require_relative 'native/version'
7
+ require_relative 'native/compiler'
8
+
9
+ # A precompiled platform gem keeps one binary per Ruby minor version in a folder named for it; a source build
10
+ # (and `rake compile`) puts the single binary next to this file.
11
+ begin
12
+ require_relative "#{RUBY_VERSION[/\A\d+\.\d+/]}/field_struct_avro_native"
13
+ rescue LoadError
14
+ require_relative 'field_struct_avro_native'
15
+ end
16
+
17
+ # The field_struct namespace.
18
+ module FieldStruct
19
+ # Apache Avro for field_struct classes.
20
+ module Avro
21
+ # An optional Rust encoder behind +FieldStruct::Avro.encode+. Requiring this file swaps the binary write: the
22
+ # datum +to_datum+ built is written by an encoding plan compiled once per parsed writer schema. A datum the plan is
23
+ # not sure about (a value of an unexpected class, out of range, needing rounding) is written by the pure-Ruby path
24
+ # instead, so the bytes and every error stay what the pure path gives.
25
+ module Native
26
+ # The instance variable a parsed schema keeps its compiled plan in, so the plan lives and dies with the
27
+ # schema the adapter's cache holds (a rebuilt or cleared entry gets a new schema and a new plan).
28
+ PLAN_IVAR = :@field_struct_avro_native_plan
29
+ private_constant :PLAN_IVAR
30
+
31
+ class << self
32
+ # @return [Boolean] whether +FieldStruct::Avro.encode+ goes through the native encoder
33
+ def enabled?
34
+ FieldStruct::Avro.singleton_class.ancestors.include?(Write)
35
+ end
36
+
37
+ # The Avro binary of a datum, or nil when the native encoder leaves it to the pure path.
38
+ #
39
+ # @param schema [::Object] a parsed +Avro::Schema+
40
+ # @param datum [::Object] what +FieldStruct::Avro.to_datum+ returns for that schema
41
+ # @return [::String, nil] binary bytes, or nil
42
+ def encode(schema, datum)
43
+ plan = plan_for(schema)
44
+ plan&.encode(datum)
45
+ end
46
+
47
+ private
48
+
49
+ def plan_for(schema)
50
+ plan = schema.instance_variable_get(PLAN_IVAR)
51
+ return plan || nil unless plan.nil?
52
+
53
+ plan = compile(schema)
54
+ schema.instance_variable_set(PLAN_IVAR, plan || false) unless schema.frozen?
55
+ plan
56
+ end
57
+
58
+ def compile(schema)
59
+ nodes, root = Compiler.call(schema)
60
+ nodes && Plan.new(nodes, root)
61
+ end
62
+ end
63
+
64
+ # Prepended to +FieldStruct::Avro+'s singleton class: the binary write tries the plan first.
65
+ module Write
66
+ private
67
+
68
+ def write(parsed, datum, struct)
69
+ Native.encode(parsed, datum) || super
70
+ end
71
+ end
72
+
73
+ FieldStruct::Avro.singleton_class.prepend(Write)
74
+ end
75
+ end
76
+ end
@@ -0,0 +1,61 @@
1
+ # The field_struct namespace.
2
+ module FieldStruct
3
+ # Apache Avro for field_struct classes.
4
+ module Avro
5
+ # An optional Rust encoder behind +FieldStruct::Avro.encode+. Requiring this file swaps the binary write: the
6
+ # datum +to_datum+ built is written by an encoding plan compiled once per parsed writer schema. A datum the plan is
7
+ # not sure about (a value of an unexpected class, out of range, needing rounding) is written by the pure-Ruby path
8
+ # instead, so the bytes and every error stay what the pure path gives.
9
+ module Native
10
+ include FieldStruct::Avro::Native::Write
11
+ VERSION: ::String
12
+
13
+ # _@return_ — whether +FieldStruct::Avro.encode+ goes through the native encoder
14
+ def self.enabled?: () -> bool
15
+
16
+ # The Avro binary of a datum, or nil when the native encoder leaves it to the pure path.
17
+ #
18
+ # _@param_ `schema` — a parsed +Avro::Schema+
19
+ #
20
+ # _@param_ `datum` — what +FieldStruct::Avro.to_datum+ returns for that schema
21
+ #
22
+ # _@return_ — binary bytes, or nil
23
+ def self.encode: (::Object schema, ::Object datum) -> ::String?
24
+
25
+ # Prepended to +FieldStruct::Avro+'s singleton class: the binary write tries the plan first.
26
+ module Write
27
+ end
28
+
29
+ # Turns a parsed +Avro::Schema+ into the flat node list +Plan.new+ takes, or +nil+ for a schema with a shape the
30
+ # native encoder does not cover (then every encode with it runs the pure path).
31
+ #
32
+ # Each node is an Array starting with a tag; child nodes are referenced by index, so a recursive record is a
33
+ # node that points back at itself.
34
+ class Compiler
35
+ TAGS: ::Hash[Symbol, Integer]
36
+ TICKS: ::Hash[String, Integer]
37
+ CONVERTED: ::Hash[String, [Symbol, String]]
38
+ PRIMITIVES: ::Array[Symbol]
39
+ COMPOSITES: ::Array[Symbol]
40
+
41
+ # _@param_ `schema` — a parsed +Avro::Schema+
42
+ #
43
+ # _@return_ — the nodes and the root index, or nil when unsupported
44
+ def self.call: (::Object schema) -> [::Array[::Array[::Object]], ::Integer]?
45
+
46
+ def initialize: () -> void
47
+
48
+ # _@param_ `schema` — a parsed +Avro::Schema+
49
+ #
50
+ # _@return_ — the node index of the schema, compiling it on first sight
51
+ def index: (::Object schema) -> ::Integer
52
+
53
+ attr_reader nodes: ::Array[::Array[::Object]]
54
+
55
+ # Raised inside the walk for a shape with no native encoding.
56
+ class Unsupported < StandardError
57
+ end
58
+ end
59
+ end
60
+ end
61
+ end
metadata ADDED
@@ -0,0 +1,98 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: field_struct-avro-native
3
+ version: !ruby/object:Gem::Version
4
+ version: 1.0.0.pre1
5
+ platform: ruby
6
+ authors:
7
+ - Adrian Madrid
8
+ bindir: bin
9
+ cert_chain: []
10
+ date: 1980-01-02 00:00:00.000000000 Z
11
+ dependencies:
12
+ - !ruby/object:Gem::Dependency
13
+ name: field_struct-avro
14
+ requirement: !ruby/object:Gem::Requirement
15
+ requirements:
16
+ - - ">="
17
+ - !ruby/object:Gem::Version
18
+ version: 1.0.0.pre1
19
+ - - "<"
20
+ - !ruby/object:Gem::Version
21
+ version: '2'
22
+ type: :runtime
23
+ prerelease: false
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ requirements:
26
+ - - ">="
27
+ - !ruby/object:Gem::Version
28
+ version: 1.0.0.pre1
29
+ - - "<"
30
+ - !ruby/object:Gem::Version
31
+ version: '2'
32
+ - !ruby/object:Gem::Dependency
33
+ name: rb_sys
34
+ requirement: !ruby/object:Gem::Requirement
35
+ requirements:
36
+ - - "~>"
37
+ - !ruby/object:Gem::Version
38
+ version: '0.9'
39
+ type: :runtime
40
+ prerelease: false
41
+ version_requirements: !ruby/object:Gem::Requirement
42
+ requirements:
43
+ - - "~>"
44
+ - !ruby/object:Gem::Version
45
+ version: '0.9'
46
+ description: Compiles each writer schema into an encoding plan and writes field_struct-avro's
47
+ datums to Avro binary in Rust. Anything it is not sure about goes to the pure-Ruby
48
+ path, so output and errors stay the same.
49
+ email:
50
+ - adrian.madrid@timeworxs.com
51
+ executables: []
52
+ extensions:
53
+ - ext/field_struct_avro_native/extconf.rb
54
+ extra_rdoc_files: []
55
+ files:
56
+ - CHANGELOG.md
57
+ - Cargo.lock
58
+ - Cargo.toml
59
+ - LICENSE.txt
60
+ - README.md
61
+ - ext/field_struct_avro_kernel/Cargo.toml
62
+ - ext/field_struct_avro_kernel/src/decimal.rs
63
+ - ext/field_struct_avro_kernel/src/lib.rs
64
+ - ext/field_struct_avro_native/Cargo.toml
65
+ - ext/field_struct_avro_native/extconf.rb
66
+ - ext/field_struct_avro_native/src/lib.rs
67
+ - ext/field_struct_avro_native/src/plan.rs
68
+ - ext/field_struct_avro_native/src/write.rs
69
+ - lib/field_struct/avro/native.rb
70
+ - lib/field_struct/avro/native/compiler.rb
71
+ - lib/field_struct/avro/native/version.rb
72
+ - sig/field_struct-avro-native.rbs
73
+ homepage: https://github.com/gliders-rb/gems
74
+ licenses:
75
+ - MIT
76
+ metadata:
77
+ source_code_uri: https://github.com/gliders-rb/gems/tree/main/adapters/field_struct-avro-native
78
+ changelog_uri: https://github.com/gliders-rb/gems/blob/main/adapters/field_struct-avro-native/CHANGELOG.md
79
+ rubygems_mfa_required: 'true'
80
+ rdoc_options: []
81
+ require_paths:
82
+ - lib
83
+ required_ruby_version: !ruby/object:Gem::Requirement
84
+ requirements:
85
+ - - ">="
86
+ - !ruby/object:Gem::Version
87
+ version: '3.3'
88
+ required_rubygems_version: !ruby/object:Gem::Requirement
89
+ requirements:
90
+ - - ">="
91
+ - !ruby/object:Gem::Version
92
+ version: '0'
93
+ requirements: []
94
+ rubygems_version: 3.6.9
95
+ specification_version: 4
96
+ summary: 'An optional Rust encoder for field_struct-avro: the same Avro bytes, written
97
+ faster.'
98
+ test_files: []