avromatic 0.13.0 → 0.14.0.rc0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 8e9a7c0252ba986f0babd9df45f33a044e4ff690
4
- data.tar.gz: 4af262744ceb5baf95a9de4b227ab3c697ff1997
3
+ metadata.gz: dd0e7ff37e01961347eb5222e574d1e1942078ec
4
+ data.tar.gz: 3894c1ed617bc9525eac01c72969bed69702c015
5
5
  SHA512:
6
- metadata.gz: b55e680ab5badd4ffde004b89030e1d8d9bf39c8f9f7d5725cdb5244106f551440780b9c5df0a85f86698a34394df115e621bbbb8ca0a31095d2ed100c90351f
7
- data.tar.gz: b92bce0d36ecb7854fb642527fae21d7355cdcb0004a799d1739fa384d63ec61c18cc023cab0f775b746c5c37cdccda6847e32b039d98bb9b23905d6c82c53c1
6
+ metadata.gz: 87701549c8cb63b5636d2647f5b3025fef2f652ca886c876223b9d44457370827d8324536b7492fe786c5b9686f67991074130d69e29901f2739ca63b8098f43
7
+ data.tar.gz: 9da95256f8eb308390bcd64517525ac8e4410a812666efcc3eb92bca6865a7f0f481ca7a086062d8844db69fb4b09ce874e7b19f7d7608b33aaa74d3565e3fde
data/CHANGELOG.md CHANGED
@@ -1,5 +1,9 @@
1
1
  # avromatic changelog
2
2
 
3
+ ## v0.14.0
4
+ - Add `Avromatic::Messaging` and `Avromatic::IO::DatumReader` classes to
5
+ optimize the decoding of Avro unions to Avromatic models.
6
+
3
7
  ## v0.13.0
4
8
  - Add interfaces to deserialize as a hash of attributes instead of a model.
5
9
 
@@ -0,0 +1,74 @@
1
+ # rubocop:disable Style/WhenThen
2
+ module Avromatic
3
+ module IO
4
+ # Subclass DatumReader to include additional information about the union
5
+ # index used.
6
+ class DatumReader < Avro::IO::DatumReader
7
+
8
+ UNION_MEMBER_INDEX = '__avromatic_member_index'.freeze
9
+
10
+ def read_data(writers_schema, readers_schema, decoder, initial_record = {})
11
+ # schema matching
12
+ unless self.class.match_schemas(writers_schema, readers_schema)
13
+ raise SchemaMatchException.new(writers_schema, readers_schema)
14
+ end
15
+
16
+ # schema resolution: reader's schema is a union, writer's schema is not
17
+ if writers_schema.type_sym != :union && readers_schema.type_sym == :union
18
+ rs_index = readers_schema.schemas.find_index do |s|
19
+ self.class.match_schemas(writers_schema, s)
20
+ end
21
+
22
+ union_info = { UNION_MEMBER_INDEX => rs_index }
23
+
24
+ return read_data(writers_schema, readers_schema.schemas[rs_index], decoder, union_info) if rs_index
25
+ raise SchemaMatchException.new(writers_schema, readers_schema)
26
+ end
27
+
28
+ # function dispatch for reading data based on type of writer's
29
+ # schema
30
+ datum = case writers_schema.type_sym
31
+ when :null; decoder.read_null
32
+ when :boolean; decoder.read_boolean
33
+ when :string; decoder.read_string
34
+ when :int; decoder.read_int
35
+ when :long; decoder.read_long
36
+ when :float; decoder.read_float
37
+ when :double; decoder.read_double
38
+ when :bytes; decoder.read_bytes
39
+ when :fixed; read_fixed(writers_schema, readers_schema, decoder)
40
+ when :enum; read_enum(writers_schema, readers_schema, decoder)
41
+ when :array; read_array(writers_schema, readers_schema, decoder)
42
+ when :map; read_map(writers_schema, readers_schema, decoder)
43
+ when :union; read_union(writers_schema, readers_schema, decoder)
44
+ when :record, :error, :request; read_record(writers_schema, readers_schema, decoder, initial_record)
45
+ else
46
+ raise AvroError.new("Cannot read unknown schema type: #{writers_schema.type}")
47
+ end
48
+
49
+ if readers_schema.respond_to?(:logical_type)
50
+ readers_schema.type_adapter.decode(datum)
51
+ else
52
+ datum
53
+ end
54
+ end
55
+
56
+ # Override to specify initial record that may contain union index
57
+ def read_record(writers_schema, readers_schema, decoder, initial_record = {})
58
+ readers_fields_hash = readers_schema.fields_hash
59
+ read_record = Avromatic.use_custom_datum_reader ? initial_record : {}
60
+ writers_schema.fields.each do |field|
61
+ readers_field = readers_fields_hash[field.name]
62
+ if readers_field
63
+ field_val = read_data(field.type, readers_field.type, decoder)
64
+ read_record[field.name] = field_val
65
+ else
66
+ skip_data(field.type, decoder)
67
+ end
68
+ end
69
+
70
+ read_record
71
+ end
72
+ end
73
+ end
74
+ end
@@ -0,0 +1,32 @@
1
+ require 'avro_turf/messaging'
2
+ require 'avromatic/io/datum_reader'
3
+
4
+ module Avromatic
5
+ # Subclass AvroTurf::Messaging to use a custom DatumReader
6
+ class Messaging < AvroTurf::Messaging
7
+ def decode(data, schema_name: nil, namespace: @namespace)
8
+ readers_schema = schema_name && @schema_store.find(schema_name, namespace)
9
+ stream = StringIO.new(data)
10
+ decoder = Avro::IO::BinaryDecoder.new(stream)
11
+
12
+ # The first byte is MAGIC!!!
13
+ magic_byte = decoder.read(1)
14
+
15
+ if magic_byte != MAGIC_BYTE
16
+ raise "Expected data to begin with a magic byte, got `#{magic_byte.inspect}`"
17
+ end
18
+
19
+ # The schema id is a 4-byte big-endian integer.
20
+ schema_id = decoder.read(4).unpack('N').first
21
+
22
+ writers_schema = @schemas_by_id.fetch(schema_id) do
23
+ schema_json = @registry.fetch(schema_id)
24
+ @schemas_by_id[schema_id] = Avro::Schema.parse(schema_json)
25
+ end
26
+
27
+ # The following line was changed to use a custom DatumReader
28
+ reader = Avromatic::IO::DatumReader.new(writers_schema, readers_schema)
29
+ reader.read(decoder)
30
+ end
31
+ end
32
+ end
@@ -1,4 +1,5 @@
1
1
  require 'avromatic/model/attribute_type/union'
2
+ require 'avromatic/io/datum_reader'
2
3
 
3
4
  module Avromatic
4
5
  module Model
@@ -12,6 +13,8 @@ module Avromatic
12
13
  class Union < Virtus::Attribute
13
14
  primitive Avromatic::Model::AttributeType::Union
14
15
 
16
+ MEMBER_INDEX = ::Avromatic::IO::DatumReader::UNION_MEMBER_INDEX
17
+
15
18
  def initialize(*)
16
19
  super
17
20
 
@@ -24,12 +27,11 @@ module Avromatic
24
27
  return input if value_coerced?(input)
25
28
 
26
29
  result = nil
27
- member_attributes.find do |union_attribute|
28
- begin
29
- coerced = union_attribute.coerce(input)
30
- result = coerced unless coerced.is_a?(Avromatic::Model::Attributes) && coerced.invalid?
31
- rescue
32
- nil
30
+ if input.key?(MEMBER_INDEX)
31
+ result = safe_coerce(member_attributes[input.delete(MEMBER_INDEX)], input)
32
+ else
33
+ member_attributes.find do |union_attribute|
34
+ result = safe_coerce(union_attribute, input)
33
35
  end
34
36
  end
35
37
  result
@@ -43,6 +45,13 @@ module Avromatic
43
45
 
44
46
  private
45
47
 
48
+ def safe_coerce(member_attribute, input)
49
+ coerced = member_attribute.coerce(input)
50
+ coerced unless coerced.is_a?(Avromatic::Model::Attributes) && coerced.invalid?
51
+ rescue
52
+ nil
53
+ end
54
+
46
55
  def member_attributes
47
56
  @member_attributes ||= Array.new
48
57
  end
@@ -1,5 +1,3 @@
1
- require 'avro_turf/messaging'
2
-
3
1
  module Avromatic
4
2
  module Model
5
3
 
@@ -88,11 +88,14 @@ module Avromatic
88
88
  end
89
89
 
90
90
  def custom_datum_reader(schema, key_or_value)
91
- Avro::IO::DatumReader.new(schema, send("#{key_or_value}_avro_schema"))
91
+ datum_reader_class.new(schema, send("#{key_or_value}_avro_schema"))
92
92
  end
93
93
  end
94
94
 
95
95
  module ClassMethods
96
+ def datum_reader_class
97
+ Avromatic::IO::DatumReader
98
+ end
96
99
 
97
100
  # Store a hash of Procs by field name (as a symbol) to convert
98
101
  # the value before Avro serialization.
@@ -111,8 +114,8 @@ module Avromatic
111
114
 
112
115
  def datum_reader
113
116
  @datum_reader ||= begin
114
- hash = { value: Avro::IO::DatumReader.new(value_avro_schema) }
115
- hash[:key] = Avro::IO::DatumReader.new(key_avro_schema) if key_avro_schema
117
+ hash = { value: datum_reader_class.new(value_avro_schema) }
118
+ hash[:key] = datum_reader_class.new(key_avro_schema) if key_avro_schema
116
119
  hash
117
120
  end
118
121
  end
@@ -1,3 +1,3 @@
1
1
  module Avromatic
2
- VERSION = '0.13.0'.freeze
2
+ VERSION = '0.14.0.rc0'.freeze
3
3
  end
data/lib/avromatic.rb CHANGED
@@ -1,14 +1,15 @@
1
1
  require 'avromatic/version'
2
+ require 'avro_turf'
2
3
  require 'avromatic/model'
3
4
  require 'avromatic/model_registry'
4
- require 'avro_turf'
5
- require 'avro_turf/messaging'
5
+ require 'avromatic/messaging'
6
6
  require 'active_support/core_ext/string/inflections'
7
7
 
8
8
  module Avromatic
9
9
  class << self
10
10
  attr_accessor :schema_registry, :registry_url, :schema_store, :logger,
11
- :messaging, :type_registry, :nested_models
11
+ :messaging, :type_registry, :nested_models,
12
+ :use_custom_datum_reader
12
13
 
13
14
  delegate :register_type, to: :type_registry
14
15
  end
@@ -16,6 +17,7 @@ module Avromatic
16
17
  self.nested_models = ModelRegistry.new
17
18
  self.logger = Logger.new($stdout)
18
19
  self.type_registry = Avromatic::Model::TypeRegistry.new
20
+ self.use_custom_datum_reader = true
19
21
 
20
22
  def self.configure
21
23
  yield self
@@ -31,7 +33,7 @@ module Avromatic
31
33
 
32
34
  def self.build_messaging
33
35
  raise 'Avromatic must be configured with a schema_store' unless schema_store
34
- AvroTurf::Messaging.new(
36
+ Avromatic::Messaging.new(
35
37
  registry: schema_registry || build_schema_registry,
36
38
  schema_store: schema_store,
37
39
  logger: logger
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: avromatic
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.13.0
4
+ version: 0.14.0.rc0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Salsify Engineering
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2017-01-10 00:00:00.000000000 Z
11
+ date: 2017-01-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: avro
@@ -245,6 +245,8 @@ files:
245
245
  - gemfiles/rails4_1.gemfile
246
246
  - gemfiles/rails4_2.gemfile
247
247
  - lib/avromatic.rb
248
+ - lib/avromatic/io/datum_reader.rb
249
+ - lib/avromatic/messaging.rb
248
250
  - lib/avromatic/model.rb
249
251
  - lib/avromatic/model/attribute/abstract_timestamp.rb
250
252
  - lib/avromatic/model/attribute/record.rb
@@ -287,9 +289,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
287
289
  version: '0'
288
290
  required_rubygems_version: !ruby/object:Gem::Requirement
289
291
  requirements:
290
- - - ">="
292
+ - - ">"
291
293
  - !ruby/object:Gem::Version
292
- version: '0'
294
+ version: 1.3.1
293
295
  requirements: []
294
296
  rubyforge_project:
295
297
  rubygems_version: 2.6.8