avromatic 2.2.4 → 3.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +89 -0
- data/.gitignore +3 -0
- data/.rubocop.yml +4 -1
- data/Appraisals +8 -14
- data/CHANGELOG.md +23 -0
- data/README.md +3 -20
- data/avromatic.gemspec +9 -8
- data/bin/console +4 -3
- data/gemfiles/avro1_10_rails6_1.gemfile +9 -0
- data/gemfiles/{avro1_8_rails5_2.gemfile → avro1_9_rails6_1.gemfile} +3 -3
- data/lib/avromatic.rb +0 -5
- data/lib/avromatic/io.rb +1 -7
- data/lib/avromatic/io/datum_reader.rb +18 -68
- data/lib/avromatic/io/datum_writer.rb +5 -11
- data/lib/avromatic/io/union_datum.rb +25 -0
- data/lib/avromatic/messaging.rb +4 -2
- data/lib/avromatic/model/attributes.rb +28 -9
- data/lib/avromatic/model/configurable.rb +30 -2
- data/lib/avromatic/model/configuration.rb +5 -0
- data/lib/avromatic/model/field_helper.rb +5 -1
- data/lib/avromatic/model/messaging_serialization.rb +2 -1
- data/lib/avromatic/model/raw_serialization.rb +67 -24
- data/lib/avromatic/model/types/abstract_timestamp_type.rb +1 -1
- data/lib/avromatic/model/types/abstract_type.rb +3 -1
- data/lib/avromatic/model/types/array_type.rb +2 -2
- data/lib/avromatic/model/types/boolean_type.rb +1 -1
- data/lib/avromatic/model/types/custom_type.rb +1 -1
- data/lib/avromatic/model/types/date_type.rb +1 -1
- data/lib/avromatic/model/types/enum_type.rb +1 -1
- data/lib/avromatic/model/types/fixed_type.rb +1 -1
- data/lib/avromatic/model/types/float_type.rb +1 -1
- data/lib/avromatic/model/types/integer_type.rb +1 -1
- data/lib/avromatic/model/types/map_type.rb +2 -2
- data/lib/avromatic/model/types/null_type.rb +1 -1
- data/lib/avromatic/model/types/record_type.rb +4 -7
- data/lib/avromatic/model/types/string_type.rb +1 -1
- data/lib/avromatic/model/types/union_type.rb +12 -9
- data/lib/avromatic/model/validation.rb +2 -2
- data/lib/avromatic/version.rb +1 -1
- metadata +45 -34
- data/.travis.yml +0 -16
- data/gemfiles/avro_patches_rails5_2.gemfile +0 -9
- data/gemfiles/avro_patches_rails6_0.gemfile +0 -9
- data/lib/avromatic/patches.rb +0 -18
- data/lib/avromatic/patches/schema_validator_patch.rb +0 -39
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d90a142c407a453ef9007373f215d8acd137937e54b46aa2c29e8edcf8440aef
|
4
|
+
data.tar.gz: c8eecbb9860859346d9a67e3e2a78a2e38447bdff4d4fb9208f57bcf89940215
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 77b078294c887e298e4ede5f1f47cfccaccffd0b065ad1d4bad9820d2af8c77e522462282d1d44db35acefaf6fc301ccb7a59dfda844c5ebee297d51792ee912
|
7
|
+
data.tar.gz: 8f57c2afee1266f790c031920efb7e1690420e4c6de36d22af3e11361bc5bf7eaabb3eba4e2974a994a7f7509e3db01335c954895c981db4069a8c7828b483de
|
@@ -0,0 +1,89 @@
|
|
1
|
+
version: 2.1
|
2
|
+
jobs:
|
3
|
+
lint:
|
4
|
+
docker:
|
5
|
+
- image: salsify/ruby_ci:2.5.8
|
6
|
+
working_directory: ~/avromatic
|
7
|
+
steps:
|
8
|
+
- checkout
|
9
|
+
- restore_cache:
|
10
|
+
keys:
|
11
|
+
- v2-gems-ruby-2.5.8-{{ checksum "avromatic.gemspec" }}-{{ checksum "Gemfile" }}
|
12
|
+
- v2-gems-ruby-2.5.8-
|
13
|
+
- run:
|
14
|
+
name: Install Gems
|
15
|
+
command: |
|
16
|
+
if ! bundle check --path=vendor/bundle; then
|
17
|
+
bundle install --path=vendor/bundle --jobs=4 --retry=3
|
18
|
+
bundle clean
|
19
|
+
fi
|
20
|
+
- save_cache:
|
21
|
+
key: v2-gems-ruby-2.5.8-{{ checksum "avromatic.gemspec" }}-{{ checksum "Gemfile" }}
|
22
|
+
paths:
|
23
|
+
- "vendor/bundle"
|
24
|
+
- "gemfiles/vendor/bundle"
|
25
|
+
- run:
|
26
|
+
name: Run Rubocop
|
27
|
+
command: bundle exec rubocop
|
28
|
+
test:
|
29
|
+
parameters:
|
30
|
+
gemfile:
|
31
|
+
type: string
|
32
|
+
ruby-version:
|
33
|
+
type: string
|
34
|
+
docker:
|
35
|
+
- image: salsify/ruby_ci:<< parameters.ruby-version >>
|
36
|
+
environment:
|
37
|
+
CIRCLE_TEST_REPORTS: "test-results"
|
38
|
+
BUNDLE_GEMFILE: << parameters.gemfile >>
|
39
|
+
working_directory: ~/avromatic
|
40
|
+
steps:
|
41
|
+
- checkout
|
42
|
+
- restore_cache:
|
43
|
+
keys:
|
44
|
+
- v2-gems-ruby-<< parameters.ruby-version >>-{{ checksum "avromatic.gemspec" }}-{{ checksum "<< parameters.gemfile >>" }}
|
45
|
+
- v2-gems-ruby-<< parameters.ruby-version >>-
|
46
|
+
- run:
|
47
|
+
name: Install Gems
|
48
|
+
command: |
|
49
|
+
if ! bundle check --path=vendor/bundle; then
|
50
|
+
bundle install --path=vendor/bundle --jobs=4 --retry=3
|
51
|
+
bundle clean
|
52
|
+
fi
|
53
|
+
- save_cache:
|
54
|
+
key: v2-gems-ruby-<< parameters.ruby-version >>-{{ checksum "avromatic.gemspec" }}-{{ checksum "<< parameters.gemfile >>" }}
|
55
|
+
paths:
|
56
|
+
- "vendor/bundle"
|
57
|
+
- "gemfiles/vendor/bundle"
|
58
|
+
- run:
|
59
|
+
name: Run Tests
|
60
|
+
command: |
|
61
|
+
bundle exec rspec --format RspecJunitFormatter --out $CIRCLE_TEST_REPORTS/rspec/junit.xml --format progress spec
|
62
|
+
- store_test_results:
|
63
|
+
path: "test-results"
|
64
|
+
workflows:
|
65
|
+
build:
|
66
|
+
jobs:
|
67
|
+
- lint
|
68
|
+
- test:
|
69
|
+
matrix:
|
70
|
+
parameters:
|
71
|
+
gemfile:
|
72
|
+
- "gemfiles/avro1_9_rails5_2.gemfile"
|
73
|
+
- "gemfiles/avro1_10_rails5_2.gemfile"
|
74
|
+
- "gemfiles/avro1_9_rails6_0.gemfile"
|
75
|
+
- "gemfiles/avro1_10_rails6_0.gemfile"
|
76
|
+
- "gemfiles/avro1_10_rails6_1.gemfile"
|
77
|
+
- "gemfiles/avro1_9_rails6_1.gemfile"
|
78
|
+
ruby-version:
|
79
|
+
- "2.5.8"
|
80
|
+
- "2.6.6"
|
81
|
+
- "2.7.2"
|
82
|
+
- test:
|
83
|
+
matrix:
|
84
|
+
parameters:
|
85
|
+
gemfile:
|
86
|
+
- "gemfiles/avro1_10_rails6_1.gemfile"
|
87
|
+
- "gemfiles/avro1_9_rails6_1.gemfile"
|
88
|
+
ruby-version:
|
89
|
+
- "3.0.0"
|
data/.gitignore
CHANGED
data/.rubocop.yml
CHANGED
data/Appraisals
CHANGED
@@ -1,11 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
appraise 'avro1_8-rails5_2' do
|
4
|
-
gem 'avro', '1.8.2'
|
5
|
-
gem 'activesupport', '~> 5.2.0'
|
6
|
-
gem 'activemodel', '~> 5.2.0'
|
7
|
-
end
|
8
|
-
|
9
3
|
appraise 'avro1_9-rails5_2' do
|
10
4
|
gem 'avro', '1.9.2'
|
11
5
|
gem 'activesupport', '~> 5.2.0'
|
@@ -30,14 +24,14 @@ appraise 'avro1_10-rails6_0' do
|
|
30
24
|
gem 'activemodel', '~> 6.0.0'
|
31
25
|
end
|
32
26
|
|
33
|
-
appraise '
|
34
|
-
gem 'avro
|
35
|
-
gem 'activesupport', '~>
|
36
|
-
gem 'activemodel', '~>
|
27
|
+
appraise 'avro1_9-rails6_1' do
|
28
|
+
gem 'avro', '1.9.2'
|
29
|
+
gem 'activesupport', '~> 6.1.0'
|
30
|
+
gem 'activemodel', '~> 6.1.0'
|
37
31
|
end
|
38
32
|
|
39
|
-
appraise '
|
40
|
-
gem 'avro
|
41
|
-
gem 'activesupport', '~> 6.
|
42
|
-
gem 'activemodel', '~> 6.
|
33
|
+
appraise 'avro1_10-rails6_1' do
|
34
|
+
gem 'avro', '~> 1.10.0'
|
35
|
+
gem 'activesupport', '~> 6.1.0'
|
36
|
+
gem 'activemodel', '~> 6.1.0'
|
43
37
|
end
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,28 @@
|
|
1
1
|
# avromatic changelog
|
2
2
|
|
3
|
+
## 3.0.0
|
4
|
+
- Drop support for Ruby 2.4.
|
5
|
+
- Add support for Ruby 3.0.
|
6
|
+
- Drop support for Avro < 1.9.
|
7
|
+
- Drop support for Rails < 5.2.
|
8
|
+
- Fix decoding of unions containing false boolean values.
|
9
|
+
|
10
|
+
## v2.4.0
|
11
|
+
- Ignore the `validate` argument and always validate during serialization. This
|
12
|
+
argument will be removed in Avromatic 3.0.
|
13
|
+
- Optimize model validation during serialization.
|
14
|
+
- Don't cache immutable model validation results or serialized Avro attributes if a model has mutable children.
|
15
|
+
|
16
|
+
## v2.3.0
|
17
|
+
- Add support for Rails 6.1.
|
18
|
+
- Optimize nested model serialization.
|
19
|
+
|
20
|
+
## v2.2.6
|
21
|
+
- Optimize memory usage when serializing models.
|
22
|
+
|
23
|
+
## v2.2.5
|
24
|
+
- Optimize memory usage when serializing, deserializing and instantiating models.
|
25
|
+
|
3
26
|
## v2.2.4
|
4
27
|
- Compatibility with Avro v1.10.x.
|
5
28
|
|
data/README.md
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
# Avromatic
|
2
2
|
|
3
|
-
[![Build Status](https://
|
3
|
+
[![Build Status](https://circleci.com/gh/salsify/avromatic.svg?style=svg)][circleci]
|
4
4
|
[![Gem Version](https://badge.fury.io/rb/avromatic.svg)](https://badge.fury.io/rb/avromatic)
|
5
5
|
|
6
|
-
[
|
6
|
+
[circleci]: https://circleci.com/gh/salsify/avromatic
|
7
7
|
|
8
8
|
`Avromatic` generates Ruby models from [Avro](http://avro.apache.org/) schemas
|
9
9
|
and provides utilities to encode and decode them.
|
@@ -142,7 +142,7 @@ instance.eql?(MyModel.new(id: 123, name: 'Tesla Model 3', enabled: true)) # => t
|
|
142
142
|
instance.hash # => -1279155042741869898
|
143
143
|
|
144
144
|
# Retrieve a hash of the model's attributes via to_h, to_hash or attributes
|
145
|
-
instance
|
145
|
+
instance.to_h # => {:id=>123, :name=>"Tesla Model 3", :enabled=>true}
|
146
146
|
```
|
147
147
|
|
148
148
|
Or an `Avro::Schema` object can be specified directly:
|
@@ -430,23 +430,6 @@ Validation of required fields is done automatically when serializing a model to
|
|
430
430
|
explicitly by calling the `valid?` or `invalid?` methods from the
|
431
431
|
[ActiveModel::Validations](https://edgeapi.rubyonrails.org/classes/ActiveModel/Validations.html) interface.
|
432
432
|
|
433
|
-
### Logical Types
|
434
|
-
|
435
|
-
Currently the official Apache Avro Ruby library does not support logical types ([AVRO-1695](https://issues.apache.org/jira/browse/AVRO-1695)).
|
436
|
-
That feature is in progress and will hopefully be merged soon.
|
437
|
-
|
438
|
-
Avromatic supports logical types as implemented in the [pull request](https://github.com/apache/avro/pull/116) referenced in AVRO-1695.
|
439
|
-
|
440
|
-
Until that change is included in the official library, you can
|
441
|
-
use the [avro-patches gem](https://github.com/salsify/avro-patches) which includes
|
442
|
-
the changes from the above pull request.
|
443
|
-
|
444
|
-
To use this gem, reference it in your Gemfile instead of `avro`:
|
445
|
-
|
446
|
-
```ruby
|
447
|
-
gem 'avro-patches'
|
448
|
-
````
|
449
|
-
|
450
433
|
### RSpec Support
|
451
434
|
|
452
435
|
This gem also includes an `"avromatic/rspec"` file that can be required to support
|
data/avromatic.gemspec
CHANGED
@@ -20,21 +20,22 @@ Gem::Specification.new do |spec|
|
|
20
20
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
21
21
|
spec.require_paths = ['lib']
|
22
22
|
|
23
|
-
spec.required_ruby_version = '>= 2.
|
23
|
+
spec.required_ruby_version = '>= 2.5'
|
24
24
|
|
25
|
-
spec.add_runtime_dependency 'activemodel', '>= 5.
|
26
|
-
spec.add_runtime_dependency 'activesupport', '>= 5.
|
27
|
-
spec.add_runtime_dependency 'avro', '>= 1.
|
28
|
-
spec.add_runtime_dependency 'avro_schema_registry-client', '>= 0.
|
25
|
+
spec.add_runtime_dependency 'activemodel', '>= 5.2', '< 6.2'
|
26
|
+
spec.add_runtime_dependency 'activesupport', '>= 5.2', '< 6.2'
|
27
|
+
spec.add_runtime_dependency 'avro', '>= 1.9.0', '< 1.11'
|
28
|
+
spec.add_runtime_dependency 'avro_schema_registry-client', '>= 0.4.0'
|
29
29
|
spec.add_runtime_dependency 'avro_turf'
|
30
30
|
spec.add_runtime_dependency 'ice_nine'
|
31
31
|
|
32
32
|
spec.add_development_dependency 'appraisal'
|
33
33
|
spec.add_development_dependency 'avro-builder', '>= 0.12.0'
|
34
|
-
spec.add_development_dependency 'bundler', '
|
34
|
+
spec.add_development_dependency 'bundler', '~> 2.0'
|
35
35
|
spec.add_development_dependency 'overcommit', '0.35.0'
|
36
|
-
spec.add_development_dependency 'rake', '~>
|
37
|
-
spec.add_development_dependency 'rspec', '~> 3.
|
36
|
+
spec.add_development_dependency 'rake', '~> 13.0'
|
37
|
+
spec.add_development_dependency 'rspec', '~> 3.8'
|
38
|
+
spec.add_development_dependency 'rspec_junit_formatter'
|
38
39
|
spec.add_development_dependency 'salsify_rubocop', '~> 0.52.1.1'
|
39
40
|
spec.add_development_dependency 'simplecov'
|
40
41
|
spec.add_development_dependency 'webmock'
|
data/bin/console
CHANGED
@@ -1,7 +1,8 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
2
3
|
|
3
|
-
require
|
4
|
-
require
|
4
|
+
require 'bundler/setup'
|
5
|
+
require 'avromatic'
|
5
6
|
|
6
7
|
# You can add fixtures and/or initialization code here to make experimenting
|
7
8
|
# with your gem easier. You can also use a different console, if you like.
|
@@ -10,5 +11,5 @@ require "avromatic"
|
|
10
11
|
# require "pry"
|
11
12
|
# Pry.start
|
12
13
|
|
13
|
-
require
|
14
|
+
require 'irb'
|
14
15
|
IRB.start
|
data/lib/avromatic.rb
CHANGED
@@ -8,7 +8,6 @@ require 'avromatic/model'
|
|
8
8
|
require 'avromatic/model_registry'
|
9
9
|
require 'avromatic/messaging'
|
10
10
|
require 'active_support/core_ext/string/inflections'
|
11
|
-
require 'avromatic/patches'
|
12
11
|
|
13
12
|
module Avromatic
|
14
13
|
class << self
|
@@ -33,10 +32,6 @@ module Avromatic
|
|
33
32
|
eager_load_models!
|
34
33
|
end
|
35
34
|
|
36
|
-
def self.use_encoding_providers?
|
37
|
-
use_custom_datum_writer && defined?(Avromatic::Patches::SchemaValidatorPatch)
|
38
|
-
end
|
39
|
-
|
40
35
|
def self.build_schema_registry
|
41
36
|
raise 'Avromatic must be configured with a registry_url' unless registry_url
|
42
37
|
if use_schema_fingerprint_lookup
|
data/lib/avromatic/io.rb
CHANGED
@@ -1,11 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
module Avromatic
|
4
|
-
module IO
|
5
|
-
UNION_MEMBER_INDEX = '__avromatic_member_index'
|
6
|
-
ENCODING_PROVIDER = '__avromatic_encoding_provider'
|
7
|
-
end
|
8
|
-
end
|
9
|
-
|
10
3
|
require 'avromatic/io/datum_reader'
|
11
4
|
require 'avromatic/io/datum_writer'
|
5
|
+
require 'avromatic/io/union_datum'
|
@@ -1,6 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
# rubocop:disable Style/WhenThen
|
4
3
|
module Avromatic
|
5
4
|
module IO
|
6
5
|
# Subclass DatumReader to include additional information about the union
|
@@ -8,81 +7,32 @@ module Avromatic
|
|
8
7
|
# branch 'salsify-master' with the tag 'v1.9.0.3'
|
9
8
|
class DatumReader < Avro::IO::DatumReader
|
10
9
|
|
11
|
-
|
12
|
-
|
13
|
-
def read_data(writers_schema, readers_schema, decoder, initial_record = nil)
|
14
|
-
# schema matching
|
15
|
-
unless self.class.match_schemas(writers_schema, readers_schema)
|
16
|
-
raise Avro::IO::SchemaMatchException.new(writers_schema, readers_schema)
|
17
|
-
end
|
18
|
-
|
10
|
+
def read_data(writers_schema, readers_schema, decoder)
|
19
11
|
# schema resolution: reader's schema is a union, writer's schema is not
|
20
|
-
|
21
|
-
rs_index = readers_schema.schemas.find_index do |s|
|
22
|
-
self.class.match_schemas(writers_schema, s)
|
23
|
-
end
|
12
|
+
return super unless writers_schema.type_sym != :union && readers_schema.type_sym == :union
|
24
13
|
|
25
|
-
|
26
|
-
|
27
|
-
# Avromatic does not treat the union of null and 1 other type as a union
|
28
|
-
{}
|
29
|
-
elsif optional
|
30
|
-
# Avromatic does not treat the null of an optional field as part of the union
|
31
|
-
{ UNION_MEMBER_INDEX => rs_index - 1 }
|
32
|
-
else
|
33
|
-
{ UNION_MEMBER_INDEX => rs_index }
|
34
|
-
end
|
35
|
-
|
36
|
-
return read_data(writers_schema, readers_schema.schemas[rs_index], decoder, union_info) if rs_index
|
37
|
-
raise Avro::IO::SchemaMatchException.new(writers_schema, readers_schema)
|
14
|
+
rs_index = readers_schema.schemas.find_index do |s|
|
15
|
+
self.class.match_schemas(writers_schema, s)
|
38
16
|
end
|
39
17
|
|
40
|
-
|
41
|
-
datum = case writers_schema.type_sym
|
42
|
-
when :null; decoder.read_null
|
43
|
-
when :boolean; decoder.read_boolean
|
44
|
-
when :string; decoder.read_string
|
45
|
-
when :int; decoder.read_int
|
46
|
-
when :long; decoder.read_long
|
47
|
-
when :float; decoder.read_float
|
48
|
-
when :double; decoder.read_double
|
49
|
-
when :bytes; decoder.read_bytes
|
50
|
-
when :fixed; read_fixed(writers_schema, readers_schema, decoder)
|
51
|
-
when :enum; read_enum(writers_schema, readers_schema, decoder)
|
52
|
-
when :array; read_array(writers_schema, readers_schema, decoder)
|
53
|
-
when :map; read_map(writers_schema, readers_schema, decoder)
|
54
|
-
when :union; read_union(writers_schema, readers_schema, decoder)
|
55
|
-
when :record, :error, :request; read_record(writers_schema, readers_schema, decoder, initial_record || {})
|
56
|
-
else
|
57
|
-
raise Avro::AvroError.new("Cannot read unknown schema type: #{writers_schema.type}")
|
58
|
-
end
|
18
|
+
raise Avro::IO::SchemaMatchException.new(writers_schema, readers_schema) unless rs_index
|
59
19
|
|
60
|
-
|
61
|
-
|
62
|
-
if readers_schema.respond_to?(:logical_type)
|
63
|
-
readers_schema.type_adapter.decode(datum)
|
64
|
-
else
|
65
|
-
datum
|
66
|
-
end
|
67
|
-
end
|
20
|
+
datum = read_data(writers_schema, readers_schema.schemas[rs_index], decoder)
|
21
|
+
optional = readers_schema.schemas.first.type_sym == :null
|
68
22
|
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
end
|
23
|
+
if readers_schema.schemas.size == 2 && optional
|
24
|
+
# Avromatic does not treat the union of null and 1 other type as a union
|
25
|
+
datum
|
26
|
+
elsif datum.nil?
|
27
|
+
# Avromatic does not treat the null of an optional field as part of the union
|
28
|
+
nil
|
29
|
+
else
|
30
|
+
# Avromatic does not treat the null of an optional field as part of the union so
|
31
|
+
# adjust the member index accordingly
|
32
|
+
member_index = optional ? rs_index - 1 : rs_index
|
33
|
+
Avromatic::IO::UnionDatum.new(member_index, datum)
|
81
34
|
end
|
82
|
-
|
83
|
-
read_record
|
84
35
|
end
|
85
36
|
end
|
86
37
|
end
|
87
38
|
end
|
88
|
-
# rubocop:enable Style/WhenThen
|