avromatic 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 329e7d74bd13cdd0237363d54e44afbefca210de
4
+ data.tar.gz: 7c9e9357c7933615a82a9c6ad76e7c679d8068ce
5
+ SHA512:
6
+ metadata.gz: 364418e416fc2bb56157765a0662177c705d0c1bf7632080f0f5ee5603e571ca253cff575f30e11ab1c22bac3ca433cd3a1a726d66c5cab441b4373546e3cdf3
7
+ data.tar.gz: af17cbc43d42a4e0545df4ff92c0d34372ad5be3877db490f0f72636328599860146f18b986a48d6fd5b703a9081a37899dcd169b5c483b1d018ec7f1a0c8141
data/.gitignore ADDED
@@ -0,0 +1,9 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
data/.rspec ADDED
@@ -0,0 +1,2 @@
1
+ --format documentation
2
+ --color
data/.travis.yml ADDED
@@ -0,0 +1,5 @@
1
+ language: ruby
2
+ rvm:
3
+ - 2.2.3
4
+ - 2.3.0
5
+ before_install: gem install bundler -v 1.11.2
data/CHANGELOG.md ADDED
@@ -0,0 +1,4 @@
1
+ # avromatic changelog
2
+
3
+ ## v0.1.0
4
+ - Initial release
data/Gemfile ADDED
@@ -0,0 +1,8 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Avromatic currently depends on a fork of AvroTurf for some changes that
4
+ # have not been accepted/released.
5
+ gem 'avro_turf', github: 'salsify/avro_turf', branch: 'salsify-master'
6
+
7
+ # Specify your gem's dependencies in avromatic.gemspec
8
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2016 Salsify, Inc
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,162 @@
1
+ # Avromatic
2
+
3
+ [![Build Status](https://travis-ci.org/salsify/avromatic.svg?branch=master)][travis]
4
+
5
+ [travis]: http://travis-ci.org/salsify/avromatic
6
+
7
+ `Avromatic` generates Ruby models from Avro schemas and provides utilities to
8
+ encode and decode them.
9
+
10
+ ## Installation
11
+
12
+ Add this line to your application's Gemfile:
13
+
14
+ ```ruby
15
+ gem 'avromatic'
16
+ ```
17
+
18
+ And then execute:
19
+
20
+ $ bundle
21
+
22
+ Or install it yourself as:
23
+
24
+ $ gem install avromatic
25
+
26
+ ## Usage
27
+
28
+ ### Configuration
29
+
30
+ `Avromatic` supports the following configuration:
31
+
32
+ * registry_url: URL for the schema registry. The schema registry is used to store
33
+ Avro schemas so that they can be referenced by id.
34
+ * schema_store: The schema store is used to load Avro schemas from the filesystem.
35
+ It should be an object that responds to `find(name, namespace = nil)` and
36
+ returns an `Avro::Schema` object.
37
+ * messaging: An `AvroTurf::Messaging` object may be specified and will be shared
38
+ by all models. If unspecified a new messaging object is created based on the
39
+ schema store and registry_url.
40
+ * logger: The logger is for the schema registry client.
41
+
42
+ ### Models
43
+
44
+ Models may be defined based on an Avro schema for a record.
45
+
46
+ The Avro schema can be specified by name and loaded using the schema store:
47
+
48
+ ```ruby
49
+ class MyModel
50
+ include Avromatic::Model.build(schema_name :my_model)
51
+ end
52
+ ```
53
+
54
+ Or an `Avro::Schema` object can be specified directly:
55
+
56
+ ```ruby
57
+ class MyModel
58
+ include Avromatic::Model.build(schema: schema_object)
59
+ end
60
+
61
+ ```
62
+
63
+ Models are generated as [Virtus](https://github.com/solnic/virtus) value
64
+ objects. `Virtus` attributes are added for each field in the Avro schema
65
+ including any default values defined in the schema. `ActiveModel` validations
66
+ are used to define validations on certain types of fields.
67
+
68
+ A model may be defined with both a key and a value schema:
69
+
70
+ ```ruby
71
+ class MyTopic
72
+ include Avromatic::Model.build(value_schema_name: :topic_value,
73
+ key_schema_name: :topic_key)
74
+ end
75
+ ```
76
+
77
+ When key and value schemas are both specified, attributes are added to the model
78
+ for the union of the fields in the two schemas.
79
+
80
+ A model can also be generated as an anonymous class that can be assigned to a
81
+ constant:
82
+
83
+ ```ruby
84
+ MyModel = Avromatic::Model.model(schema_name :my_model)
85
+ ```
86
+
87
+ #### Encode/Decode
88
+
89
+ Models can be encoded using Avro leveraging a schema registry to encode a schema
90
+ id at the beginning of the value.
91
+
92
+ ```ruby
93
+ model.avro_message_value
94
+ ```
95
+
96
+ If a model has a Avro schema for a key, then the key can also be encoded
97
+ prefixed with a schema id.
98
+
99
+ ```ruby
100
+ model.avro_message_key
101
+ ```
102
+
103
+ A model instance can be created from an Avro-encoded value and an Avro-encoded
104
+ optional key:
105
+
106
+ ```ruby
107
+ MyTopic.deserialize(message_key, message_value)
108
+ ```
109
+
110
+ Or just a value if only one schema is used:
111
+
112
+ ```ruby
113
+ MyValue.deserialize(message_value)
114
+ ```
115
+
116
+ #### Decoder
117
+
118
+ A stream of messages encoded from various models can be deserialized using
119
+ `Avromatic::Model::Decoder`. The decoder must be initialized with the list
120
+ of models to decode:
121
+
122
+ ```ruby
123
+ decoder = Avromatic::Model::Decoder.new(MyModel1, MyModel2)
124
+
125
+ decoder.decode(model1_key, model1_value)
126
+ # => instance of MyModel1
127
+ decoder.decode(model2_value)
128
+ # => instance of MyModel2
129
+ ```
130
+
131
+ #### Validations
132
+
133
+ The following validations are supported:
134
+
135
+ - The size of the value for a fixed type field.
136
+ - The value for an enum type field is in the declared set of values.
137
+ - Presence of a value for required fields.
138
+
139
+ #### Unsupported/Future
140
+
141
+ The following types/features are not supported for generated models:
142
+
143
+ - Generic union fields: The special case of an optional field, the union of `:null` and
144
+ another type, is supported.
145
+ - Reused models for nested records: Currently an anonymous model class is
146
+ generated for each subrecord.
147
+
148
+ ## Development
149
+
150
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
151
+
152
+ To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
153
+
154
+ ## Contributing
155
+
156
+ Bug reports and pull requests are welcome on GitHub at https://github.com/salsify/avromatic.
157
+
158
+
159
+ ## License
160
+
161
+ The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
162
+
data/Rakefile ADDED
@@ -0,0 +1,22 @@
1
+ require "bundler/gem_tasks"
2
+ require "rspec/core/rake_task"
3
+ require "avro/builder"
4
+
5
+ RSpec::Core::RakeTask.new(:spec)
6
+
7
+ namespace :avro do
8
+ desc 'Generate Avro schema files used by specs'
9
+ task :generate_spec do
10
+ root = 'spec/avro/dsl'
11
+ Avro::Builder.add_load_path(root)
12
+ Dir["#{root}/**/*.rb"].each do |dsl_file|
13
+ puts "Generating Avro schema from #{dsl_file}"
14
+ output_file = dsl_file.sub('/dsl/', '/schema/').sub(/\.rb$/, '.avsc')
15
+ schema = Avro::Builder.build(File.read(dsl_file))
16
+ FileUtils.mkdir_p(File.dirname(output_file))
17
+ File.write(output_file, schema.end_with?("\n") ? schema : schema << "\n")
18
+ end
19
+ end
20
+ end
21
+
22
+ task :default => :spec
data/avromatic.gemspec ADDED
@@ -0,0 +1,37 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'avromatic/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "avromatic"
8
+ spec.version = Avromatic::VERSION
9
+ spec.authors = ["Salsify Engineering"]
10
+ spec.email = ["engineering@salsify.com"]
11
+
12
+ spec.summary = %q{Generate Ruby models from Avro schemas}
13
+ spec.description = spec.summary
14
+ spec.homepage = "https://github.com/salsify/avromatic.git"
15
+ spec.license = "MIT"
16
+
17
+ spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
18
+ spec.bindir = "exe"
19
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
20
+ spec.require_paths = ["lib"]
21
+
22
+ spec.add_runtime_dependency "avro", ">= 1.7.7"
23
+ spec.add_runtime_dependency "virtus"
24
+ spec.add_runtime_dependency "activesupport"
25
+ spec.add_runtime_dependency "activemodel"
26
+ spec.add_runtime_dependency "avro_turf"
27
+ spec.add_runtime_dependency "private_attr"
28
+
29
+ spec.add_development_dependency "bundler", "~> 1.11"
30
+ spec.add_development_dependency "rake", "~> 10.0"
31
+ spec.add_development_dependency "rspec", "~> 3.0"
32
+ spec.add_development_dependency "simplecov"
33
+ spec.add_development_dependency "webmock"
34
+ spec.add_development_dependency "avro-builder", ">= 0.3.2"
35
+ # For FakeSchemaRegistryServer
36
+ spec.add_development_dependency "sinatra"
37
+ end
data/bin/console ADDED
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "avromatic"
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ # require "pry"
11
+ # Pry.start
12
+
13
+ require "irb"
14
+ IRB.start
data/bin/setup ADDED
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
@@ -0,0 +1,141 @@
1
+ require 'active_support/core_ext/object/duplicable'
2
+ require 'ice_nine/core_ext/object'
3
+
4
+ module Avromatic
5
+ module Model
6
+
7
+ # This module supports defining Virtus attributes for a model based on the
8
+ # fields of Avro schemas.
9
+ module Attributes
10
+ extend ActiveSupport::Concern
11
+
12
+ module ClassMethods
13
+ def add_avro_fields
14
+ if key_avro_schema
15
+ check_for_field_conflicts!
16
+ define_avro_attributes(key_avro_schema)
17
+ end
18
+ define_avro_attributes(avro_schema)
19
+ end
20
+
21
+ private
22
+
23
+ def check_for_field_conflicts!
24
+ (key_avro_field_names & value_avro_field_names).each_with_object([]) do |name, conflicts|
25
+ if schema_fields_differ?(name)
26
+ conflicts << "Field '#{name}' has a different type in each schema: "\
27
+ "value #{value_avro_fields_by_name[name]}, "\
28
+ "key #{key_avro_fields_by_name[name]}"
29
+ end
30
+ end.tap do |conflicts|
31
+ raise conflicts.join("\n") if conflicts.any?
32
+ end
33
+ end
34
+
35
+ # The Avro::Schema::Field#== method is lame. It just compares
36
+ # <field>.type.type_sym.
37
+ def schema_fields_differ?(name)
38
+ key_avro_fields_by_name[name].to_avro !=
39
+ value_avro_fields_by_name[name].to_avro
40
+ end
41
+
42
+ def define_avro_attributes(schema)
43
+ schema.fields.each do |field|
44
+ field_class = avro_field_class(field.type)
45
+
46
+ attribute(field.name,
47
+ field_class,
48
+ avro_field_options(field))
49
+
50
+ add_validation(field)
51
+ end
52
+ end
53
+
54
+ def add_validation(field)
55
+ case field.type.type_sym
56
+ when :enum
57
+ validates(field.name,
58
+ inclusion: { in: Set.new(field.type.symbols.map(&:freeze)).freeze })
59
+ when :fixed
60
+ validates(field.name, length: { is: field.type.size })
61
+ end
62
+
63
+ add_required_validation(field)
64
+ end
65
+
66
+ def add_required_validation(field)
67
+ if required?(field) && field.default.nil?
68
+ validates(field.name, presence: true)
69
+ end
70
+ end
71
+
72
+ # An optional field is represented as a union where the first member
73
+ # is null.
74
+ def optional?(field)
75
+ field.type.type_sym == :union &&
76
+ field.type.schemas.first.type_sym == :null
77
+ end
78
+
79
+ def required?(field)
80
+ !optional?(field)
81
+ end
82
+
83
+ def avro_field_class(field_type)
84
+ case field_type.type_sym
85
+ when :string, :bytes, :fixed
86
+ String
87
+ when :boolean
88
+ Axiom::Types::Boolean
89
+ when :int, :long
90
+ Integer
91
+ when :float, :double
92
+ Float
93
+ when :enum
94
+ String
95
+ when :null
96
+ NilClass
97
+ when :array
98
+ Array[avro_field_class(field_type.items)]
99
+ when :map
100
+ Hash[String => avro_field_class(field_type.values)]
101
+ when :union
102
+ union_field_class(field_type)
103
+ when :record
104
+ # TODO: This should add the generated model to a module.
105
+ # A hash of generated models should be kept by name for reuse.
106
+ Class.new do
107
+ include Avromatic::Model.build(schema: field_type)
108
+ end
109
+ else
110
+ raise "Unsupported type #{field_type}"
111
+ end
112
+ end
113
+
114
+ def union_field_class(field_type)
115
+ # TODO: This is a hack until I find a better solution for unions with
116
+ # Virtus. This only handles a union for a optional field with :null
117
+ # and one other type.
118
+ schemas = field_type.schemas.reject { |schema| schema.type_sym == :null }
119
+ raise "Only the union of null with one other type is supported #{field_type}" if schemas.size > 1
120
+ avro_field_class(schemas.first)
121
+ end
122
+
123
+ def avro_field_options(field)
124
+ if field.default
125
+ {
126
+ default: default_for(field.default),
127
+ lazy: true
128
+ }
129
+ else
130
+ { }
131
+ end
132
+ end
133
+
134
+ def default_for(value)
135
+ value.duplicable? ? value.dup.deep_freeze : value
136
+ end
137
+ end
138
+
139
+ end
140
+ end
141
+ end
@@ -0,0 +1,65 @@
1
+ require 'virtus'
2
+ require 'active_support/concern'
3
+ require 'active_model'
4
+ require 'avromatic/model/configuration'
5
+ require 'avromatic/model/value_object'
6
+ require 'avromatic/model/configurable'
7
+ require 'avromatic/model/attributes'
8
+ require 'avromatic/model/serialization'
9
+
10
+ module Avromatic
11
+ module Model
12
+
13
+ # This class implements generating models from Avro schemas.
14
+ class Builder
15
+
16
+ attr_reader :mod, :config
17
+
18
+ # For options see Avromatic::Model.build
19
+ def self.model(**options)
20
+ Class.new do
21
+ include Avromatic::Model::Builder.new(**options).mod
22
+
23
+ # Name is required for attribute validations on an anonymous class.
24
+ def self.name
25
+ super || (@name ||= config.avro_schema.name.classify)
26
+ end
27
+ end
28
+ end
29
+
30
+ # For options see Avromatic::Model.build
31
+ def initialize(**options)
32
+ @mod = Module.new
33
+ @config = Avromatic::Model::Configuration.new(**options)
34
+ define_included_method
35
+ end
36
+
37
+ def inclusions
38
+ [
39
+ ActiveModel::Validations,
40
+ Virtus.value_object,
41
+ Avromatic::Model::Configurable,
42
+ Avromatic::Model::Attributes,
43
+ Avromatic::Model::ValueObject,
44
+ Avromatic::Model::Serialization
45
+ ]
46
+ end
47
+
48
+ private
49
+
50
+ def define_included_method
51
+ with_builder do |builder|
52
+ mod.define_singleton_method(:included) do |model_class|
53
+ model_class.include(*builder.inclusions)
54
+ model_class.config = builder.config
55
+ model_class.add_avro_fields
56
+ end
57
+ end
58
+ end
59
+
60
+ def with_builder
61
+ yield(self)
62
+ end
63
+ end
64
+ end
65
+ end
@@ -0,0 +1,43 @@
1
+ module Avromatic
2
+ module Model
3
+
4
+ # This concern adds methods for configuration for a model generated from
5
+ # Avro schema(s).
6
+ module Configurable
7
+ extend ActiveSupport::Concern
8
+
9
+ module ClassMethods
10
+ attr_accessor :config
11
+ delegate :avro_schema, :value_avro_schema, :key_avro_schema, to: :config
12
+
13
+ def value_avro_field_names
14
+ @value_avro_field_names ||= value_avro_schema.fields.map(&:name).map(&:to_sym).freeze
15
+ end
16
+
17
+ def key_avro_field_names
18
+ @key_avro_field_names ||= key_avro_schema.fields.map(&:name).map(&:to_sym).freeze
19
+ end
20
+
21
+ def value_avro_fields_by_name
22
+ @value_avro_fields_by_name ||= mapped_by_name(value_avro_schema)
23
+ end
24
+
25
+ def key_avro_fields_by_name
26
+ @key_avro_fields_by_name ||= mapped_by_name(key_avro_schema)
27
+ end
28
+
29
+ private
30
+
31
+ def mapped_by_name(schema)
32
+ schema.fields.each_with_object(Hash.new) do |field, result|
33
+ result[field.name.to_sym] = field
34
+ end
35
+ end
36
+ end
37
+
38
+ delegate :avro_schema, :value_avro_schema, :key_avro_schema,
39
+ :value_avro_field_names, :key_avro_field_names,
40
+ to: :class
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,47 @@
1
+ module Avromatic
2
+ module Model
3
+
4
+ # This class holds configuration for a model build from Avro schema(s).
5
+ class Configuration
6
+
7
+ attr_reader :avro_schema, :key_avro_schema
8
+ delegate :schema_store, to: Avromatic
9
+
10
+ # Either schema(_name) or value_schema(_name), but not both, must be
11
+ # specified.
12
+ #
13
+ # @param options [Hash]
14
+ # @option options [Avro::Schema] :schema
15
+ # @option options [String, Symbol] :schema_name
16
+ # @option options [Avro::Schema] :value_schema
17
+ # @option options [String, Symbol] :value_schema_name
18
+ # @option options [Avro::Schema] :key_schema
19
+ # @option options [String, Symbol] :key_schema_name
20
+ # @option options [schema store] :schema_store
21
+ def initialize(**options)
22
+ @avro_schema = find_avro_schema(**options)
23
+ raise ArgumentError.new('value_schema(_name) or schema(_name) must be specified') unless avro_schema
24
+ @key_avro_schema = find_schema_by_option(:key_schema, **options)
25
+ end
26
+
27
+ alias_method :value_avro_schema, :avro_schema
28
+
29
+ private
30
+
31
+ def find_avro_schema(**options)
32
+ if (options[:value_schema] || options[:value_schema_name]) &&
33
+ (options[:schema] || options[:schema_name])
34
+ raise ArgumentError.new('Only one of value_schema(_name) and schema(_name) can be specified')
35
+ end
36
+ find_schema_by_option(:value_schema, **options) || find_schema_by_option(:schema, **options)
37
+ end
38
+
39
+ def find_schema_by_option(option_name, **options)
40
+ schema_name_option = :"#{option_name}_name"
41
+ options[option_name] ||
42
+ (options[schema_name_option] && schema_store.find(options[schema_name_option]))
43
+
44
+ end
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,102 @@
1
+ require 'private_attr'
2
+ require 'avro_turf/schema_registry'
3
+
4
+ module Avromatic
5
+ module Model
6
+
7
+ # This class is used to decode Avro messages to their corresponding models.
8
+ class Decoder
9
+ extend PrivateAttr
10
+
11
+ MAGIC_BYTE = [0].pack("C").freeze
12
+
13
+ class UnexpectedKeyError < StandardError
14
+ def initialize(schema_key)
15
+ super("Unexpected schemas #{schema_key}")
16
+ end
17
+ end
18
+
19
+ class MagicByteError < StandardError
20
+ def initialize(magic_byte)
21
+ super("Expected data to begin with a magic byte, got '#{magic_byte}'")
22
+ end
23
+ end
24
+
25
+ class DuplicateKeyError < StandardError
26
+ def initialize(*models)
27
+ super("Multiple models #{models} have the same key "\
28
+ "'#{Avromatic::Model::Decoder.model_key(models.first)}'")
29
+ end
30
+ end
31
+
32
+ private_attr_reader :schema_names_by_id, :model_map, :schema_registry
33
+
34
+ def self.model_key(model)
35
+ [model.key_avro_schema && model.key_avro_schema.fullname,
36
+ model.value_avro_schema.fullname]
37
+ end
38
+
39
+ delegate :model_key, to: :class
40
+
41
+ # @param *models [generated models] Models to register for decoding.
42
+ # @param schema_registry [Avromatic::SchemaRegistryClient] Optional schema
43
+ # registry client.
44
+ # @param registry_url [String] Optional URL for schema registry server.
45
+ def initialize(*models, schema_registry: nil, registry_url: nil)
46
+ @model_map = build_model_map(models)
47
+ @schema_names_by_id = {}
48
+ @schema_registry = schema_registry ||
49
+ (registry_url && AvroTurf::SchemaRegistry.new(registry_url, logger: Avromatic.logger)) ||
50
+ Avromatic.build_schema_registry
51
+ end
52
+
53
+ # If two arguments are specified then the first is interpreted as the
54
+ # message key and the second is the message value. If there is only one
55
+ # arg then it is used as the message value.
56
+ # @return [Avromatic model]
57
+ def decode(*args)
58
+ message_key, message_value = args.size > 1 ? args : [nil, args.first]
59
+ value_schema_name = schema_name_for_data(message_value)
60
+ key_schema_name = schema_name_for_data(message_key) if message_key
61
+ deserialize([key_schema_name, value_schema_name], message_key, message_value)
62
+ end
63
+
64
+ private
65
+
66
+ def deserialize(model_key, message_key, message_value)
67
+ raise UnexpectedKeyError.new(model_key) unless model_map.key?(model_key)
68
+ model_map[model_key].deserialize(message_key, message_value)
69
+ end
70
+
71
+ def schema_name_for_data(data)
72
+ validate_magic_byte!(data)
73
+ schema_id = extract_schema_id(data)
74
+ lookup_schema_name(schema_id)
75
+ end
76
+
77
+ def lookup_schema_name(schema_id)
78
+ schema_names_by_id.fetch(schema_id) do
79
+ schema = Avro::Schema.parse(schema_registry.fetch(schema_id))
80
+ schema_names_by_id[schema_id] = schema.fullname
81
+ end
82
+ end
83
+
84
+ def extract_schema_id(data)
85
+ data[1..4].unpack('N').first
86
+ end
87
+
88
+ def validate_magic_byte!(data)
89
+ first_byte = data[0]
90
+ raise MagicByteError.new(first_byte) if first_byte != MAGIC_BYTE
91
+ end
92
+
93
+ def build_model_map(models)
94
+ models.each_with_object(Hash.new) do |model, map|
95
+ key = model_key(model)
96
+ raise DuplicateKeyError.new(map[key], model) if map.key?(key) && !model.equal?(map[key])
97
+ map[key] = model
98
+ end
99
+ end
100
+ end
101
+ end
102
+ end
@@ -0,0 +1,81 @@
1
+ require 'avro_turf/messaging'
2
+
3
+ module Avromatic
4
+ module Model
5
+
6
+ # This concern adds support for serialization to a model
7
+ # generated from Avro schema(s).
8
+ module Serialization
9
+ extend ActiveSupport::Concern
10
+
11
+ delegate :messaging, to: :class
12
+
13
+ included do |model_class|
14
+ model_class.messaging = Avromatic.messaging ||
15
+ Avromatic.build_messaging
16
+ end
17
+
18
+ module Encode
19
+ def avro_message_value
20
+ messaging.encode(
21
+ value_attributes_for_avro,
22
+ schema_name: value_avro_schema.fullname)
23
+ end
24
+
25
+ def avro_message_key
26
+ raise 'Model has no key schema' unless key_avro_schema
27
+ messaging.encode(
28
+ key_attributes_for_avro,
29
+ schema_name: key_avro_schema.fullname)
30
+ end
31
+
32
+ protected
33
+
34
+ def value_attributes_for_avro
35
+ avro_hash(value_avro_field_names)
36
+ end
37
+
38
+ private
39
+
40
+ def key_attributes_for_avro
41
+ avro_hash(key_avro_field_names)
42
+ end
43
+
44
+ def avro_hash(fields)
45
+ attributes.slice(*fields).each_with_object(Hash.new) do |(key, value), result|
46
+ result[key.to_s] = if value.is_a?(Avromatic::Model::Attributes)
47
+ value.value_attributes_for_avro
48
+ else
49
+ value
50
+ end
51
+ end
52
+ end
53
+ end
54
+ include Encode
55
+
56
+ # This module provides methods to deserialize an Avro-encoded value and
57
+ # an optional Avro-encoded key as a new model instance.
58
+ module Decode
59
+
60
+ # If two arguments are specified then the first is interpreted as the
61
+ # message key and the second is the message value. If there is only one
62
+ # arg then it is used as the message value.
63
+ def deserialize(*args)
64
+ message_key, message_value = args.size > 1 ? args : [nil, args.first]
65
+ key_attributes = message_key && messaging.decode(message_key, schema_name: key_avro_schema.fullname)
66
+ value_attributes = messaging.decode(message_value, schema_name: avro_schema.fullname)
67
+
68
+ new(value_attributes.merge!(key_attributes || {}))
69
+ end
70
+ end
71
+
72
+ module ClassMethods
73
+ # The messaging object acts as an intermediary talking to the schema
74
+ # registry and using returned/specified schemas to decode/encode.
75
+ attr_accessor :messaging
76
+
77
+ include Decode
78
+ end
79
+ end
80
+ end
81
+ end
@@ -0,0 +1,25 @@
1
+ module Avromatic
2
+ module Model
3
+
4
+ # This module is used to override the comparisons defined by
5
+ # Virtus::Equalizer which is pulled in by Virtus::ValueObject.
6
+ module ValueObject
7
+ def eql?(other)
8
+ other.instance_of?(self.class) && attributes == other.attributes
9
+ end
10
+ alias_method :==, :eql?
11
+
12
+ def hash
13
+ attributes.hash
14
+ end
15
+
16
+ def inspect
17
+ "#<#{self.class.name} #{attributes.map { |k, v| "#{k}: #{v.inspect}" }.join(', ') }>"
18
+ end
19
+
20
+ def to_s
21
+ "#<%s:0x00%x>" % [self.class.name, object_id.abs * 2]
22
+ end
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,49 @@
1
+ require 'avromatic/model/builder'
2
+ require 'avromatic/model/decoder'
3
+
4
+ module Avromatic
5
+ module Model
6
+
7
+ # Returns a module that can be included in a class to define a model
8
+ # based on Avro schema(s).
9
+ #
10
+ # Example:
11
+ # class MyTopic
12
+ # include Avromatic::Model.build(schema_name: :topic_value,
13
+ # key_schema_name: :topic_key)
14
+ # end
15
+ #
16
+ # Either schema(_name) or value_schema(_name) must be specified.
17
+ #
18
+ # value_schema(_name) is handled identically to schema(_name) and is
19
+ # treated like an alias for use when both a value and a key schema are
20
+ # specified.
21
+ #
22
+ # Options:
23
+ # value_schema_name:
24
+ # The full name of an Avro schema. The schema will be loaded
25
+ # using the schema store.
26
+ # value_schema:
27
+ # An Avro::Schema.
28
+ # schema_name:
29
+ # The full name of an Avro schema. The schema will be loaded
30
+ # using the schema store.
31
+ # schema:
32
+ # An Avro::Schema.
33
+ # key_schema_name:
34
+ # The full name of an Avro schema for the key. When an instance of
35
+ # the model is encoded, this schema will be used to encode the key.
36
+ # The schema will be loaded using the schema store.
37
+ # key_schema:
38
+ # An Avro::Schema for the key.
39
+ def self.build(**options)
40
+ Builder.new(**options).mod
41
+ end
42
+
43
+ # Returns an anonymous class, that can be assigned to a constant,
44
+ # defined based on Avro schema(s). See Avromatic::Model.build.
45
+ def self.model(**options)
46
+ Builder.model(**options)
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,9 @@
1
+ module Avromatic
2
+ class Railtie < Rails::Railtie
3
+ initializer 'avromatic.configure' do
4
+ SalsifyAvro.configure do |config|
5
+ config.logger = Rails.logger
6
+ end
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,3 @@
1
+ module Avromatic
2
+ VERSION = "0.1.0"
3
+ end
data/lib/avromatic.rb ADDED
@@ -0,0 +1,33 @@
1
+ require 'avromatic/version'
2
+ require 'avromatic/model'
3
+ require 'avro_turf'
4
+ require 'avro_turf/messaging'
5
+
6
+ module Avromatic
7
+ class << self
8
+ attr_accessor :registry_url, :schema_store, :logger, :messaging
9
+ end
10
+
11
+ self.logger = Logger.new($stdout)
12
+
13
+ def self.configure
14
+ yield self
15
+ end
16
+
17
+ def self.build_schema_registry
18
+ raise 'Avromatic must be configured with a registry_url' unless registry_url
19
+ AvroTurf::CachedSchemaRegistry.new(
20
+ AvroTurf::SchemaRegistry.new(registry_url, logger: logger))
21
+ end
22
+
23
+ def self.build_messaging
24
+ raise 'Avromatic must be configured with a registry_url' unless registry_url
25
+ raise 'Avromatic must be configured with a schema_store' unless schema_store
26
+ AvroTurf::Messaging.new(
27
+ registry_url: Avromatic.registry_url,
28
+ schema_store: Avromatic.schema_store,
29
+ logger: Avromatic.logger)
30
+ end
31
+ end
32
+
33
+ require 'avromatic/railtie' if defined?(Rails)
data/log/.gitkeep ADDED
File without changes
metadata ADDED
@@ -0,0 +1,249 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: avromatic
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Salsify Engineering
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2016-05-06 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: avro
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: 1.7.7
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: 1.7.7
27
+ - !ruby/object:Gem::Dependency
28
+ name: virtus
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: activesupport
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: activemodel
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
69
+ - !ruby/object:Gem::Dependency
70
+ name: avro_turf
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :runtime
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: private_attr
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - ">="
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
90
+ type: :runtime
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ - !ruby/object:Gem::Dependency
98
+ name: bundler
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - "~>"
102
+ - !ruby/object:Gem::Version
103
+ version: '1.11'
104
+ type: :development
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - "~>"
109
+ - !ruby/object:Gem::Version
110
+ version: '1.11'
111
+ - !ruby/object:Gem::Dependency
112
+ name: rake
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - "~>"
116
+ - !ruby/object:Gem::Version
117
+ version: '10.0'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - "~>"
123
+ - !ruby/object:Gem::Version
124
+ version: '10.0'
125
+ - !ruby/object:Gem::Dependency
126
+ name: rspec
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - "~>"
130
+ - !ruby/object:Gem::Version
131
+ version: '3.0'
132
+ type: :development
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - "~>"
137
+ - !ruby/object:Gem::Version
138
+ version: '3.0'
139
+ - !ruby/object:Gem::Dependency
140
+ name: simplecov
141
+ requirement: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - ">="
144
+ - !ruby/object:Gem::Version
145
+ version: '0'
146
+ type: :development
147
+ prerelease: false
148
+ version_requirements: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - ">="
151
+ - !ruby/object:Gem::Version
152
+ version: '0'
153
+ - !ruby/object:Gem::Dependency
154
+ name: webmock
155
+ requirement: !ruby/object:Gem::Requirement
156
+ requirements:
157
+ - - ">="
158
+ - !ruby/object:Gem::Version
159
+ version: '0'
160
+ type: :development
161
+ prerelease: false
162
+ version_requirements: !ruby/object:Gem::Requirement
163
+ requirements:
164
+ - - ">="
165
+ - !ruby/object:Gem::Version
166
+ version: '0'
167
+ - !ruby/object:Gem::Dependency
168
+ name: avro-builder
169
+ requirement: !ruby/object:Gem::Requirement
170
+ requirements:
171
+ - - ">="
172
+ - !ruby/object:Gem::Version
173
+ version: 0.3.2
174
+ type: :development
175
+ prerelease: false
176
+ version_requirements: !ruby/object:Gem::Requirement
177
+ requirements:
178
+ - - ">="
179
+ - !ruby/object:Gem::Version
180
+ version: 0.3.2
181
+ - !ruby/object:Gem::Dependency
182
+ name: sinatra
183
+ requirement: !ruby/object:Gem::Requirement
184
+ requirements:
185
+ - - ">="
186
+ - !ruby/object:Gem::Version
187
+ version: '0'
188
+ type: :development
189
+ prerelease: false
190
+ version_requirements: !ruby/object:Gem::Requirement
191
+ requirements:
192
+ - - ">="
193
+ - !ruby/object:Gem::Version
194
+ version: '0'
195
+ description: Generate Ruby models from Avro schemas
196
+ email:
197
+ - engineering@salsify.com
198
+ executables: []
199
+ extensions: []
200
+ extra_rdoc_files: []
201
+ files:
202
+ - ".gitignore"
203
+ - ".rspec"
204
+ - ".travis.yml"
205
+ - CHANGELOG.md
206
+ - Gemfile
207
+ - LICENSE.txt
208
+ - README.md
209
+ - Rakefile
210
+ - avromatic.gemspec
211
+ - bin/console
212
+ - bin/setup
213
+ - lib/avromatic.rb
214
+ - lib/avromatic/model.rb
215
+ - lib/avromatic/model/attributes.rb
216
+ - lib/avromatic/model/builder.rb
217
+ - lib/avromatic/model/configurable.rb
218
+ - lib/avromatic/model/configuration.rb
219
+ - lib/avromatic/model/decoder.rb
220
+ - lib/avromatic/model/serialization.rb
221
+ - lib/avromatic/model/value_object.rb
222
+ - lib/avromatic/railtie.rb
223
+ - lib/avromatic/version.rb
224
+ - log/.gitkeep
225
+ homepage: https://github.com/salsify/avromatic.git
226
+ licenses:
227
+ - MIT
228
+ metadata: {}
229
+ post_install_message:
230
+ rdoc_options: []
231
+ require_paths:
232
+ - lib
233
+ required_ruby_version: !ruby/object:Gem::Requirement
234
+ requirements:
235
+ - - ">="
236
+ - !ruby/object:Gem::Version
237
+ version: '0'
238
+ required_rubygems_version: !ruby/object:Gem::Requirement
239
+ requirements:
240
+ - - ">="
241
+ - !ruby/object:Gem::Version
242
+ version: '0'
243
+ requirements: []
244
+ rubyforge_project:
245
+ rubygems_version: 2.4.8
246
+ signing_key:
247
+ specification_version: 4
248
+ summary: Generate Ruby models from Avro schemas
249
+ test_files: []