avro-builder 0.10.0 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 77208340502bdb2f785d8dfec4074ed2ad29c368
-  data.tar.gz: 8597a93d23ceabc2bb2df55fa3407eaa0afc1077
+  metadata.gz: 927dca1e52261de526556bb1a692ac90406d10bf
+  data.tar.gz: 89e990b81db93413c7f055c1ec95eb23a237b8f3
 SHA512:
-  metadata.gz: 739f0bb19f344589693a8afee826a23b61119941ee6f241a107773f151a97ef64654bf62cffee15429b5a1516622b6aa210eeb846cf166cfa67b6b1d7b4fdb0f
-  data.tar.gz: ee228210b7daaed7973b5c56980f30cea2166f3420cff62dfeaf21fd4ea0787617cbd6722b92bcc5774c71226e8a5a926f866ae4ecaeb66c839bc68203189df7
+  metadata.gz: c69e346811dfbd2574602bd6724b83dc2421a822123c5973b8b6def6b8b9aa552231af78c7778cf2feaac9054ea72145263f2ce41ce5b43eb7351f099dfb480e
+  data.tar.gz: aeb7b896dbe7146a7b4e8d0c4e7eb5533f48c7e15179ff3899ef74f733a91230ec4b9693268bad38792cca40739e7a68265abfcf91cae7c033ebda99d5d51f69

data/.gitignore

@@ -7,3 +7,4 @@
 /pkg/
 /spec/reports/
 /tmp/
+/gemfiles/*.lock

data/.rspec

@@ -1,2 +1,3 @@
 --format documentation
 --color
+--require spec_helper

data/.travis.yml

@@ -2,6 +2,9 @@ language: ruby
 rvm:
   - 2.2.4
   - 2.3.1
+before_script:
+  - bundle exec appraisal install --jobs=3
 script:
   - bundle exec rubocop
-  - bundle exec rspec
+  - bundle exec appraisal avro-official rspec
+  - bundle exec appraisal avro-salsify-fork rspec

data/Appraisals

@@ -0,0 +1,7 @@
+appraise 'avro-official' do
+  gem 'avro', '1.8.1'
+end
+
+appraise 'avro-salsify-fork' do
+  gem 'avro-salsify-fork', '1.9.0.0', require: 'avro'
+end

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 # avro-builder changelog
 
+## v0.11.0
+- Add support for generating schemas that contain logical types. The official
+  Ruby `avro` gem does not yet support logical types. The `avro-salsify-fork` gem
+  can be used to encode and decode logical types.
+- Add support for `union`, `map`, and `array` methods to embed those types
+  within other complex types.
+- Add methods for primitives types (`string`, `int`, `long`, `float`, etc)
+  that allow those types, including a logical type attribute, to be embedded
+  within complex types.
+
 ## v0.10.0
 - Include DSL file line numbers in stack traces.
 - Add optional `filename` keyword argument to `Avro::Builder.build` and

data/README.md

@@ -173,6 +173,20 @@ record :my_record_with_named do
 end
 ```
 
+### Complex Types
+
+Array, maps and unions can each be embedded within another complex type using
+methods that match the type name:
+
+```ruby
+record :complex_types do
+  required :array_of_unions, :array, items: union(:int, :string)
+  required :array_or_map, :union, types: [array(:int), map(:int)]
+end
+```
+
+For more on unions see [below](#unions).
+
 ### Nested Records
 
 Nested records may be created by referring to the name of the previously
@@ -240,6 +254,48 @@ end
 For an optional union, `null` is automatically added as the first type for
 the union and the field defaults to `null`.
 
+### Logical Types
+
+The DSL supports setting a logical type on any type except a union. The logical
+types defined in the Avro [spec](https://avro.apache.org/docs/1.8.1/spec.html#Logical+Types)
+are more limited.
+
+The official Ruby `avro` gem does not yet support logical types:
+[AVRO-1695](https://issues.apache.org/jira/browse/AVRO-1695).
+
+There is a `avro-salsify-fork` gem released from this
+[fork](https://github.com/salsify/avro) that includes changes to support
+encoding and decoding logical types. To use this gem, reference it in your Gemfile:
+
+```ruby
+gem 'avro-salsify-fork', require: 'avro'
+```
+
+A logical type can be specified for a field using the `logical_type` attribute:
+
+```ruby
+record :with_timestamp
+ required :created_at, :long, logical_type: 'timestamp-micros'
+end
+```
+
+Primitive types with a logical type can also be embedded within complex types
+using either the generic `type` method:
+
+```ruby
+record :with_date_array
+  required :date_array, :array, type(:int, logical_type: date)
+end
+```
+
+Or using a primitive type specific method:
+
+```ruby
+record :with_date_array
+  required :date_array, :array, int(logical_type: date)
+end
+```
+
 ### Auto-loading and Imports
 
 Specify paths to search for definitions:

data/Rakefile

@@ -1,6 +1,14 @@
 require 'bundler/gem_tasks'
 require 'rspec/core/rake_task'
+require 'appraisal/task'
 
-RSpec::Core::RakeTask.new(:spec)
+RSpec::Core::RakeTask.new(:default_spec)
 
-task default: :spec
+Appraisal::Task.new
+
+if !ENV['APPRAISAL_INITIALIZED']
+  task default: :appraisal
+  task spec: :appraisal
+else
+  task default: :default_spec
+end

data/avro-builder.gemspec

@@ -21,6 +21,7 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency 'avro', '>= 1.7.0'
 
+  spec.add_development_dependency 'appraisal'
   spec.add_development_dependency 'bundler', '~> 1.11'
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'rspec', '~> 3.0'

data/bin/setup

@@ -4,4 +4,5 @@ IFS=$'\n\t'
 set -vx
 
 bundle install
+appraisal install
 overcommit --install

data/gemfiles/avro_official.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "avro", "1.8.1"
+
+gemspec :path => "../"

data/gemfiles/avro_salsify_fork.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "avro-salsify-fork", "1.9.0.0", :require => "avro"
+
+gemspec :path => "../"

data/lib/avro/builder/anonymous_types.rb

@@ -0,0 +1,34 @@
+module Avro
+  module Builder
+
+    # This concern is included by contexts where anonymous types can be defined.
+    module AnonymousTypes
+      include Avro::Builder::TypeFactory
+
+      Avro::Schema::PRIMITIVE_TYPES_SYM.each do |type_name|
+        define_method(type_name) do |options = {}, &block|
+          type(type_name, options, &block)
+        end
+      end
+
+      def union(*types, &block)
+        type(__method__, { types: types }, &block)
+      end
+
+      def array(items, options = {}, &block)
+        type(__method__, { items: items }.merge(options), &block)
+      end
+
+      def map(values, options = {}, &block)
+        type(__method__, { values: values }.merge(options), &block)
+      end
+
+      def type(type_name, options = {}, &block)
+        create_and_configure_builtin_type(type_name,
+                                          cache: cache,
+                                          internal: options,
+                                          &block)
+      end
+    end
+  end
+end

data/lib/avro/builder/dsl.rb

@@ -5,6 +5,7 @@ require 'avro/builder/dsl_attributes'
 require 'avro/builder/namespaceable'
 require 'avro/builder/definition_cache'
 require 'avro/builder/type_factory'
+require 'avro/builder/anonymous_types'
 require 'avro/builder/types'
 require 'avro/builder/field'
 require 'avro/builder/record'
@@ -89,15 +90,12 @@ module Avro
       end
 
       def create_named_type(name, avro_type_name, options = {}, &block)
-        create_and_configure_builtin_type(avro_type_name,
-                                          cache: cache,
-                                          internal: { _name: name,
-                                                      namespace: namespace },
-                                          options: options,
-                                          &block).tap do |type|
-          type.validate!
-          @last_object = type
-        end
+        @last_object = create_and_configure_builtin_type(avro_type_name,
+                                                         cache: cache,
+                                                         internal: { _name: name,
+                                                                     namespace: namespace },
+                                                         options: options,
+                                                         &block)
       end
 
       def eval_file(name)

data/lib/avro/builder/field.rb

@@ -9,8 +9,8 @@ module Avro
     class Field
       include Avro::Builder::DslOptions
       include Avro::Builder::DslAttributes
-      include Avro::Builder::TypeFactory
       include Avro::Builder::Aliasable
+      include Avro::Builder::AnonymousTypes
 
       INTERNAL_ATTRIBUTES = %i(optional_field).to_set.freeze
 
@@ -31,29 +31,33 @@ module Avro
           send(key, type_options.delete(key)) if dsl_attribute?(key)
         end
 
-        @type = if builtin_type?(avro_type_name)
-                  create_and_configure_builtin_type(avro_type_name,
-                                                    field: self,
-                                                    cache: cache,
-                                                    internal: internal,
-                                                    options: type_options)
-                else
-                  cache.lookup_named_type(avro_type_name, namespace)
-                end
+        @field_type = if builtin_type?(avro_type_name)
+                        create_and_configure_builtin_type(avro_type_name,
+                                                          field: self,
+                                                          cache: cache,
+                                                          internal: internal,
+                                                          validate_type: false,
+                                                          options: type_options)
+                      elsif avro_type_name.is_a?(Avro::Builder::Types::Type)
+                        raise 'Type name must be an Avro builtin type '\
+                              "or a previously defined type name. Got #{avro_type_name}"
+                      else
+                        cache.lookup_named_type(avro_type_name, namespace)
+                      end
 
         # DSL calls must be evaluated after the type has been constructed
         instance_eval(&block) if block_given?
-        @type.validate!
+        @field_type.validate!
       end
 
       ## Delegate additional DSL calls to the type
 
       def respond_to_missing?(id, _include_all)
-        type.dsl_respond_to?(id) || super
+        field_type.dsl_respond_to?(id) || super
       end
 
       def method_missing(id, *args, &block)
-        type.dsl_respond_to?(id) ? type.send(id, *args, &block) : super
+        field_type.dsl_respond_to?(id) ? field_type.send(id, *args, &block) : super
       end
 
       def name_fragment
@@ -64,7 +68,7 @@ module Avro
       # and return the namespace value from the enclosing record.
       def namespace(value = nil)
         if value
-          type.namespace(value)
+          field_type.namespace(value)
         else
           record.namespace
         end
@@ -73,7 +77,7 @@ module Avro
       # Delegate setting name explicitly via DSL to type
       def name(value = nil)
         if value
-          type.name(value)
+          field_type.name(value)
         else
           # Return the name of the field
           @name
@@ -95,12 +99,12 @@ module Avro
 
       private
 
-      attr_accessor :type, :optional_field, :cache, :record
+      attr_accessor :field_type, :optional_field, :cache, :record
 
       # Optional fields must be serialized as a union -- an array of types.
       def serialized_type(reference_state)
-        result = type.serialize(reference_state)
-        optional_field ? type.class.union_with_null(result) : result
+        result = field_type.serialize(reference_state)
+        optional_field ? field_type.class.union_with_null(result) : result
       end
     end
   end

data/lib/avro/builder/type_factory.rb

@@ -31,11 +31,13 @@ module Avro
                                             cache: nil,
                                             internal: {},
                                             options: {},
+                                            validate_type: true,
                                             &block)
         create_builtin_type(avro_type_name, field: field, cache: cache).tap do |type|
           type.configure_options(internal.merge(options))
           type.cache!
           type.instance_eval(&block) if block_given?
+          type.validate! if validate_type
         end
       end
 

data/lib/avro/builder/types.rb

@@ -1,6 +1,5 @@
 require 'avro/builder/types/type'
 require 'avro/builder/types/complex_type'
-require 'avro/builder/types/configurable_type'
 require 'avro/builder/types/type_referencer'
 require 'avro/builder/types/named_type'
 require 'avro/builder/types/record_type'

data/lib/avro/builder/types/array_type.rb

@@ -3,7 +3,6 @@ module Avro
     module Types
       class ArrayType < Type
         include Avro::Builder::Types::ComplexType
-        include Avro::Builder::Types::ConfigurableType
         include Avro::Builder::Types::TypeReferencer
 
         dsl_attribute :items do |items_type = nil|
@@ -19,10 +18,8 @@ module Avro
         end
 
         def serialize(referenced_state)
-          {
-            type: avro_type_name,
-            items: items.serialize(referenced_state)
-          }
+          super(referenced_state,
+                overrides: { items: items.serialize(referenced_state) })
         end
       end
     end

data/lib/avro/builder/types/complex_type.rb

@@ -19,6 +19,13 @@ module Avro
           field.namespace
         end
 
+        def serialize(_reference_state, overrides: {})
+          {
+            type: avro_type_name,
+            logicalType: logical_type
+          }.merge(overrides).reject { |_, v| v.nil? }
+        end
+
         module ClassMethods
 
           # Infer avro_type_name based on class

data/lib/avro/builder/types/map_type.rb

@@ -3,7 +3,6 @@ module Avro
     module Types
       class MapType < Type
         include Avro::Builder::Types::ComplexType
-        include Avro::Builder::Types::ConfigurableType
         include Avro::Builder::Types::TypeReferencer
 
         dsl_attribute :values do |value_type = nil|
@@ -15,10 +14,8 @@ module Avro
         end
 
         def serialize(referenced_state)
-          {
-            type: avro_type_name,
-            values: values.serialize(referenced_state)
-          }
+          super(referenced_state,
+                overrides: { values: values.serialize(referenced_state) })
         end
 
         def validate!

data/lib/avro/builder/types/named_type.rb

@@ -1,4 +1,3 @@
-require 'avro/builder/types/configurable_type'
 require 'avro/builder/namespaceable'
 require 'avro/builder/aliasable'
 require 'avro/builder/types/named_error_handling'
@@ -12,7 +11,6 @@ module Avro
       class NamedType < Type
         include Avro::Builder::Types::ComplexType
         include Avro::Builder::Namespaceable
-        include Avro::Builder::Types::ConfigurableType
         include Avro::Builder::Aliasable
 
         dsl_option :name, dsl_name: :type_name
@@ -59,11 +57,7 @@ module Avro
         # to the serialized value.
         def serialize(reference_state, overrides: {})
           reference_state.definition_or_reference(fullname) do
-            {
-              name: name,
-              type: avro_type_name,
-              namespace: namespace
-            }.merge(overrides).reject { |_, v| v.nil? }
+            serialized_attribute_hash.merge(overrides).reject { |_, v| v.nil? }
           end
         end
 
@@ -71,12 +65,21 @@ module Avro
         # Subclasses may call super with additional overrides to be added
         # to the hash representation.
         def to_h(_reference_state, overrides: {})
+          serialized_attribute_hash
+            .merge(aliases: aliases)
+            .merge(overrides)
+            .reject { |_, v| v.nil? }
+        end
+
+        private
+
+        def serialized_attribute_hash
           {
             name: name,
             type: avro_type_name,
             namespace: namespace,
-            aliases: aliases
-          }.merge(overrides).reject { |_, v| v.nil? }
+            logicalType: logical_type
+          }
         end
       end
     end

data/lib/avro/builder/types/record_type.rb

@@ -4,6 +4,7 @@ module Avro
       # This class represents a record in an Avro schema. Records may be defined
       # at the top-level or as the type for a field in a record.
       class RecordType < Avro::Builder::Types::NamedType
+        include Avro::Builder::AnonymousTypes
 
         DSL_METHODS = %i(required optional extends).to_set.freeze
 
@@ -64,6 +65,7 @@ module Avro
               namespace: namespace,
               doc: doc,
               aliases: aliases,
+              logicalType: logical_type,
               fields: fields.values.map { |field| field.serialize(reference_state) }
             }.reject { |_, v| v.nil? }
           end

data/lib/avro/builder/types/type.rb

@@ -8,6 +8,8 @@ module Avro
         include Avro::Builder::DslOptions
         include Avro::Builder::DslAttributes
 
+        dsl_attribute :logical_type
+
         attr_reader :avro_type_name
 
         def initialize(avro_type_name, cache:, field: nil)
@@ -17,15 +19,21 @@ module Avro
         end
 
         def serialize(_reference_state)
-          avro_type_name
+          if logical_type
+            { type: avro_type_name, logicalType: logical_type }
+          else
+            avro_type_name
+          end
         end
 
         def namespace
           nil
         end
 
-        def configure_options(_options = {})
-          # No-op
+        def configure_options(options = {})
+          options.each do |key, value|
+            send("#{key}=", value) if dsl_option?(key)
+          end
         end
 
         # Optional fields are represented as a union of the type with :null.

data/lib/avro/builder/types/type_referencer.rb

@@ -8,11 +8,13 @@ module Avro
       module TypeReferencer
         include Avro::Builder::TypeFactory
 
-        def create_builtin_or_lookup_named_type(avro_type_name)
-          if builtin_type?(avro_type_name)
-            create_builtin_type(avro_type_name, field: field, cache: cache)
+        def create_builtin_or_lookup_named_type(avro_type_or_name)
+          if avro_type_or_name.is_a?(Avro::Builder::Types::Type)
+            avro_type_or_name
+          elsif builtin_type?(avro_type_or_name)
+            create_builtin_type(avro_type_or_name, field: field, cache: cache)
           else
-            cache.lookup_named_type(avro_type_name)
+            cache.lookup_named_type(avro_type_or_name)
           end
         end
       end

data/lib/avro/builder/types/union_type.rb

@@ -3,7 +3,6 @@ module Avro
     module Types
       class UnionType < Type
         include Avro::Builder::Types::ComplexType
-        include Avro::Builder::Types::ConfigurableType
         include Avro::Builder::Types::TypeReferencer
 
         NULL_TYPE = 'null'.freeze
@@ -30,6 +29,10 @@ module Avro
         def validate!
           validate_required_attribute!(:types)
         end
+
+        def logical_type=(value)
+          raise AttributeError.new("Logical types are not supported for unions: #{value}.")
+        end
       end
     end
   end

data/lib/avro/builder/version.rb

@@ -1,5 +1,5 @@
 module Avro
   module Builder
-    VERSION = '0.10.0'.freeze
+    VERSION = '0.11.0'.freeze
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: avro-builder
 version: !ruby/object:Gem::Version
-  version: 0.10.0
+  version: 0.11.0
 platform: ruby
 authors:
 - Salsify Engineering
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.7.0
+- !ruby/object:Gem::Dependency
+  name: appraisal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -135,6 +149,7 @@ files:
 - ".rubocop.yml"
 - ".ruby-version"
 - ".travis.yml"
+- Appraisals
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt
@@ -143,8 +158,11 @@ files:
 - avro-builder.gemspec
 - bin/console
 - bin/setup
+- gemfiles/avro_official.gemfile
+- gemfiles/avro_salsify_fork.gemfile
 - lib/avro/builder.rb
 - lib/avro/builder/aliasable.rb
+- lib/avro/builder/anonymous_types.rb
 - lib/avro/builder/definition_cache.rb
 - lib/avro/builder/dsl.rb
 - lib/avro/builder/dsl_attributes.rb
@@ -165,7 +183,6 @@ files:
 - lib/avro/builder/types.rb
 - lib/avro/builder/types/array_type.rb
 - lib/avro/builder/types/complex_type.rb
-- lib/avro/builder/types/configurable_type.rb
 - lib/avro/builder/types/enum_type.rb
 - lib/avro/builder/types/fixed_type.rb
 - lib/avro/builder/types/map_type.rb

data/lib/avro/builder/types/configurable_type.rb

@@ -1,16 +0,0 @@
-module Avro
-  module Builder
-    module Types
-
-      # This concern is used by Types that can be configured via the DSL.
-      # Only attributes that can be set via options are configured here.
-      module ConfigurableType
-        def configure_options(options = {})
-          options.each do |key, value|
-            send("#{key}=", value) if dsl_option?(key)
-          end
-        end
-      end
-    end
-  end
-end