avro_turf 0.6.2 → 0.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f8eb00c2c29b7e52a48030bca31e7bebf9e84b49
-  data.tar.gz: f651f0a33989a08b6b17b43b75fa493e083954ba
+  metadata.gz: 4973e043a05b5317719217586f8cee3b9f913d1a
+  data.tar.gz: fb36f804b15c0bed61161d7b73242b7cb0a4cd83
 SHA512:
-  metadata.gz: 5011cc4ad6e53bf61d3fb88f3c89c17cb195044d37fa9bf91ec5f30003f3cea59b0edd9e7618aaae57e307aadc2cf000057032d87432fb91a8118ba8609d6f7f
-  data.tar.gz: 9ffb5c68c4ba50dd19ccf550e3fc861e9ffd7bbaa7117f57f0d7d0a987954f00f95f1385ff6ddf7d5953090623be38f6e2549f89ab1669c81741f66ed7d6292a
+  metadata.gz: 234fa5e60fa5d41ead791b6f48c0db08b2ae02a53848b0fbd31dcf13ee07610800c8407a760e86b9329da3f0b0a2d96ea6c44768f7db9438e18aebe72a3f7ab6
+  data.tar.gz: deaa4c2afa94cabe4d50070d17c3cd455623461d1fb91397e2ed0a2ff473df9892e409836efe7e4bc13d55658de6b340955c1342a8251bee1a52aef2b8d875cb

data/README.md

@@ -101,9 +101,13 @@ The Messaging API will automatically register schemas used for encoding data, an
 
 **NOTE:** The Messaging format is _not_ compatible with the Avro data file API.
 
+The Messaging API is not included by default, so you must require 'avro_turf/messaging' explicitly if you want to use it.
+
 Using the Messaging API is simple once you have set up a Schema Registry service:
 
 ```ruby
+require 'avro_turf/messaging'
+
 # You need to pass the URL of your Schema Registry.
 avro = AvroTurf::Messaging.new(registry_url: "http://my-registry:8081/")
 
@@ -116,3 +120,55 @@ data = avro.encode({ "title" => "hello, world" }, schema_name: "greeting")
 # instances of the same schema id will be served by the cache.
 avro.decode(data) #=> { "title" => "hello, world" }
 ```
+
+In addition to encoding and decoding data, you can check whether a schema is compatible
+with a subject in the registry using the [Compatibility API](http://docs.confluent.io/2.0.0/schema-registry/docs/api.html#compatibility)
+
+```ruby
+require 'avro_turf/messaging'
+
+schema = <<-JSON
+{
+  "name": "person",
+  "type": "record",
+  "fields": [
+    {
+      "name": "full_name",
+      "type": "string"
+    },
+    {
+      "name": "address",
+      "type": "address"
+    }
+  ]
+}
+JSON
+
+avro = AvroTurf::Messaging.new(registry_url: "http://my-registry:8081/")
+
+# Returns true if the schema is compatible, false otherwise.
+avro.compatible?("person", schema)
+```
+
+### Testing Support
+
+AvroTurf includes a `FakeSchemaRegistryServer` that can be used in tests. The
+fake schema registry server depends on Sinatra but it is _not_ listed as a runtime
+dependency for AvroTurf. Sinatra must be added to your Gemfile or gemspec in order
+to use the fake server.
+
+Example using RSpec:
+
+```ruby
+require 'avro_turf/test/fake_schema_registry_server'
+require 'webmock/rspec'
+
+# within an example
+let(:registry_url) { "http://registry.example.com" }
+before do
+  stub_request(:any, /^#{registry_url}/).to_rack(FakeSchemaRegistryServer)
+  FakeSchemaRegistryServer.clear
+end
+
+# Messaging objects created with the same registry_url will now use the fake server.
+```

data/avro_turf.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "avro", ">= 1.7.7", "< 1.9"
-  spec.add_dependency "excon", "~> 0.45.4"
+  spec.add_dependency "excon", "~> 0.45"
 
   spec.add_development_dependency "bundler", "~> 1.7"
   spec.add_development_dependency "rake", "~> 10.0"

data/lib/avro_turf.rb

@@ -3,6 +3,7 @@ require 'avro'
 require 'json'
 require 'avro_turf/schema_store'
 require 'avro_turf/core_ext'
+require 'avro_turf/schema_to_avro_patch'
 
 class AvroTurf
   class Error < StandardError; end

data/lib/avro_turf/cached_schema_registry.rb

@@ -10,7 +10,7 @@ class AvroTurf::CachedSchemaRegistry
   end
 
   # Delegate the following methods to the upstream
-  %i(subjects subject_versions subject_version check).each do |name|
+  %i(subjects subject_versions subject_version check compatible?).each do |name|
     define_method(name) do |*args|
       instance_variable_get(:@upstream).send(name, *args)
     end

data/lib/avro_turf/messaging.rb

@@ -19,15 +19,19 @@ class AvroTurf
 
     # Instantiate a new Messaging instance with the given configuration.
     #
+    # registry     - A schema registry object that responds to all methods in the
+    #                AvroTurf::SchemaRegistry interface.
     # registry_url - The String URL of the schema registry that should be used.
+    # schema_store - A schema store object that responds to #find(schema_name, namespace).
     # schemas_path - The String file system path where local schemas are stored.
     # namespace    - The String default schema namespace.
     # logger       - The Logger that should be used to log information (optional).
-    def initialize(registry_url: nil, schemas_path: nil, namespace: nil, logger: nil)
+    def initialize(registry: nil, registry_url: nil, schema_store: nil, schemas_path: nil, namespace: nil, logger: nil)
       @logger = logger || Logger.new($stderr)
       @namespace = namespace
-      @schema_store = SchemaStore.new(path: schemas_path || DEFAULT_SCHEMAS_PATH)
-      @registry = CachedSchemaRegistry.new(SchemaRegistry.new(registry_url, logger: @logger))
+      @schema_store = schema_store || SchemaStore.new(path: schemas_path || DEFAULT_SCHEMAS_PATH)
+      @registry = registry || CachedSchemaRegistry.new(SchemaRegistry.new(registry_url, logger: @logger))
+      @schemas_by_id = {}
     end
 
     # Encodes a message using the specified schema.
@@ -85,8 +89,10 @@ class AvroTurf
       # The schema id is a 4-byte big-endian integer.
       schema_id = decoder.read(4).unpack("N").first
 
-      writers_schema_json = @registry.fetch(schema_id)
-      writers_schema = Avro::Schema.parse(writers_schema_json)
+      writers_schema = @schemas_by_id.fetch(schema_id) do
+        schema_json = @registry.fetch(schema_id)
+        @schemas_by_id[schema_id] = Avro::Schema.parse(schema_json)
+      end
 
       reader = Avro::IO::DatumReader.new(writers_schema, readers_schema)
       reader.read(decoder)

data/lib/avro_turf/schema_registry.rb

@@ -51,6 +51,16 @@ class AvroTurf::SchemaRegistry
     data unless data.has_key?("error_code")
   end
 
+  # Check if a schema is compatible with the stored version.
+  # Returns true if compatible, false otherwise
+  # http://docs.confluent.io/2.0.0/schema-registry/docs/api.html#compatibility
+  def compatible?(subject, schema, version = 'latest')
+    data = post("/compatibility/subjects/#{subject}/versions/#{version}",
+                expects: [200, 404],
+                body: { schema: schema.to_s }.to_json)
+    data.fetch('is_compatible', false) unless data.has_key?('error_code')
+  end
+
   private
 
   def get(path, **options)

data/lib/avro_turf/schema_to_avro_patch.rb

@@ -0,0 +1,41 @@
+class AvroTurf
+  module AvroGemPatch
+    module RecordSchema
+      module ClassMethods
+        def make_field_objects(field_data, names, namespace=nil)
+          new_field_data = []
+          field_data.each do |field|
+            if field.respond_to?(:[]) && !field.key?('default')
+              field = field.clone
+              field['default'] = :no_default
+            end
+            new_field_data << field
+          end
+          super(new_field_data, names, namespace)
+        end
+      end
+      
+      def self.prepended(base)
+        class << base
+          prepend ClassMethods
+        end
+      end
+    end
+    
+    module Field
+      def initialize(type, name, default=:no_default, order=nil, names=nil, namespace=nil)
+        super(type, name, default, order, names, namespace)
+      end
+      
+      def to_avro(names=Set.new)
+        {'name' => name, 'type' => type.to_avro(names)}.tap do |avro|
+          avro['default'] = default unless default == :no_default
+          avro['order'] = order if order
+        end
+      end
+    end
+  end
+end
+
+Avro::Schema::RecordSchema.send(:prepend, AvroTurf::AvroGemPatch::RecordSchema)
+Avro::Schema::Field.send(:prepend, AvroTurf::AvroGemPatch::Field)

data/lib/avro_turf/version.rb

@@ -1,3 +1,3 @@
 class AvroTurf
-  VERSION = "0.6.2"
+  VERSION = "0.7.1"
 end

data/spec/cached_schema_registry_spec.rb

@@ -1,6 +1,6 @@
 require 'webmock/rspec'
 require 'avro_turf/cached_schema_registry'
-require_relative 'fake_schema_registry_server'
+require 'avro_turf/test/fake_schema_registry_server'
 
 describe AvroTurf::CachedSchemaRegistry do
   let(:upstream) { instance_double(AvroTurf::SchemaRegistry) }

data/spec/messaging_spec.rb

@@ -1,6 +1,6 @@
 require 'webmock/rspec'
 require 'avro_turf/messaging'
-require_relative 'fake_schema_registry_server'
+require 'avro_turf/test/fake_schema_registry_server'
 
 describe AvroTurf::Messaging do
   let(:registry_url) { "http://registry.example.com" }
@@ -14,6 +14,8 @@ describe AvroTurf::Messaging do
     )
   }
 
+  let(:message) { { "full_name" => "John Doe" } }
+
   before do
     FileUtils.mkdir_p("spec/schemas")
   end
@@ -38,42 +40,66 @@ describe AvroTurf::Messaging do
     AVSC
   end
 
-  it "encodes and decodes messages" do
-    message = { "full_name" => "John Doe" }
-    data = avro.encode(message, schema_name: "person")
-    expect(avro.decode(data)).to eq message
-  end
+  shared_examples_for "encoding and decoding" do
+    it "encodes and decodes messages" do
+      data = avro.encode(message, schema_name: "person")
+      expect(avro.decode(data)).to eq message
+    end
 
-  it "allows specifying a reader's schema" do
-    message = { "full_name" => "John Doe" }
-    data = avro.encode(message, schema_name: "person")
-    expect(avro.decode(data, schema_name: "person")).to eq message
+    it "allows specifying a reader's schema" do
+      data = avro.encode(message, schema_name: "person")
+      expect(avro.decode(data, schema_name: "person")).to eq message
+    end
+
+    it "caches parsed schemas for decoding" do
+      data = avro.encode(message, schema_name: "person")
+      avro.decode(data)
+      allow(Avro::Schema).to receive(:parse).and_call_original
+      expect(avro.decode(data)).to eq message
+      expect(Avro::Schema).not_to have_received(:parse)
+    end
   end
 
-  context "when active_support/core_ext is present" do
+  it_behaves_like "encoding and decoding"
+
+  context "with a provided registry" do
+    let(:registry) { AvroTurf::SchemaRegistry.new(registry_url, logger: logger) }
+
     let(:avro) do
-      super().tap do |messaging|
-        # Simulate the presence of active_support/core_ext by monkey patching
-        # the schema store to monkey patch #to_json on the returned schema.
-        schema_store = messaging.instance_variable_get(:@schema_store)
-        def schema_store.find(*)
-          super.extend(Module.new do
-            # Replace to_json on the returned schema with an implementation
-            # that returns something similar to active_support/core_ext/json
-            def to_json(*args)
-              instance_variables.each_with_object(Hash.new) do |ivar, result|
-                result[ivar.to_s.sub('@','')] = instance_variable_get(ivar)
-              end.to_json(*args)
-            end
-          end)
-        end
-      end
+      AvroTurf::Messaging.new(
+        registry: registry,
+        schemas_path: "spec/schemas",
+        logger: logger
+      )
     end
 
-    it "encodes and decodes messages" do
+    it_behaves_like "encoding and decoding"
+
+    it "uses the provided registry" do
+      allow(registry).to receive(:register).and_call_original
       message = { "full_name" => "John Doe" }
-      data = avro.encode(message, schema_name: "person")
-      expect(avro.decode(data)).to eq message
+      avro.encode(message, schema_name: "person")
+      expect(registry).to have_received(:register)
+    end
+  end
+
+  context "with a provided schema store" do
+    let(:schema_store) { AvroTurf::SchemaStore.new(path: "spec/schemas") }
+
+    let(:avro) do
+      AvroTurf::Messaging.new(
+        registry_url: registry_url,
+        schema_store: schema_store,
+        logger: logger
+      )
+    end
+
+    it_behaves_like "encoding and decoding"
+
+    it "uses the provided schema store" do
+      allow(schema_store).to receive(:find).and_call_original
+      avro.encode(message, schema_name: "person")
+      expect(schema_store).to have_received(:find).with("person", nil)
     end
   end
 end

data/spec/schema_registry_spec.rb

@@ -1,6 +1,6 @@
 require 'webmock/rspec'
 require 'avro_turf/schema_registry'
-require_relative 'fake_schema_registry_server'
+require 'avro_turf/test/fake_schema_registry_server'
 
 describe AvroTurf::SchemaRegistry do
   it_behaves_like "a schema registry client" do

data/spec/schema_to_avro_patch_spec.rb

@@ -0,0 +1,24 @@
+require 'webmock/rspec'
+
+# This spec verifies the monkey-patch that we have to apply until the avro
+# gem releases a fix for bug AVRO-1848:
+# https://issues.apache.org/jira/browse/AVRO-1848
+
+describe Avro::Schema do
+  it "correctly handles falsey field defaults" do
+    schema = Avro::Schema.parse <<-SCHEMA
+      {"type": "record", "name": "Record", "namespace": "my.name.space",
+        "fields": [
+          {"name": "is_usable", "type": "boolean", "default": false}
+        ]
+      }
+    SCHEMA
+    
+    expect(schema.to_avro).to eq({
+      'type' => 'record', 'name' => 'Record', 'namespace' => 'my.name.space',
+      'fields' => [
+        {'name' => 'is_usable', 'type' => 'boolean', 'default' => false}
+      ]
+    })
+  end
+end

data/spec/support/schema_registry_context.rb

@@ -26,6 +26,26 @@ shared_examples_for "a schema registry client" do
 
       expect(fetched_schema).to eq(schema)
     end
+
+    context "with an Avro::Schema" do
+      let(:avro_schema) { Avro::Schema.parse(schema) }
+
+      it "allows registration using an Avro::Schema" do
+        id = registry.register(subject_name, avro_schema)
+        expect(registry.fetch(id)).to eq(avro_schema.to_s)
+      end
+
+      context "with ActiveSupport present" do
+        before do
+          break_to_json(avro_schema)
+        end
+
+        it "allows registering an Avro schema" do
+          id = registry.register(subject_name, avro_schema)
+          expect(registry.fetch(id)).to eq(avro_schema.to_s)
+        end
+      end
+    end
   end
 
   describe "#fetch" do
@@ -133,6 +153,22 @@ shared_examples_for "a schema registry client" do
       it "returns the schema details" do
         expect(registry.check(subject_name, schema)).to eq(JSON.parse(expected))
       end
+
+      context "with an Avro::Schema" do
+        let(:avro_schema) { Avro::Schema.parse(schema) }
+
+        it "supports a check using an Avro schema" do
+          expect(registry.check(subject_name, avro_schema)).to eq(JSON.parse(expected))
+        end
+
+        context "with ActiveSupport present" do
+          before { break_to_json(avro_schema) }
+
+          it "supports a check using an Avro schema" do
+            expect(registry.check(subject_name, avro_schema)).to eq(JSON.parse(expected))
+          end
+        end
+      end
     end
 
     context "when the schema is not registered" do
@@ -141,4 +177,14 @@ shared_examples_for "a schema registry client" do
       end
     end
   end
+
+  # Monkey patch an Avro::Schema to simulate the presence of
+  # active_support/core_ext.
+  def break_to_json(avro_schema)
+    def avro_schema.to_json(*args)
+      instance_variables.each_with_object(Hash.new) do |ivar, result|
+        result[ivar.to_s.sub('@', '')] = instance_variable_get(ivar)
+      end.to_json(*args)
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: avro_turf
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.7.1
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-03 00:00:00.000000000 Z
+date: 2016-08-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro
@@ -36,14 +36,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.45.4
+        version: '0.45'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.45.4
+        version: '0.45'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -173,6 +173,8 @@ files:
 - lib/avro_turf/messaging.rb
 - lib/avro_turf/schema_registry.rb
 - lib/avro_turf/schema_store.rb
+- lib/avro_turf/schema_to_avro_patch.rb
+- lib/avro_turf/test/fake_schema_registry_server.rb
 - lib/avro_turf/version.rb
 - perf/address.avsc
 - perf/encoding_size.rb
@@ -190,10 +192,10 @@ files:
 - spec/core_ext/symbol_spec.rb
 - spec/core_ext/time_spec.rb
 - spec/core_ext/true_class_spec.rb
-- spec/fake_schema_registry_server.rb
 - spec/messaging_spec.rb
 - spec/schema_registry_spec.rb
 - spec/schema_store_spec.rb
+- spec/schema_to_avro_patch_spec.rb
 - spec/spec_helper.rb
 - spec/support/schema_registry_context.rb
 homepage: https://github.com/dasch/avro_turf
@@ -234,10 +236,10 @@ test_files:
 - spec/core_ext/symbol_spec.rb
 - spec/core_ext/time_spec.rb
 - spec/core_ext/true_class_spec.rb
-- spec/fake_schema_registry_server.rb
 - spec/messaging_spec.rb
 - spec/schema_registry_spec.rb
 - spec/schema_store_spec.rb
+- spec/schema_to_avro_patch_spec.rb
 - spec/spec_helper.rb
 - spec/support/schema_registry_context.rb
 has_rdoc: