avro_turf 0.7.1 → 0.10.0

data/spec/messaging_spec.rb

@@ -1,6 +1,6 @@
 require 'webmock/rspec'
 require 'avro_turf/messaging'
-require 'avro_turf/test/fake_schema_registry_server'
+require 'avro_turf/test/fake_confluent_schema_registry_server'
 
 describe AvroTurf::Messaging do
   let(:registry_url) { "http://registry.example.com" }
@@ -15,18 +15,8 @@ describe AvroTurf::Messaging do
   }
 
   let(:message) { { "full_name" => "John Doe" } }
-
-  before do
-    FileUtils.mkdir_p("spec/schemas")
-  end
-
-  before do
-    stub_request(:any, /^#{registry_url}/).to_rack(FakeSchemaRegistryServer)
-    FakeSchemaRegistryServer.clear
-  end
-
-  before do
-    define_schema "person.avsc", <<-AVSC
+  let(:schema_json) do
+    <<-AVSC
       {
         "name": "person",
         "type": "record",
@@ -39,8 +29,22 @@ describe AvroTurf::Messaging do
       }
     AVSC
   end
+  let(:schema) { Avro::Schema.parse(schema_json) }
 
-  shared_examples_for "encoding and decoding" do
+  before do
+    FileUtils.mkdir_p("spec/schemas")
+  end
+
+  before do
+    stub_request(:any, /^#{registry_url}/).to_rack(FakeConfluentSchemaRegistryServer)
+    FakeConfluentSchemaRegistryServer.clear
+  end
+
+  before do
+    define_schema "person.avsc", schema_json
+  end
+
+  shared_examples_for "encoding and decoding with the schema from schema store" do
     it "encodes and decodes messages" do
       data = avro.encode(message, schema_name: "person")
       expect(avro.decode(data)).to eq message
@@ -60,10 +64,69 @@ describe AvroTurf::Messaging do
     end
   end
 
-  it_behaves_like "encoding and decoding"
+  shared_examples_for 'encoding and decoding with the schema from registry' do
+    before do
+      registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger)
+      registry.register('person', schema)
+      registry.register('people', schema)
+    end
+
+    it 'encodes and decodes messages' do
+      data = avro.encode(message, subject: 'person', version: 1)
+      expect(avro.decode(data)).to eq message
+    end
+
+    it "allows specifying a reader's schema by subject and version" do
+      data = avro.encode(message, subject: 'person', version: 1)
+      expect(avro.decode(data, schema_name: 'person')).to eq message
+    end
+
+    it 'raises AvroTurf::SchemaNotFoundError when the schema does not exist on registry' do
+      expect { avro.encode(message, subject: 'missing', version: 1) }.to raise_error(AvroTurf::SchemaNotFoundError)
+    end
+
+    it 'caches parsed schemas for decoding' do
+      data = avro.encode(message, subject: 'person', version: 1)
+      avro.decode(data)
+      allow(Avro::Schema).to receive(:parse).and_call_original
+      expect(avro.decode(data)).to eq message
+      expect(Avro::Schema).not_to have_received(:parse)
+    end
+  end
+
+  shared_examples_for 'encoding and decoding with the schema_id from registry' do
+    before do
+      registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger)
+      registry.register('person', schema)
+      registry.register('people', schema)
+    end
+
+    it 'encodes and decodes messages' do
+      data = avro.encode(message, schema_id: 1)
+      expect(avro.decode(data)).to eq message
+    end
+
+    it 'raises AvroTurf::SchemaNotFoundError when the schema does not exist on registry' do
+      expect { avro.encode(message, schema_id: 5) }.to raise_error(AvroTurf::SchemaNotFoundError)
+    end
+
+    it 'caches parsed schemas for decoding' do
+      data = avro.encode(message, schema_id: 1)
+      avro.decode(data)
+      allow(Avro::Schema).to receive(:parse).and_call_original
+      expect(avro.decode(data)).to eq message
+      expect(Avro::Schema).not_to have_received(:parse)
+    end
+  end
+
+  it_behaves_like "encoding and decoding with the schema from schema store"
+
+  it_behaves_like 'encoding and decoding with the schema from registry'
+
+  it_behaves_like 'encoding and decoding with the schema_id from registry'
 
   context "with a provided registry" do
-    let(:registry) { AvroTurf::SchemaRegistry.new(registry_url, logger: logger) }
+    let(:registry) { AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger) }
 
     let(:avro) do
       AvroTurf::Messaging.new(
@@ -73,13 +136,24 @@ describe AvroTurf::Messaging do
       )
     end
 
-    it_behaves_like "encoding and decoding"
+    it_behaves_like "encoding and decoding with the schema from schema store"
+
+    it_behaves_like 'encoding and decoding with the schema from registry'
+
+    it_behaves_like 'encoding and decoding with the schema_id from registry'
 
     it "uses the provided registry" do
       allow(registry).to receive(:register).and_call_original
       message = { "full_name" => "John Doe" }
       avro.encode(message, schema_name: "person")
-      expect(registry).to have_received(:register)
+      expect(registry).to have_received(:register).with("person", anything)
+    end
+
+    it "allows specifying a schema registry subject" do
+      allow(registry).to receive(:register).and_call_original
+      message = { "full_name" => "John Doe" }
+      avro.encode(message, schema_name: "person", subject: "people")
+      expect(registry).to have_received(:register).with("people", anything)
     end
   end
 
@@ -94,7 +168,7 @@ describe AvroTurf::Messaging do
       )
     end
 
-    it_behaves_like "encoding and decoding"
+    it_behaves_like "encoding and decoding with the schema from schema store"
 
     it "uses the provided schema store" do
       allow(schema_store).to receive(:find).and_call_original
@@ -102,4 +176,119 @@ describe AvroTurf::Messaging do
       expect(schema_store).to have_received(:find).with("person", nil)
     end
   end
+
+  describe 'decoding with #decode_message' do
+    shared_examples_for "encoding and decoding with the schema from schema store" do
+      it "encodes and decodes messages" do
+        data = avro.encode(message, schema_name: "person")
+        result = avro.decode_message(data)
+        expect(result.message).to eq message
+        expect(result.schema_id).to eq 0
+        expect(result.writer_schema).to eq schema
+        expect(result.reader_schema).to eq nil
+      end
+
+      it "allows specifying a reader's schema" do
+        data = avro.encode(message, schema_name: "person")
+        result = avro.decode_message(data, schema_name: "person")
+        expect(result.message).to eq message
+        expect(result.writer_schema).to eq schema
+        expect(result.reader_schema).to eq schema
+      end
+
+      it "caches parsed schemas for decoding" do
+        data = avro.encode(message, schema_name: "person")
+        avro.decode_message(data)
+        allow(Avro::Schema).to receive(:parse).and_call_original
+        expect(avro.decode_message(data).message).to eq message
+        expect(Avro::Schema).not_to have_received(:parse)
+      end
+    end
+
+    shared_examples_for 'encoding and decoding with the schema from registry' do
+      before do
+        registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger)
+        registry.register('person', schema)
+        registry.register('people', schema)
+      end
+
+      it 'encodes and decodes messages' do
+        data = avro.encode(message, subject: 'person', version: 1)
+        result = avro.decode_message(data)
+        expect(result.message).to eq message
+        expect(result.schema_id).to eq 0
+      end
+
+      it "allows specifying a reader's schema by subject and version" do
+        data = avro.encode(message, subject: 'person', version: 1)
+        expect(avro.decode_message(data, schema_name: 'person').message).to eq message
+      end
+
+      it 'raises AvroTurf::SchemaNotFoundError when the schema does not exist on registry' do
+        expect { avro.encode(message, subject: 'missing', version: 1) }.to raise_error(AvroTurf::SchemaNotFoundError)
+      end
+
+      it 'caches parsed schemas for decoding' do
+        data = avro.encode(message, subject: 'person', version: 1)
+        avro.decode_message(data)
+        allow(Avro::Schema).to receive(:parse).and_call_original
+        expect(avro.decode_message(data).message).to eq message
+        expect(Avro::Schema).not_to have_received(:parse)
+      end
+    end
+
+    it_behaves_like "encoding and decoding with the schema from schema store"
+
+    it_behaves_like 'encoding and decoding with the schema from registry'
+
+    context "with a provided registry" do
+      let(:registry) { AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger) }
+
+      let(:avro) do
+        AvroTurf::Messaging.new(
+          registry: registry,
+          schemas_path: "spec/schemas",
+          logger: logger
+        )
+      end
+
+      it_behaves_like "encoding and decoding with the schema from schema store"
+
+      it_behaves_like 'encoding and decoding with the schema from registry'
+
+      it "uses the provided registry" do
+        allow(registry).to receive(:register).and_call_original
+        message = { "full_name" => "John Doe" }
+        avro.encode(message, schema_name: "person")
+        expect(registry).to have_received(:register).with("person", anything)
+      end
+
+      it "allows specifying a schema registry subject" do
+        allow(registry).to receive(:register).and_call_original
+        message = { "full_name" => "John Doe" }
+        avro.encode(message, schema_name: "person", subject: "people")
+        expect(registry).to have_received(:register).with("people", anything)
+      end
+    end
+
+    context "with a provided schema store" do
+      let(:schema_store) { AvroTurf::SchemaStore.new(path: "spec/schemas") }
+
+      let(:avro) do
+        AvroTurf::Messaging.new(
+          registry_url: registry_url,
+          schema_store: schema_store,
+          logger: logger
+        )
+      end
+
+      it_behaves_like "encoding and decoding with the schema from schema store"
+
+      it "uses the provided schema store" do
+        allow(schema_store).to receive(:find).and_call_original
+        avro.encode(message, schema_name: "person")
+        expect(schema_store).to have_received(:find).with("person", nil)
+      end
+    end
+  end
 end

data/spec/schema_store_spec.rb

@@ -197,6 +197,42 @@ describe AvroTurf::SchemaStore do
       schema = store.find("person")
       expect(schema.fullname).to eq "person"
     end
+
+    it "is thread safe" do
+      define_schema "address.avsc", <<-AVSC
+        {
+          "type": "record",
+          "name": "address",
+          "fields": []
+        }
+      AVSC
+
+      # Set a Thread breakpoint right in the core place of race condition
+      expect(Avro::Name)
+        .to receive(:add_name)
+        .and_wrap_original { |m, *args|
+          Thread.stop
+          m.call(*args)
+        }
+
+      # Run two concurring threads which both will trigger the same schema loading
+      threads = 2.times.map { Thread.new { store.find("address") } }
+      # Wait for the moment when both threads will reach the breakpoint
+      sleep 0.001 until threads.all?(&:stop?)
+
+      expect {
+        # Resume the threads evaluation, one after one
+        threads.each do |thread|
+          next unless thread.status == 'sleep'
+
+          thread.run
+          sleep 0.001 until thread.stop?
+        end
+
+        # Ensure that threads are finished
+        threads.each(&:join)
+      }.to_not raise_error
+    end
   end
 
   describe "#load_schemas!" do

data/spec/schema_to_avro_patch_spec.rb

@@ -22,3 +22,45 @@ describe Avro::Schema do
     })
   end
 end
+
+
+describe Avro::IO::DatumReader do
+  let(:writer_schema) do
+    Avro::Schema.parse <<-AVSC
+      {
+        "name": "no_default",
+        "type": "record",
+        "fields": [
+          { "type": "string", "name": "one" }
+        ]
+      }
+    AVSC
+  end
+  let(:reader_schema) do
+    Avro::Schema.parse <<-AVSC
+      {
+        "name": "no_default",
+        "type": "record",
+        "fields": [
+          { "type": "string", "name": "one" },
+          { "type": "string", "name": "two" }
+        ]
+      }
+    AVSC
+  end
+
+  it "raises an error for missing fields without a default" do
+    stream = StringIO.new
+    writer = Avro::IO::DatumWriter.new(writer_schema)
+    encoder = Avro::IO::BinaryEncoder.new(stream)
+    writer.write({ 'one' => 'first' }, encoder)
+    encoded = stream.string
+
+    stream = StringIO.new(encoded)
+    decoder = Avro::IO::BinaryDecoder.new(stream)
+    reader = Avro::IO::DatumReader.new(writer_schema, reader_schema)
+    expect do
+      reader.read(decoder)
+    end.to raise_error(Avro::AvroError, 'Missing data for "string" with no default')
+  end
+end

data/spec/spec_helper.rb

@@ -12,6 +12,14 @@ module Helpers
       f.write(content)
     end
   end
+
+  def store_cache(path, hash)
+    File.write(File.join("spec/cache", path), JSON.generate(hash))
+  end
+
+  def load_cache(path)
+    JSON.parse(File.read(File.join("spec/cache", path)))
+  end
 end
 
 RSpec.configure do |config|

data/spec/support/{schema_registry_context.rb → confluent_schema_registry_context.rb}

@@ -1,6 +1,6 @@
 # This shared example expects a registry variable to be defined
 # with an instance of the registry class being tested.
-shared_examples_for "a schema registry client" do
+shared_examples_for "a confluent schema registry client" do
   let(:logger) { Logger.new(StringIO.new) }
   let(:registry_url) { "http://registry.example.com" }
   let(:subject_name) { "some-subject" }
@@ -15,8 +15,8 @@ shared_examples_for "a schema registry client" do
   end
 
   before do
-    stub_request(:any, /^#{registry_url}/).to_rack(FakeSchemaRegistryServer)
-    FakeSchemaRegistryServer.clear
+    stub_request(:any, /^#{registry_url}/).to_rack(FakeConfluentSchemaRegistryServer)
+    FakeConfluentSchemaRegistryServer.clear
   end
 
   describe "#register and #fetch" do
@@ -88,16 +88,18 @@ shared_examples_for "a schema registry client" do
   end
 
   describe "#subject_version" do
-    before do
-      2.times do |n|
-        registry.register(subject_name,
-                          { type: :record, name: "r#{n}", fields: [] }.to_json)
-      end
+    let!(:schema_id1) do
+      registry.register(subject_name, { type: :record, name: "r0", fields: [] }.to_json)
+    end
+    let!(:schema_id2) do
+      registry.register(subject_name, { type: :record, name: "r1", fields: [] }.to_json)
     end
+
     let(:expected) do
       {
         name: subject_name,
         version: 1,
+        id: schema_id1,
         schema: { type: :record, name: "r0", fields: [] }.to_json
       }.to_json
     end
@@ -112,6 +114,7 @@ shared_examples_for "a schema registry client" do
         {
           name: subject_name,
           version: 2,
+          id: schema_id2,
           schema: { type: :record, name: "r1", fields: [] }.to_json
         }.to_json
       end
@@ -178,6 +181,67 @@ shared_examples_for "a schema registry client" do
     end
   end
 
+  describe "#global_config" do
+    let(:expected) do
+      { compatibility: 'BACKWARD' }.to_json
+    end
+
+    it "returns the global configuration" do
+      expect(registry.global_config).to eq(JSON.parse(expected))
+    end
+  end
+
+  describe "#update_global_config" do
+    let(:config) do
+      { compatibility: 'FORWARD' }
+    end
+    let(:expected) { config.to_json }
+
+    it "updates the global configuration and returns it" do
+      expect(registry.update_global_config(config)).to eq(JSON.parse(expected))
+      expect(registry.global_config).to eq(JSON.parse(expected))
+    end
+  end
+
+  describe "#subject_config" do
+    let(:expected) do
+      { compatibility: 'BACKWARD' }.to_json
+    end
+
+    context "when the subject config is not set" do
+      it "returns the global configuration" do
+        expect(registry.subject_config(subject_name)).to eq(JSON.parse(expected))
+      end
+    end
+
+    context "when the subject config is set" do
+      let(:config) do
+        { compatibility: 'FULL' }
+      end
+      let(:expected) { config.to_json }
+
+      before do
+        registry.update_subject_config(subject_name, config)
+      end
+
+      it "returns the subject config" do
+        expect(registry.subject_config(subject_name)).to eq(JSON.parse(expected))
+      end
+    end
+  end
+
+  describe "#update_subject_config" do
+    let(:config) do
+      { compatibility: 'NONE' }
+    end
+    let(:expected) { config.to_json }
+
+    it "updates the subject config and returns it" do
+      expect(registry.update_subject_config(subject_name, config)).to eq(JSON.parse(expected))
+      expect(registry.subject_config(subject_name)).to eq(JSON.parse(expected))
+    end
+  end
+
   # Monkey patch an Avro::Schema to simulate the presence of
   # active_support/core_ext.
   def break_to_json(avro_schema)