avro_turf 0.8.0 → 1.0.0

data/spec/messaging_spec.rb

@@ -4,29 +4,25 @@ require 'avro_turf/test/fake_confluent_schema_registry_server'
 
 describe AvroTurf::Messaging do
   let(:registry_url) { "http://registry.example.com" }
+  let(:client_cert) { "test client cert" }
+  let(:client_key) { "test client key" }
+  let(:client_key_pass) { "test client key password" }
   let(:logger) { Logger.new(StringIO.new) }
 
   let(:avro) {
     AvroTurf::Messaging.new(
       registry_url: registry_url,
       schemas_path: "spec/schemas",
-      logger: logger
+      logger: logger,
+      client_cert: client_cert,
+      client_key: client_key,
+      client_key_pass: client_key_pass
     )
   }
 
   let(:message) { { "full_name" => "John Doe" } }
-
-  before do
-    FileUtils.mkdir_p("spec/schemas")
-  end
-
-  before do
-    stub_request(:any, /^#{registry_url}/).to_rack(FakeConfluentSchemaRegistryServer)
-    FakeConfluentSchemaRegistryServer.clear
-  end
-
-  before do
-    define_schema "person.avsc", <<-AVSC
+  let(:schema_json) do
+    <<-AVSC
       {
         "name": "person",
         "type": "record",
@@ -39,8 +35,22 @@ describe AvroTurf::Messaging do
       }
     AVSC
   end
+  let(:schema) { Avro::Schema.parse(schema_json) }
 
-  shared_examples_for "encoding and decoding" do
+  before do
+    FileUtils.mkdir_p("spec/schemas")
+  end
+
+  before do
+    stub_request(:any, /^#{registry_url}/).to_rack(FakeConfluentSchemaRegistryServer)
+    FakeConfluentSchemaRegistryServer.clear
+  end
+
+  before do
+    define_schema "person.avsc", schema_json
+  end
+
+  shared_examples_for "encoding and decoding with the schema from schema store" do
     it "encodes and decodes messages" do
       data = avro.encode(message, schema_name: "person")
       expect(avro.decode(data)).to eq message
@@ -60,7 +70,66 @@ describe AvroTurf::Messaging do
     end
   end
 
-  it_behaves_like "encoding and decoding"
+  shared_examples_for 'encoding and decoding with the schema from registry' do
+    before do
+      registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger)
+      registry.register('person', schema)
+      registry.register('people', schema)
+    end
+
+    it 'encodes and decodes messages' do
+      data = avro.encode(message, subject: 'person', version: 1)
+      expect(avro.decode(data)).to eq message
+    end
+
+    it "allows specifying a reader's schema by subject and version" do
+      data = avro.encode(message, subject: 'person', version: 1)
+      expect(avro.decode(data, schema_name: 'person')).to eq message
+    end
+
+    it 'raises AvroTurf::SchemaNotFoundError when the schema does not exist on registry' do
+      expect { avro.encode(message, subject: 'missing', version: 1) }.to raise_error(AvroTurf::SchemaNotFoundError)
+    end
+
+    it 'caches parsed schemas for decoding' do
+      data = avro.encode(message, subject: 'person', version: 1)
+      avro.decode(data)
+      allow(Avro::Schema).to receive(:parse).and_call_original
+      expect(avro.decode(data)).to eq message
+      expect(Avro::Schema).not_to have_received(:parse)
+    end
+  end
+
+  shared_examples_for 'encoding and decoding with the schema_id from registry' do
+    before do
+      registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger)
+      registry.register('person', schema)
+      registry.register('people', schema)
+    end
+
+    it 'encodes and decodes messages' do
+      data = avro.encode(message, schema_id: 1)
+      expect(avro.decode(data)).to eq message
+    end
+
+    it 'raises AvroTurf::SchemaNotFoundError when the schema does not exist on registry' do
+      expect { avro.encode(message, schema_id: 5) }.to raise_error(AvroTurf::SchemaNotFoundError)
+    end
+
+    it 'caches parsed schemas for decoding' do
+      data = avro.encode(message, schema_id: 1)
+      avro.decode(data)
+      allow(Avro::Schema).to receive(:parse).and_call_original
+      expect(avro.decode(data)).to eq message
+      expect(Avro::Schema).not_to have_received(:parse)
+    end
+  end
+
+  it_behaves_like "encoding and decoding with the schema from schema store"
+
+  it_behaves_like 'encoding and decoding with the schema from registry'
+
+  it_behaves_like 'encoding and decoding with the schema_id from registry'
 
   context "with a provided registry" do
     let(:registry) { AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger) }
@@ -73,7 +142,11 @@ describe AvroTurf::Messaging do
       )
     end
 
-    it_behaves_like "encoding and decoding"
+    it_behaves_like "encoding and decoding with the schema from schema store"
+
+    it_behaves_like 'encoding and decoding with the schema from registry'
+
+    it_behaves_like 'encoding and decoding with the schema_id from registry'
 
     it "uses the provided registry" do
       allow(registry).to receive(:register).and_call_original
@@ -101,7 +174,7 @@ describe AvroTurf::Messaging do
       )
     end
 
-    it_behaves_like "encoding and decoding"
+    it_behaves_like "encoding and decoding with the schema from schema store"
 
     it "uses the provided schema store" do
       allow(schema_store).to receive(:find).and_call_original
@@ -109,4 +182,119 @@ describe AvroTurf::Messaging do
       expect(schema_store).to have_received(:find).with("person", nil)
     end
   end
+
+  describe 'decoding with #decode_message' do
+    shared_examples_for "encoding and decoding with the schema from schema store" do
+      it "encodes and decodes messages" do
+        data = avro.encode(message, schema_name: "person")
+        result = avro.decode_message(data)
+        expect(result.message).to eq message
+        expect(result.schema_id).to eq 0
+        expect(result.writer_schema).to eq schema
+        expect(result.reader_schema).to eq nil
+      end
+
+      it "allows specifying a reader's schema" do
+        data = avro.encode(message, schema_name: "person")
+        result = avro.decode_message(data, schema_name: "person")
+        expect(result.message).to eq message
+        expect(result.writer_schema).to eq schema
+        expect(result.reader_schema).to eq schema
+      end
+
+      it "caches parsed schemas for decoding" do
+        data = avro.encode(message, schema_name: "person")
+        avro.decode_message(data)
+        allow(Avro::Schema).to receive(:parse).and_call_original
+        expect(avro.decode_message(data).message).to eq message
+        expect(Avro::Schema).not_to have_received(:parse)
+      end
+    end
+
+    shared_examples_for 'encoding and decoding with the schema from registry' do
+      before do
+        registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger)
+        registry.register('person', schema)
+        registry.register('people', schema)
+      end
+
+      it 'encodes and decodes messages' do
+        data = avro.encode(message, subject: 'person', version: 1)
+        result = avro.decode_message(data)
+        expect(result.message).to eq message
+        expect(result.schema_id).to eq 0
+      end
+
+      it "allows specifying a reader's schema by subject and version" do
+        data = avro.encode(message, subject: 'person', version: 1)
+        expect(avro.decode_message(data, schema_name: 'person').message).to eq message
+      end
+
+      it 'raises AvroTurf::SchemaNotFoundError when the schema does not exist on registry' do
+        expect { avro.encode(message, subject: 'missing', version: 1) }.to raise_error(AvroTurf::SchemaNotFoundError)
+      end
+
+      it 'caches parsed schemas for decoding' do
+        data = avro.encode(message, subject: 'person', version: 1)
+        avro.decode_message(data)
+        allow(Avro::Schema).to receive(:parse).and_call_original
+        expect(avro.decode_message(data).message).to eq message
+        expect(Avro::Schema).not_to have_received(:parse)
+      end
+    end
+
+    it_behaves_like "encoding and decoding with the schema from schema store"
+
+    it_behaves_like 'encoding and decoding with the schema from registry'
+
+    context "with a provided registry" do
+      let(:registry) { AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger) }
+
+      let(:avro) do
+        AvroTurf::Messaging.new(
+          registry: registry,
+          schemas_path: "spec/schemas",
+          logger: logger
+        )
+      end
+
+      it_behaves_like "encoding and decoding with the schema from schema store"
+
+      it_behaves_like 'encoding and decoding with the schema from registry'
+
+      it "uses the provided registry" do
+        allow(registry).to receive(:register).and_call_original
+        message = { "full_name" => "John Doe" }
+        avro.encode(message, schema_name: "person")
+        expect(registry).to have_received(:register).with("person", anything)
+      end
+
+      it "allows specifying a schema registry subject" do
+        allow(registry).to receive(:register).and_call_original
+        message = { "full_name" => "John Doe" }
+        avro.encode(message, schema_name: "person", subject: "people")
+        expect(registry).to have_received(:register).with("people", anything)
+      end
+    end
+
+    context "with a provided schema store" do
+      let(:schema_store) { AvroTurf::SchemaStore.new(path: "spec/schemas") }
+
+      let(:avro) do
+        AvroTurf::Messaging.new(
+          registry_url: registry_url,
+          schema_store: schema_store,
+          logger: logger
+        )
+      end
+
+      it_behaves_like "encoding and decoding with the schema from schema store"
+
+      it "uses the provided schema store" do
+        allow(schema_store).to receive(:find).and_call_original
+        avro.encode(message, schema_name: "person")
+        expect(schema_store).to have_received(:find).with("person", nil)
+      end
+    end
+  end
 end

data/spec/schema_store_spec.rb

@@ -197,6 +197,140 @@ describe AvroTurf::SchemaStore do
       schema = store.find("person")
       expect(schema.fullname).to eq "person"
     end
+
+    # This test would fail under avro_turf <= v0.11.0
+    it "does NOT cache *nested* schemas in memory" do
+      FileUtils.mkdir_p("spec/schemas/test")
+
+      define_schema "test/person.avsc", <<-AVSC
+        {
+          "name": "person",
+          "namespace": "test",
+          "type": "record",
+          "fields": [
+            {
+              "name": "address",
+              "type": {
+                "name": "address",
+                "type": "record",
+                "fields": [
+                  { "name": "addr1", "type": "string" },
+                  { "name": "addr2", "type": "string" },
+                  { "name": "city", "type": "string" },
+                  { "name": "zip", "type": "string" }
+                ]
+              }
+            }
+          ]
+        }
+      AVSC
+
+      schema = store.find('person', 'test')
+      expect(schema.fullname).to eq "test.person"
+
+      expect { store.find('address', 'test') }.
+        to raise_error(AvroTurf::SchemaNotFoundError)
+    end
+
+    # This test would fail under avro_turf <= v0.11.0
+    it "allows two different avsc files to define nested sub-schemas with the same fullname" do
+      FileUtils.mkdir_p("spec/schemas/test")
+
+      define_schema "test/person.avsc", <<-AVSC
+        {
+          "name": "person",
+          "namespace": "test",
+          "type": "record",
+          "fields": [
+            {
+              "name": "location",
+              "type": {
+                "name": "location",
+                "type": "record",
+                "fields": [
+                  { "name": "city", "type": "string" },
+                  { "name": "zipcode", "type": "string" }
+                ]
+              }
+            }
+          ]
+        }
+      AVSC
+
+      define_schema "test/company.avsc", <<-AVSC
+        {
+          "name": "company",
+          "namespace": "test",
+          "type": "record",
+          "fields": [
+            {
+              "name": "headquarters",
+              "type": {
+                "name": "location",
+                "type": "record",
+                "fields": [
+                  { "name": "city", "type": "string" },
+                  { "name": "postcode", "type": "string" }
+                ]
+              }
+            }
+          ]
+        }
+      AVSC
+
+      company = nil
+      person = store.find('person', 'test')
+
+      # This should *NOT* raise the error:
+      # #<Avro::SchemaParseError: The name "test.location" is already in use.>
+      expect { company = store.find('company', 'test') }.not_to raise_error
+
+      person_location_field = person.fields_hash['location']
+      expect(person_location_field.type.name).to eq('location')
+      expect(person_location_field.type.fields_hash).to include('zipcode')
+      expect(person_location_field.type.fields_hash).not_to include('postcode')
+
+      company_headquarters_field = company.fields_hash['headquarters']
+      expect(company_headquarters_field.type.name).to eq('location')
+      expect(company_headquarters_field.type.fields_hash).to include('postcode')
+      expect(company_headquarters_field.type.fields_hash).not_to include('zipcode')
+    end
+
+    it "is thread safe" do
+      define_schema "address.avsc", <<-AVSC
+        {
+          "type": "record",
+          "name": "address",
+          "fields": []
+        }
+      AVSC
+
+      # Set a Thread breakpoint right in the core place of race condition
+      expect(Avro::Name)
+        .to receive(:add_name)
+        .and_wrap_original { |m, *args|
+          Thread.stop
+          m.call(*args)
+        }
+
+      # Run two concurring threads which both will trigger the same schema loading
+      threads = 2.times.map { Thread.new { store.find("address") } }
+      # Wait for the moment when both threads will reach the breakpoint
+      sleep 0.001 until threads.all?(&:stop?)
+
+      expect {
+        # Resume the threads evaluation, one after one
+        threads.each do |thread|
+          next unless thread.status == 'sleep'
+
+          thread.run
+          sleep 0.001 until thread.stop?
+        end
+
+        # Ensure that threads are finished
+        threads.each(&:join)
+      }.to_not raise_error
+    end
   end
 
   describe "#load_schemas!" do

data/spec/spec_helper.rb

@@ -12,6 +12,14 @@ module Helpers
       f.write(content)
     end
   end
+
+  def store_cache(path, hash)
+    File.write(File.join("spec/cache", path), JSON.generate(hash))
+  end
+
+  def load_cache(path)
+    JSON.parse(File.read(File.join("spec/cache", path)))
+  end
 end
 
 RSpec.configure do |config|

data/spec/support/confluent_schema_registry_context.rb

@@ -88,16 +88,18 @@ shared_examples_for "a confluent schema registry client" do
   end
 
   describe "#subject_version" do
-    before do
-      2.times do |n|
-        registry.register(subject_name,
-                          { type: :record, name: "r#{n}", fields: [] }.to_json)
-      end
+    let!(:schema_id1) do
+      registry.register(subject_name, { type: :record, name: "r0", fields: [] }.to_json)
+    end
+    let!(:schema_id2) do
+      registry.register(subject_name, { type: :record, name: "r1", fields: [] }.to_json)
     end
+
     let(:expected) do
       {
         name: subject_name,
         version: 1,
+        id: schema_id1,
         schema: { type: :record, name: "r0", fields: [] }.to_json
       }.to_json
     end
@@ -112,6 +114,7 @@ shared_examples_for "a confluent schema registry client" do
         {
           name: subject_name,
           version: 2,
+          id: schema_id2,
           schema: { type: :record, name: "r1", fields: [] }.to_json
         }.to_json
       end