avro_turf 0.8.1 → 0.11.0

data/lib/avro_turf/schema_store.rb

@@ -3,6 +3,7 @@ class AvroTurf::SchemaStore
   def initialize(path: nil)
     @path = path or raise "Please specify a schema path"
     @schemas = Hash.new
+    @mutex = Mutex.new
   end
 
   # Resolves and returns a schema.
@@ -12,9 +13,40 @@ class AvroTurf::SchemaStore
   # Returns an Avro::Schema.
   def find(name, namespace = nil)
     fullname = Avro::Name.make_fullname(name, namespace)
-
+    # Optimistic non-blocking read from @schemas
+    # No sense to lock the resource when all the schemas already loaded
     return @schemas[fullname] if @schemas.key?(fullname)
 
+    # Pessimistic blocking write to @schemas
+    @mutex.synchronize do
+      # Still need to check is the schema already loaded
+      return @schemas[fullname] if @schemas.key?(fullname)
+
+      load_schema!(fullname, namespace)
+    end
+  end
+
+  # Loads all schema definition files in the `schemas_dir`.
+  def load_schemas!
+    pattern = [@path, "**", "*.avsc"].join("/")
+
+    Dir.glob(pattern) do |schema_path|
+      # Remove the path prefix.
+      schema_path.sub!(/^\/?#{@path}\//, "")
+
+      # Replace `/` with `.` and chop off the file extension.
+      schema_name = File.basename(schema_path.tr("/", "."), ".avsc")
+
+      # Load and cache the schema.
+      find(schema_name)
+    end
+  end
+
+  private
+
+  # Loads single schema
+  # Such method is not thread-safe, do not call it of from mutex synchronization routine
+  def load_schema!(fullname, namespace = nil)
     *namespace, schema_name = fullname.split(".")
     schema_path = File.join(@path, *namespace, schema_name + ".avsc")
     schema_json = JSON.parse(File.read(schema_path))
@@ -29,32 +61,15 @@ class AvroTurf::SchemaStore
     # This is a hack in order to figure out exactly which type was missing. The
     # Avro gem ought to provide this data directly.
     if e.to_s =~ /"([\w\.]+)" is not a schema we know about/
-      find($1)
+      load_schema!($1)
 
       # Re-resolve the original schema now that the dependency has been resolved.
       @schemas.delete(fullname)
-      find(fullname)
+      load_schema!(fullname)
     else
       raise
     end
   rescue Errno::ENOENT, Errno::ENAMETOOLONG
     raise AvroTurf::SchemaNotFoundError, "could not find Avro schema at `#{schema_path}'"
   end
-
-  # Loads all schema definition files in the `schemas_dir`.
-  def load_schemas!
-    pattern = [@path, "**", "*.avsc"].join("/")
-
-    Dir.glob(pattern) do |schema_path|
-      # Remove the path prefix.
-      schema_path.sub!(/^\/?#{@path}\//, "")
-
-      # Replace `/` with `.` and chop off the file extension.
-      schema_name = File.basename(schema_path.tr("/", "."), ".avsc")
-
-      # Load and cache the schema.
-      find(schema_name)
-    end
-  end
-
 end

data/lib/avro_turf/test/fake_confluent_schema_registry_server.rb

@@ -34,10 +34,21 @@ class FakeConfluentSchemaRegistryServer < Sinatra::Base
   end
 
   post "/subjects/:subject/versions" do
-    SCHEMAS << parse_schema
+    schema = parse_schema
+    ids_for_subject = SUBJECTS[params[:subject]]
+
+    schemas_for_subject =
+      SCHEMAS.select
+             .with_index { |_, i| ids_for_subject.include?(i) }
+
+    if schemas_for_subject.include?(schema)
+      schema_id = SCHEMAS.index(schema)
+    else
+      SCHEMAS << schema
+      schema_id = SCHEMAS.size - 1
+      SUBJECTS[params[:subject]] = SUBJECTS[params[:subject]] << schema_id
+    end
 
-    schema_id = SCHEMAS.size - 1
-    SUBJECTS[params[:subject]] = SUBJECTS[params[:subject]] << schema_id
     { id: schema_id }.to_json
   end
 
@@ -73,6 +84,7 @@ class FakeConfluentSchemaRegistryServer < Sinatra::Base
     {
       name: params[:subject],
       version: schema_ids.index(schema_id) + 1,
+      id: schema_id,
       schema: schema
     }.to_json
   end

data/lib/avro_turf/version.rb

@@ -1,3 +1,3 @@
 class AvroTurf
-  VERSION = "0.8.1"
+  VERSION = "0.11.0"
 end

data/spec/cached_confluent_schema_registry_spec.rb

@@ -16,8 +16,9 @@ describe AvroTurf::CachedConfluentSchemaRegistry do
 
   describe "#fetch" do
     it "caches the result of fetch" do
+      # multiple calls return same result, with only one upstream call
       allow(upstream).to receive(:fetch).with(id).and_return(schema)
-      registry.fetch(id)
+      expect(registry.fetch(id)).to eq(schema)
       expect(registry.fetch(id)).to eq(schema)
       expect(upstream).to have_received(:fetch).exactly(1).times
     end
@@ -27,13 +28,34 @@ describe AvroTurf::CachedConfluentSchemaRegistry do
     let(:subject_name) { "a_subject" }
 
     it "caches the result of register" do
+      # multiple calls return same result, with only one upstream call
       allow(upstream).to receive(:register).with(subject_name, schema).and_return(id)
-      registry.register(subject_name, schema)
+      expect(registry.register(subject_name, schema)).to eq(id)
       expect(registry.register(subject_name, schema)).to eq(id)
       expect(upstream).to have_received(:register).exactly(1).times
     end
   end
 
+  describe '#subject_version' do
+    let(:subject_name) { 'a_subject' }
+    let(:version) { 1 }
+    let(:schema_with_meta) do
+      {
+        subject: subject_name,
+        id: 1,
+        version: 1,
+        schema: schema
+      }
+    end
+
+    it 'caches the result of subject_version' do
+      allow(upstream).to receive(:subject_version).with(subject_name, version).and_return(schema_with_meta)
+      registry.subject_version(subject_name, version)
+      registry.subject_version(subject_name, version)
+      expect(upstream).to have_received(:subject_version).exactly(1).times
+    end
+  end
+
   it_behaves_like "a confluent schema registry client" do
     let(:upstream) { AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger) }
     let(:registry) { described_class.new(upstream) }

data/spec/confluent_schema_registry_spec.rb

@@ -3,7 +3,19 @@ require 'avro_turf/confluent_schema_registry'
 require 'avro_turf/test/fake_confluent_schema_registry_server'
 
 describe AvroTurf::ConfluentSchemaRegistry do
+  let(:client_cert) { "test client cert" }
+  let(:client_key) { "test client key" }
+  let(:client_key_pass) { "test client key password" }
+
   it_behaves_like "a confluent schema registry client" do
-    let(:registry) { described_class.new(registry_url, logger: logger) }
+    let(:registry) {
+      described_class.new(
+        registry_url,
+        logger: logger,
+        client_cert: client_cert,
+        client_key: client_key,
+        client_key_pass: client_key_pass
+      )
+    }
   end
 end

data/spec/disk_cached_confluent_schema_registry_spec.rb

@@ -0,0 +1,159 @@
+require 'webmock/rspec'
+require 'avro_turf/cached_confluent_schema_registry'
+require 'avro_turf/test/fake_confluent_schema_registry_server'
+
+describe AvroTurf::CachedConfluentSchemaRegistry do
+  let(:upstream) { instance_double(AvroTurf::ConfluentSchemaRegistry) }
+  let(:cache)    { AvroTurf::DiskCache.new("spec/cache")}
+  let(:registry) { described_class.new(upstream, cache: cache) }
+  let(:id) { rand(999) }
+  let(:schema) do
+    {
+      type: "record",
+      name: "person",
+      fields: [{ name: "name", type: "string" }]
+    }.to_json
+  end
+
+  let(:city_id) { rand(999) }
+  let(:city_schema) do
+    {
+      type: "record",
+      name: "city",
+      fields: [{ name: "name", type: "string" }]
+    }.to_json
+  end
+
+  let(:subject) { 'subject' }
+  let(:version) { rand(999) }
+  let(:subject_version_schema) do
+    {
+      subject: subject,
+      version: version,
+      id: id,
+      schema:  {
+        type: "record",
+        name: "city",
+        fields: { name: "name", type: "string" }
+      }
+    }.to_json
+  end
+
+  before do
+    FileUtils.mkdir_p("spec/cache")
+  end
+
+  describe "#fetch" do
+    let(:cache_before) do
+      {
+        "#{id}" => "#{schema}"
+      }
+    end
+    let(:cache_after) do
+      {
+        "#{id}" => "#{schema}",
+        "#{city_id}" => "#{city_schema}"
+      }
+    end
+
+    # setup the disk cache to avoid performing the upstream fetch
+    before do
+      store_cache("schemas_by_id.json", cache_before)
+    end
+
+    it "uses preloaded disk cache" do
+      # multiple calls return same result, with zero upstream calls
+      allow(upstream).to receive(:fetch).with(id).and_return(schema)
+      expect(registry.fetch(id)).to eq(schema)
+      expect(registry.fetch(id)).to eq(schema)
+      expect(upstream).to have_received(:fetch).exactly(0).times
+      expect(load_cache("schemas_by_id.json")).to eq cache_before
+    end
+
+    it "writes thru to disk cache" do
+      # multiple calls return same result, with only one upstream call
+      allow(upstream).to receive(:fetch).with(city_id).and_return(city_schema)
+      expect(registry.fetch(city_id)).to eq(city_schema)
+      expect(registry.fetch(city_id)).to eq(city_schema)
+      expect(upstream).to have_received(:fetch).exactly(1).times
+      expect(load_cache("schemas_by_id.json")).to eq cache_after
+    end
+  end
+
+  describe "#register" do
+    let(:subject_name) { "a_subject" }
+    let(:cache_before) do
+      {
+        "#{subject_name}#{schema}" => id
+      }
+    end
+
+    let(:city_name) { "a_city" }
+    let(:cache_after) do 
+      {
+        "#{subject_name}#{schema}" => id,
+        "#{city_name}#{city_schema}" => city_id
+      }
+    end
+
+    # setup the disk cache to avoid performing the upstream register
+    before do
+      store_cache("ids_by_schema.json", cache_before)
+    end
+
+    it "uses preloaded disk cache" do
+      # multiple calls return same result, with zero upstream calls
+      allow(upstream).to receive(:register).with(subject_name, schema).and_return(id)
+      expect(registry.register(subject_name, schema)).to eq(id) 
+      expect(registry.register(subject_name, schema)).to eq(id)
+      expect(upstream).to have_received(:register).exactly(0).times
+      expect(load_cache("ids_by_schema.json")).to eq cache_before
+    end
+
+    it "writes thru to disk cache" do
+      # multiple calls return same result, with only one upstream call
+      allow(upstream).to receive(:register).with(city_name, city_schema).and_return(city_id)
+      expect(registry.register(city_name, city_schema)).to eq(city_id)
+      expect(registry.register(city_name, city_schema)).to eq(city_id)
+      expect(upstream).to have_received(:register).exactly(1).times
+      expect(load_cache("ids_by_schema.json")).to eq cache_after
+    end
+  end
+
+  describe "#subject_version" do
+    it "writes thru to disk cache" do
+      # multiple calls return same result, with zero upstream calls
+      allow(upstream).to receive(:subject_version).with(subject, version).and_return(subject_version_schema)
+      expect(File).not_to exist("./spec/cache/schemas_by_subject_version.json")
+
+      expect(registry.subject_version(subject, version)).to eq(subject_version_schema)
+
+      json = JSON.parse(File.read("./spec/cache/schemas_by_subject_version.json"))["#{subject}#{version}"]
+      expect(json).to eq(subject_version_schema)
+
+      expect(registry.subject_version(subject, version)).to eq(subject_version_schema)
+      expect(upstream).to have_received(:subject_version).exactly(1).times
+    end
+
+    it "reads from disk cache and populates mem cache" do
+      allow(upstream).to receive(:subject_version).with(subject, version).and_return(subject_version_schema)
+      key = "#{subject}#{version}"
+      hash = {key => subject_version_schema}
+      cache.send(:write_to_disk_cache, "./spec/cache/schemas_by_subject_version.json", hash)
+
+      cached_schema = cache.instance_variable_get(:@schemas_by_subject_version)
+      expect(cached_schema).to eq({})
+
+      expect(registry.subject_version(subject, version)).to eq(subject_version_schema)
+      expect(upstream).to have_received(:subject_version).exactly(0).times
+
+      cached_schema = cache.instance_variable_get(:@schemas_by_subject_version)
+      expect(cached_schema).to eq({key => subject_version_schema})
+    end
+  end
+
+  it_behaves_like "a confluent schema registry client" do
+    let(:upstream) { AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger) }
+    let(:registry) { described_class.new(upstream) }
+  end
+end

data/spec/messaging_spec.rb

@@ -4,29 +4,25 @@ require 'avro_turf/test/fake_confluent_schema_registry_server'
 
 describe AvroTurf::Messaging do
   let(:registry_url) { "http://registry.example.com" }
+  let(:client_cert) { "test client cert" }
+  let(:client_key) { "test client key" }
+  let(:client_key_pass) { "test client key password" }
   let(:logger) { Logger.new(StringIO.new) }
 
   let(:avro) {
     AvroTurf::Messaging.new(
       registry_url: registry_url,
       schemas_path: "spec/schemas",
-      logger: logger
+      logger: logger,
+      client_cert: client_cert,
+      client_key: client_key,
+      client_key_pass: client_key_pass
     )
   }
 
   let(:message) { { "full_name" => "John Doe" } }
-
-  before do
-    FileUtils.mkdir_p("spec/schemas")
-  end
-
-  before do
-    stub_request(:any, /^#{registry_url}/).to_rack(FakeConfluentSchemaRegistryServer)
-    FakeConfluentSchemaRegistryServer.clear
-  end
-
-  before do
-    define_schema "person.avsc", <<-AVSC
+  let(:schema_json) do
+    <<-AVSC
       {
         "name": "person",
         "type": "record",
@@ -39,8 +35,22 @@ describe AvroTurf::Messaging do
       }
     AVSC
   end
+  let(:schema) { Avro::Schema.parse(schema_json) }
 
-  shared_examples_for "encoding and decoding" do
+  before do
+    FileUtils.mkdir_p("spec/schemas")
+  end
+
+  before do
+    stub_request(:any, /^#{registry_url}/).to_rack(FakeConfluentSchemaRegistryServer)
+    FakeConfluentSchemaRegistryServer.clear
+  end
+
+  before do
+    define_schema "person.avsc", schema_json
+  end
+
+  shared_examples_for "encoding and decoding with the schema from schema store" do
     it "encodes and decodes messages" do
       data = avro.encode(message, schema_name: "person")
       expect(avro.decode(data)).to eq message
@@ -60,7 +70,66 @@ describe AvroTurf::Messaging do
     end
   end
 
-  it_behaves_like "encoding and decoding"
+  shared_examples_for 'encoding and decoding with the schema from registry' do
+    before do
+      registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger)
+      registry.register('person', schema)
+      registry.register('people', schema)
+    end
+
+    it 'encodes and decodes messages' do
+      data = avro.encode(message, subject: 'person', version: 1)
+      expect(avro.decode(data)).to eq message
+    end
+
+    it "allows specifying a reader's schema by subject and version" do
+      data = avro.encode(message, subject: 'person', version: 1)
+      expect(avro.decode(data, schema_name: 'person')).to eq message
+    end
+
+    it 'raises AvroTurf::SchemaNotFoundError when the schema does not exist on registry' do
+      expect { avro.encode(message, subject: 'missing', version: 1) }.to raise_error(AvroTurf::SchemaNotFoundError)
+    end
+
+    it 'caches parsed schemas for decoding' do
+      data = avro.encode(message, subject: 'person', version: 1)
+      avro.decode(data)
+      allow(Avro::Schema).to receive(:parse).and_call_original
+      expect(avro.decode(data)).to eq message
+      expect(Avro::Schema).not_to have_received(:parse)
+    end
+  end
+
+  shared_examples_for 'encoding and decoding with the schema_id from registry' do
+    before do
+      registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger)
+      registry.register('person', schema)
+      registry.register('people', schema)
+    end
+
+    it 'encodes and decodes messages' do
+      data = avro.encode(message, schema_id: 1)
+      expect(avro.decode(data)).to eq message
+    end
+
+    it 'raises AvroTurf::SchemaNotFoundError when the schema does not exist on registry' do
+      expect { avro.encode(message, schema_id: 5) }.to raise_error(AvroTurf::SchemaNotFoundError)
+    end
+
+    it 'caches parsed schemas for decoding' do
+      data = avro.encode(message, schema_id: 1)
+      avro.decode(data)
+      allow(Avro::Schema).to receive(:parse).and_call_original
+      expect(avro.decode(data)).to eq message
+      expect(Avro::Schema).not_to have_received(:parse)
+    end
+  end
+
+  it_behaves_like "encoding and decoding with the schema from schema store"
+
+  it_behaves_like 'encoding and decoding with the schema from registry'
+
+  it_behaves_like 'encoding and decoding with the schema_id from registry'
 
   context "with a provided registry" do
     let(:registry) { AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger) }
@@ -73,7 +142,11 @@ describe AvroTurf::Messaging do
       )
     end
 
-    it_behaves_like "encoding and decoding"
+    it_behaves_like "encoding and decoding with the schema from schema store"
+
+    it_behaves_like 'encoding and decoding with the schema from registry'
+
+    it_behaves_like 'encoding and decoding with the schema_id from registry'
 
     it "uses the provided registry" do
       allow(registry).to receive(:register).and_call_original
@@ -101,7 +174,7 @@ describe AvroTurf::Messaging do
       )
     end
 
-    it_behaves_like "encoding and decoding"
+    it_behaves_like "encoding and decoding with the schema from schema store"
 
     it "uses the provided schema store" do
       allow(schema_store).to receive(:find).and_call_original
@@ -109,4 +182,119 @@ describe AvroTurf::Messaging do
       expect(schema_store).to have_received(:find).with("person", nil)
     end
   end
+
+  describe 'decoding with #decode_message' do
+    shared_examples_for "encoding and decoding with the schema from schema store" do
+      it "encodes and decodes messages" do
+        data = avro.encode(message, schema_name: "person")
+        result = avro.decode_message(data)
+        expect(result.message).to eq message
+        expect(result.schema_id).to eq 0
+        expect(result.writer_schema).to eq schema
+        expect(result.reader_schema).to eq nil
+      end
+
+      it "allows specifying a reader's schema" do
+        data = avro.encode(message, schema_name: "person")
+        result = avro.decode_message(data, schema_name: "person")
+        expect(result.message).to eq message
+        expect(result.writer_schema).to eq schema
+        expect(result.reader_schema).to eq schema
+      end
+
+      it "caches parsed schemas for decoding" do
+        data = avro.encode(message, schema_name: "person")
+        avro.decode_message(data)
+        allow(Avro::Schema).to receive(:parse).and_call_original
+        expect(avro.decode_message(data).message).to eq message
+        expect(Avro::Schema).not_to have_received(:parse)
+      end
+    end
+
+    shared_examples_for 'encoding and decoding with the schema from registry' do
+      before do
+        registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger)
+        registry.register('person', schema)
+        registry.register('people', schema)
+      end
+
+      it 'encodes and decodes messages' do
+        data = avro.encode(message, subject: 'person', version: 1)
+        result = avro.decode_message(data)
+        expect(result.message).to eq message
+        expect(result.schema_id).to eq 0
+      end
+
+      it "allows specifying a reader's schema by subject and version" do
+        data = avro.encode(message, subject: 'person', version: 1)
+        expect(avro.decode_message(data, schema_name: 'person').message).to eq message
+      end
+
+      it 'raises AvroTurf::SchemaNotFoundError when the schema does not exist on registry' do
+        expect { avro.encode(message, subject: 'missing', version: 1) }.to raise_error(AvroTurf::SchemaNotFoundError)
+      end
+
+      it 'caches parsed schemas for decoding' do
+        data = avro.encode(message, subject: 'person', version: 1)
+        avro.decode_message(data)
+        allow(Avro::Schema).to receive(:parse).and_call_original
+        expect(avro.decode_message(data).message).to eq message
+        expect(Avro::Schema).not_to have_received(:parse)
+      end
+    end
+
+    it_behaves_like "encoding and decoding with the schema from schema store"
+
+    it_behaves_like 'encoding and decoding with the schema from registry'
+
+    context "with a provided registry" do
+      let(:registry) { AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger) }
+
+      let(:avro) do
+        AvroTurf::Messaging.new(
+          registry: registry,
+          schemas_path: "spec/schemas",
+          logger: logger
+        )
+      end
+
+      it_behaves_like "encoding and decoding with the schema from schema store"
+
+      it_behaves_like 'encoding and decoding with the schema from registry'
+
+      it "uses the provided registry" do
+        allow(registry).to receive(:register).and_call_original
+        message = { "full_name" => "John Doe" }
+        avro.encode(message, schema_name: "person")
+        expect(registry).to have_received(:register).with("person", anything)
+      end
+
+      it "allows specifying a schema registry subject" do
+        allow(registry).to receive(:register).and_call_original
+        message = { "full_name" => "John Doe" }
+        avro.encode(message, schema_name: "person", subject: "people")
+        expect(registry).to have_received(:register).with("people", anything)
+      end
+    end
+
+    context "with a provided schema store" do
+      let(:schema_store) { AvroTurf::SchemaStore.new(path: "spec/schemas") }
+
+      let(:avro) do
+        AvroTurf::Messaging.new(
+          registry_url: registry_url,
+          schema_store: schema_store,
+          logger: logger
+        )
+      end
+
+      it_behaves_like "encoding and decoding with the schema from schema store"
+
+      it "uses the provided schema store" do
+        allow(schema_store).to receive(:find).and_call_original
+        avro.encode(message, schema_name: "person")
+        expect(schema_store).to have_received(:find).with("person", nil)
+      end
+    end
+  end
 end