avro_turf 0.8.1 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8e58564680b9399ae8df438412385f23bdabd46cee8deafc0dfa1c8b827d7792
-  data.tar.gz: 1df38f38434777fab06fddec69a8834442a4814e269edbed154fc74153f6b198
+  metadata.gz: 01a2bdf42a996a7a89bab1769672adc11888d973e5c0c70df42d07b63e87e2bf
+  data.tar.gz: c80b96eace3ae4fdb254f046335e7aaf21939cde9da86b671d7450e3d1d2ceb4
 SHA512:
-  metadata.gz: 6e47f299a673911614be989feefb56f2cd48be6a556e240919ad23b13b55928c3cd0837d5d5f43aa4c2f72e1f416465f811de24ae7e344f858dc147dd23be136
-  data.tar.gz: 187c4f087cf7ed656ef3bfed6bf0593938f57da698b0b99550e476a059e529cf355f33ce5bc1839680c1c0513c730e285a117f33ec9e6947c476f0301ab3c597
+  metadata.gz: 978d5323dd68d2a1518dadcaa90c1871b065e184ffd06ca7a9a86ad1402c16a46f557ba5847e0a841a269b38bfdcb0769a66ec961eddd6b40dd4f025c6eb2c9e
+  data.tar.gz: c2abf819298e5c925f60e7d5e6536aced127e1bee0ef4a66893ea25c16f1e8ea9905de9e142e3f7e0a5989901e7708a9ce000cd85384ef2a703d6e77eb27ce7d

data/.circleci/config.yml

@@ -0,0 +1,35 @@
+version: 2
+jobs:
+  build:
+    environment:
+      CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
+      CIRCLE_TEST_REPORTS: /tmp/circleci-test-results
+    docker:
+    - image: circleci/ruby:2.6.2
+    steps:
+    - checkout
+    - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS
+    - restore_cache:
+        keys:
+        # This branch if available
+        - v1-dep-{{ .Branch }}-
+        # Default branch if not
+        - v1-dep-master-
+        # Any branch if there are none on the default branch - this should be unnecessary if you have your default branch configured correctly
+        - v1-dep-
+    - run: 'bundle check --path=vendor/bundle || bundle install --path=vendor/bundle --jobs=4 --retry=3'
+    # Save dependency cache
+    - save_cache:
+        key: v1-dep-{{ .Branch }}-{{ epoch }}
+        paths:
+        - vendor/bundle
+        - ~/.bundle
+    - run: mkdir -p $CIRCLE_TEST_REPORTS/rspec
+    - run:
+        command: bundle exec rspec --color --require spec_helper --format RspecJunitFormatter --out $CIRCLE_TEST_REPORTS/rspec/rspec.xml --format progress spec
+    - store_test_results:
+        path: /tmp/circleci-test-results
+    - store_artifacts:
+        path: /tmp/circleci-artifacts
+    - store_artifacts:
+        path: /tmp/circleci-test-results

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 # avro_turf
 
+## Unreleased
+
+## v0.9.0
+
+- Compatibility with Avro v1.9.0 (#94)
+- Disable the auto registeration of schema (#95)
+- abstracted caching from CachedConfluentSchemaRegistry (#74)
+- Load avro-patches if installed to silence deprecation errors (#85)
+- Make schema store to be thread safe (#92)
+
 ## v0.8.1
 
 - Allow accessing schema store from outside AvroTurf (#68).

data/README.md

@@ -124,6 +124,12 @@ avro = AvroTurf::Messaging.new(registry_url: "http://my-registry:8081/")
 # time a schema is used.
 data = avro.encode({ "title" => "hello, world" }, schema_name: "greeting")
 
+# If you don't want to automatically register new schemas, you can pass explicitly
+# subject and version to specify which schema should be used for encoding.
+# It will fetch that schema from the registry and cache it. Subsequent instances 
+# of the same schema version will be served by the cache.
+data = avro.encode({ "title" => "hello, world" }, subject: 'greeting', version: 1)
+
 # When decoding, the schema will be fetched from the registry and cached. Subsequent
 # instances of the same schema id will be served by the cache.
 avro.decode(data) #=> { "title" => "hello, world" }

data/avro_turf.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "avro", ">= 1.7.7", "< 1.9"
+  spec.add_dependency "avro", ">= 1.7.7", "< 1.10"
   spec.add_dependency "excon", "~> 0.45"
 
   spec.add_development_dependency "bundler", "~> 1.7"
@@ -27,6 +27,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "webmock"
   spec.add_development_dependency "sinatra"
   spec.add_development_dependency "json_spec"
+  spec.add_development_dependency "rack-test"
 
   spec.post_install_message = %{
 avro_turf v0.8.0 deprecates the names AvroTurf::SchemaRegistry,

data/lib/avro_turf.rb

@@ -1,9 +1,18 @@
+begin
+  require 'avro-patches'
+rescue LoadError
+  false
+end
 require 'avro_turf/version'
 require 'avro'
 require 'json'
 require 'avro_turf/schema_store'
 require 'avro_turf/core_ext'
-require 'avro_turf/schema_to_avro_patch'
+
+# check for something that indicates Avro v1.9.0 or later
+unless defined?(::Avro::LogicalTypes)
+  require 'avro_turf/schema_to_avro_patch'
+end
 
 class AvroTurf
   class Error < StandardError; end

data/lib/avro_turf/cached_confluent_schema_registry.rb

@@ -1,16 +1,23 @@
 require 'avro_turf/confluent_schema_registry'
+require 'avro_turf/in_memory_cache'
+require 'avro_turf/disk_cache'
 
 # Caches registrations and lookups to the schema registry in memory.
 class AvroTurf::CachedConfluentSchemaRegistry
 
-  def initialize(upstream)
+  # Instantiate a new CachedConfluentSchemaRegistry instance with the given configuration.
+  # By default, uses a provided InMemoryCache to prevent repeated calls to the upstream registry.
+  #
+  # upstream  - The upstream schema registry object that fully responds to all methods in the
+  #             AvroTurf::ConfluentSchemaRegistry interface.
+  # cache     - Optional user provided Cache object that responds to all methods in the AvroTurf::InMemoryCache interface.
+  def initialize(upstream, cache: nil)
     @upstream = upstream
-    @schemas_by_id = {}
-    @ids_by_schema = {}
+    @cache = cache || AvroTurf::InMemoryCache.new()
   end
 
   # Delegate the following methods to the upstream
-  %i(subjects subject_versions subject_version check compatible?
+  %i(subjects subject_versions check compatible?
      global_config update_global_config subject_config update_subject_config).each do |name|
     define_method(name) do |*args|
       instance_variable_get(:@upstream).send(name, *args)
@@ -18,10 +25,15 @@ class AvroTurf::CachedConfluentSchemaRegistry
   end
 
   def fetch(id)
-    @schemas_by_id[id] ||= @upstream.fetch(id)
+    @cache.lookup_by_id(id) || @cache.store_by_id(id, @upstream.fetch(id))
   end
 
   def register(subject, schema)
-    @ids_by_schema[subject + schema.to_s] ||= @upstream.register(subject, schema)
+    @cache.lookup_by_schema(subject, schema) || @cache.store_by_schema(subject, schema, @upstream.register(subject, schema))
+  end
+
+  def subject_version(subject, version = 'latest')
+    @cache.lookup_by_version(subject, version) ||
+      @cache.store_by_version(subject, version, @upstream.subject_version(subject, version))
   end
 end

data/lib/avro_turf/disk_cache.rb

@@ -0,0 +1,38 @@
+# A cache for the CachedConfluentSchemaRegistry.
+# Extends the InMemoryCache to provide a write-thru to disk for persistent cache.
+class AvroTurf::DiskCache < AvroTurf::InMemoryCache
+
+  def initialize(disk_path)
+    super()
+
+    # load the write-thru cache on startup, if it exists
+    @schemas_by_id_path = File.join(disk_path, 'schemas_by_id.json')
+    @schemas_by_id = JSON.parse(File.read(@schemas_by_id_path)) if File.exist?(@schemas_by_id_path)
+
+    @ids_by_schema_path = File.join(disk_path, 'ids_by_schema.json')
+    @ids_by_schema = JSON.parse(File.read(@ids_by_schema_path)) if File.exist?(@ids_by_schema_path)
+  end
+
+  # override
+  # the write-thru cache (json) does not store keys in numeric format
+  # so, convert id to a string for caching purposes
+  def lookup_by_id(id)
+    super(id.to_s)
+  end
+
+  # override to include write-thru cache after storing result from upstream
+  def store_by_id(id, schema)
+    # must return the value from storing the result (i.e. do not return result from file write)
+    value = super(id.to_s, schema)
+    File.write(@schemas_by_id_path, JSON.pretty_generate(@schemas_by_id))
+    return value
+  end
+
+  # override to include write-thru cache after storing result from upstream
+  def store_by_schema(subject, schema, id)
+    # must return the value from storing the result (i.e. do not return result from file write)
+    value = super
+    File.write(@ids_by_schema_path, JSON.pretty_generate(@ids_by_schema))
+    return value
+  end
+end

data/lib/avro_turf/in_memory_cache.rb

@@ -0,0 +1,38 @@
+# A cache for the CachedConfluentSchemaRegistry.
+# Simply stores the schemas and ids in in-memory hashes.
+class AvroTurf::InMemoryCache
+
+  def initialize
+    @schemas_by_id = {}
+    @ids_by_schema = {}
+    @schema_by_subject_version = {}
+  end
+
+  def lookup_by_id(id)
+    @schemas_by_id[id]
+  end
+
+  def store_by_id(id, schema)
+    @schemas_by_id[id] = schema
+  end
+
+  def lookup_by_schema(subject, schema)
+    key = subject + schema.to_s
+    @ids_by_schema[key]
+  end
+
+  def store_by_schema(subject, schema, id)
+    key = subject + schema.to_s
+    @ids_by_schema[key] = id
+  end
+
+  def lookup_by_version(subject, version)
+    key = "#{subject}#{version}"
+    @schema_by_subject_version[key]
+  end
+
+  def store_by_version(subject, version, schema)
+    key = "#{subject}#{version}"
+    @schema_by_subject_version[key] = schema
+  end
+end

data/lib/avro_turf/messaging.rb

@@ -46,14 +46,20 @@ class AvroTurf
     # schema_name - The String name of the schema that should be used to encode
     #               the data.
     # namespace   - The namespace of the schema (optional).
+    # subject     - The subject name the schema should be registered under in
+    #               the schema registry (optional).
+    # version     - The integer version of the schema that should be used to decode
+    #               the data. Must match the schema used when encoding (optional).
     #
     # Returns the encoded data as a String.
-    def encode(message, schema_name: nil, namespace: @namespace, subject: nil)
-      schema = @schema_store.find(schema_name, namespace)
-
-      # Schemas are registered under the full name of the top level Avro record
-      # type, or `subject` if it's provided.
-      schema_id = @registry.register(subject || schema.fullname, schema)
+    def encode(message, schema_name: nil, namespace: @namespace, subject: nil, version: nil)
+      schema_id, schema = if subject && version
+        fetch_schema(subject, version)
+      elsif schema_name
+        register_schema(subject, schema_name, namespace)
+      else
+        raise ArgumentError.new('Neither schema_name nor subject + version provided to determine the schema.')
+      end
 
       stream = StringIO.new
       writer = Avro::IO::DatumWriter.new(schema)
@@ -69,6 +75,8 @@ class AvroTurf
       writer.write(message, encoder)
 
       stream.string
+    rescue Excon::Error::NotFound
+      raise SchemaNotFoundError.new("Schema with subject: `#{subject}` version: `#{version}` is not found on registry")
     end
 
     # Decodes data into the original message.
@@ -101,6 +109,28 @@ class AvroTurf
 
       reader = Avro::IO::DatumReader.new(writers_schema, readers_schema)
       reader.read(decoder)
+    rescue Excon::Error::NotFound
+      raise SchemaNotFoundError.new("Schema with id: #{schema_id} is not found on registry")
+    end
+
+    private
+
+    # Providing subject and version to determine the schema,
+    # which skips the auto registeration of schema on the schema registry.
+    # Fetch the schema from registry with the provided subject name and version.
+    def fetch_schema(subject, version)
+      schema_data = @registry.subject_version(subject, version)
+      schema_id = schema_data.fetch('id')
+      schema = Avro::Schema.parse(schema_data.fetch('schema'))
+      [schema_id, schema]
+    end
+
+    # Schemas are registered under the full name of the top level Avro record
+    # type, or `subject` if it's provided.
+    def register_schema(subject, schema_name, namespace)
+      schema = @schema_store.find(schema_name, namespace)
+      schema_id = @registry.register(subject || schema.fullname, schema)
+      [schema_id, schema]
     end
   end
 end

data/lib/avro_turf/schema_store.rb

@@ -3,6 +3,7 @@ class AvroTurf::SchemaStore
   def initialize(path: nil)
     @path = path or raise "Please specify a schema path"
     @schemas = Hash.new
+    @mutex = Mutex.new
   end
 
   # Resolves and returns a schema.
@@ -12,9 +13,40 @@ class AvroTurf::SchemaStore
   # Returns an Avro::Schema.
   def find(name, namespace = nil)
     fullname = Avro::Name.make_fullname(name, namespace)
-
+    # Optimistic non-blocking read from @schemas
+    # No sense to lock the resource when all the schemas already loaded
     return @schemas[fullname] if @schemas.key?(fullname)
 
+    # Pessimistic blocking write to @schemas
+    @mutex.synchronize do
+      # Still need to check is the schema already loaded
+      return @schemas[fullname] if @schemas.key?(fullname)
+
+      load_schema!(fullname, namespace)
+    end
+  end
+
+  # Loads all schema definition files in the `schemas_dir`.
+  def load_schemas!
+    pattern = [@path, "**", "*.avsc"].join("/")
+
+    Dir.glob(pattern) do |schema_path|
+      # Remove the path prefix.
+      schema_path.sub!(/^\/?#{@path}\//, "")
+
+      # Replace `/` with `.` and chop off the file extension.
+      schema_name = File.basename(schema_path.tr("/", "."), ".avsc")
+
+      # Load and cache the schema.
+      find(schema_name)
+    end
+  end
+
+  private
+
+  # Loads single schema
+  # Such method is not thread-safe, do not call it of from mutex synchronization routine
+  def load_schema!(fullname, namespace = nil)
     *namespace, schema_name = fullname.split(".")
     schema_path = File.join(@path, *namespace, schema_name + ".avsc")
     schema_json = JSON.parse(File.read(schema_path))
@@ -29,32 +61,15 @@ class AvroTurf::SchemaStore
     # This is a hack in order to figure out exactly which type was missing. The
     # Avro gem ought to provide this data directly.
     if e.to_s =~ /"([\w\.]+)" is not a schema we know about/
-      find($1)
+      load_schema!($1)
 
       # Re-resolve the original schema now that the dependency has been resolved.
       @schemas.delete(fullname)
-      find(fullname)
+      load_schema!(fullname)
     else
       raise
     end
   rescue Errno::ENOENT, Errno::ENAMETOOLONG
     raise AvroTurf::SchemaNotFoundError, "could not find Avro schema at `#{schema_path}'"
   end
-
-  # Loads all schema definition files in the `schemas_dir`.
-  def load_schemas!
-    pattern = [@path, "**", "*.avsc"].join("/")
-
-    Dir.glob(pattern) do |schema_path|
-      # Remove the path prefix.
-      schema_path.sub!(/^\/?#{@path}\//, "")
-
-      # Replace `/` with `.` and chop off the file extension.
-      schema_name = File.basename(schema_path.tr("/", "."), ".avsc")
-
-      # Load and cache the schema.
-      find(schema_name)
-    end
-  end
-
 end

data/lib/avro_turf/test/fake_confluent_schema_registry_server.rb

@@ -34,10 +34,21 @@ class FakeConfluentSchemaRegistryServer < Sinatra::Base
   end
 
   post "/subjects/:subject/versions" do
-    SCHEMAS << parse_schema
+    schema = parse_schema
+    ids_for_subject = SUBJECTS[params[:subject]]
+
+    schemas_for_subject =
+      SCHEMAS.select
+             .with_index { |_, i| ids_for_subject.include?(i) }
+
+    if schemas_for_subject.include?(schema)
+      schema_id = SCHEMAS.index(schema)
+    else
+      SCHEMAS << schema
+      schema_id = SCHEMAS.size - 1
+      SUBJECTS[params[:subject]] = SUBJECTS[params[:subject]] << schema_id
+    end
 
-    schema_id = SCHEMAS.size - 1
-    SUBJECTS[params[:subject]] = SUBJECTS[params[:subject]] << schema_id
     { id: schema_id }.to_json
   end
 
@@ -73,6 +84,7 @@ class FakeConfluentSchemaRegistryServer < Sinatra::Base
     {
       name: params[:subject],
       version: schema_ids.index(schema_id) + 1,
+      id: schema_id,
       schema: schema
     }.to_json
   end

data/lib/avro_turf/version.rb

@@ -1,3 +1,3 @@
 class AvroTurf
-  VERSION = "0.8.1"
+  VERSION = "0.9.0"
 end

data/spec/cached_confluent_schema_registry_spec.rb

@@ -16,8 +16,9 @@ describe AvroTurf::CachedConfluentSchemaRegistry do
 
   describe "#fetch" do
     it "caches the result of fetch" do
+      # multiple calls return same result, with only one upstream call
       allow(upstream).to receive(:fetch).with(id).and_return(schema)
-      registry.fetch(id)
+      expect(registry.fetch(id)).to eq(schema)
       expect(registry.fetch(id)).to eq(schema)
       expect(upstream).to have_received(:fetch).exactly(1).times
     end
@@ -27,13 +28,34 @@ describe AvroTurf::CachedConfluentSchemaRegistry do
     let(:subject_name) { "a_subject" }
 
     it "caches the result of register" do
+      # multiple calls return same result, with only one upstream call
       allow(upstream).to receive(:register).with(subject_name, schema).and_return(id)
-      registry.register(subject_name, schema)
+      expect(registry.register(subject_name, schema)).to eq(id)
       expect(registry.register(subject_name, schema)).to eq(id)
       expect(upstream).to have_received(:register).exactly(1).times
     end
   end
 
+  describe '#subject_version' do
+    let(:subject_name) { 'a_subject' }
+    let(:version) { 1 }
+    let(:schema_with_meta) do
+      {
+        subject: subject_name,
+        id: 1,
+        version: 1,
+        schema: schema
+      }
+    end
+
+    it 'caches the result of subject_version' do
+      allow(upstream).to receive(:subject_version).with(subject_name, version).and_return(schema_with_meta)
+      registry.subject_version(subject_name, version)
+      registry.subject_version(subject_name, version)
+      expect(upstream).to have_received(:subject_version).exactly(1).times
+    end
+  end
+
   it_behaves_like "a confluent schema registry client" do
     let(:upstream) { AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger) }
     let(:registry) { described_class.new(upstream) }

data/spec/disk_cached_confluent_schema_registry_spec.rb

@@ -0,0 +1,112 @@
+require 'webmock/rspec'
+require 'avro_turf/cached_confluent_schema_registry'
+require 'avro_turf/test/fake_confluent_schema_registry_server'
+
+describe AvroTurf::CachedConfluentSchemaRegistry do
+  let(:upstream) { instance_double(AvroTurf::ConfluentSchemaRegistry) }
+  let(:cache)    { AvroTurf::DiskCache.new("spec/cache")}
+  let(:registry) { described_class.new(upstream, cache: cache) }
+  let(:id) { rand(999) }
+  let(:schema) do
+    {
+      type: "record",
+      name: "person",
+      fields: [{ name: "name", type: "string" }]
+    }.to_json
+  end
+
+  let(:city_id) { rand(999) }
+  let(:city_schema) do
+    {
+      type: "record",
+      name: "city",
+      fields: [{ name: "name", type: "string" }]
+    }.to_json
+  end
+
+  before do
+    FileUtils.mkdir_p("spec/cache")
+  end
+
+  describe "#fetch" do
+    let(:cache_before) do
+      {
+        "#{id}" => "#{schema}"
+      }
+    end
+    let(:cache_after) do
+      {
+        "#{id}" => "#{schema}",
+        "#{city_id}" => "#{city_schema}"
+      }
+    end
+
+    # setup the disk cache to avoid performing the upstream fetch
+    before do
+      store_cache("schemas_by_id.json", cache_before)
+    end
+
+    it "uses preloaded disk cache" do
+      # multiple calls return same result, with zero upstream calls
+      allow(upstream).to receive(:fetch).with(id).and_return(schema)
+      expect(registry.fetch(id)).to eq(schema)
+      expect(registry.fetch(id)).to eq(schema)
+      expect(upstream).to have_received(:fetch).exactly(0).times
+      expect(load_cache("schemas_by_id.json")).to eq cache_before
+    end
+
+    it "writes thru to disk cache" do
+      # multiple calls return same result, with only one upstream call
+      allow(upstream).to receive(:fetch).with(city_id).and_return(city_schema)
+      expect(registry.fetch(city_id)).to eq(city_schema)
+      expect(registry.fetch(city_id)).to eq(city_schema)
+      expect(upstream).to have_received(:fetch).exactly(1).times
+      expect(load_cache("schemas_by_id.json")).to eq cache_after
+    end
+  end
+
+  describe "#register" do
+    let(:subject_name) { "a_subject" }
+    let(:cache_before) do
+      {
+        "#{subject_name}#{schema}" => id
+      }
+    end
+
+    let(:city_name) { "a_city" }
+    let(:cache_after) do 
+      {
+        "#{subject_name}#{schema}" => id,
+        "#{city_name}#{city_schema}" => city_id
+      }
+    end
+
+    # setup the disk cache to avoid performing the upstream register
+    before do
+      store_cache("ids_by_schema.json", cache_before)
+    end
+
+    it "uses preloaded disk cache" do
+      # multiple calls return same result, with zero upstream calls
+      allow(upstream).to receive(:register).with(subject_name, schema).and_return(id)
+      expect(registry.register(subject_name, schema)).to eq(id) 
+      expect(registry.register(subject_name, schema)).to eq(id)
+      expect(upstream).to have_received(:register).exactly(0).times
+      expect(load_cache("ids_by_schema.json")).to eq cache_before
+    end
+
+    it "writes thru to disk cache" do
+      # multiple calls return same result, with only one upstream call
+      allow(upstream).to receive(:register).with(city_name, city_schema).and_return(city_id)
+      expect(registry.register(city_name, city_schema)).to eq(city_id)
+      expect(registry.register(city_name, city_schema)).to eq(city_id)
+      expect(upstream).to have_received(:register).exactly(1).times
+      expect(load_cache("ids_by_schema.json")).to eq cache_after
+    end
+  end
+
+  it_behaves_like "a confluent schema registry client" do
+    let(:upstream) { AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger) }
+    let(:registry) { described_class.new(upstream) }
+  end
+end

data/spec/messaging_spec.rb

@@ -15,18 +15,8 @@ describe AvroTurf::Messaging do
   }
 
   let(:message) { { "full_name" => "John Doe" } }
-
-  before do
-    FileUtils.mkdir_p("spec/schemas")
-  end
-
-  before do
-    stub_request(:any, /^#{registry_url}/).to_rack(FakeConfluentSchemaRegistryServer)
-    FakeConfluentSchemaRegistryServer.clear
-  end
-
-  before do
-    define_schema "person.avsc", <<-AVSC
+  let(:schema_json) do
+    <<-AVSC
       {
         "name": "person",
         "type": "record",
@@ -40,7 +30,20 @@ describe AvroTurf::Messaging do
     AVSC
   end
 
-  shared_examples_for "encoding and decoding" do
+  before do
+    FileUtils.mkdir_p("spec/schemas")
+  end
+
+  before do
+    stub_request(:any, /^#{registry_url}/).to_rack(FakeConfluentSchemaRegistryServer)
+    FakeConfluentSchemaRegistryServer.clear
+  end
+
+  before do
+    define_schema "person.avsc", schema_json
+  end
+
+  shared_examples_for "encoding and decoding with the schema from schema store" do
     it "encodes and decodes messages" do
       data = avro.encode(message, schema_name: "person")
       expect(avro.decode(data)).to eq message
@@ -60,7 +63,39 @@ describe AvroTurf::Messaging do
     end
   end
 
-  it_behaves_like "encoding and decoding"
+  shared_examples_for 'encoding and decoding with the schema from registry' do
+    before do
+      registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger)
+      registry.register('person', Avro::Schema.parse(schema_json))
+      registry.register('people', Avro::Schema.parse(schema_json))
+    end
+
+    it 'encodes and decodes messages' do
+      data = avro.encode(message, subject: 'person', version: 1)
+      expect(avro.decode(data)).to eq message
+    end
+
+    it "allows specifying a reader's schema by subject and version" do
+      data = avro.encode(message, subject: 'person', version: 1)
+      expect(avro.decode(data, schema_name: 'person')).to eq message
+    end
+
+    it 'raises AvroTurf::SchemaNotFoundError when the schema does not exist on registry' do
+      expect { avro.encode(message, subject: 'missing', version: 1) }.to raise_error(AvroTurf::SchemaNotFoundError)
+    end
+
+    it 'caches parsed schemas for decoding' do
+      data = avro.encode(message, subject: 'person', version: 1)
+      avro.decode(data)
+      allow(Avro::Schema).to receive(:parse).and_call_original
+      expect(avro.decode(data)).to eq message
+      expect(Avro::Schema).not_to have_received(:parse)
+    end
+  end
+
+  it_behaves_like "encoding and decoding with the schema from schema store"
+
+  it_behaves_like 'encoding and decoding with the schema from registry'
 
   context "with a provided registry" do
     let(:registry) { AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger) }
@@ -73,7 +108,9 @@ describe AvroTurf::Messaging do
       )
     end
 
-    it_behaves_like "encoding and decoding"
+    it_behaves_like "encoding and decoding with the schema from schema store"
+
+    it_behaves_like 'encoding and decoding with the schema from registry'
 
     it "uses the provided registry" do
       allow(registry).to receive(:register).and_call_original
@@ -101,7 +138,7 @@ describe AvroTurf::Messaging do
       )
     end
 
-    it_behaves_like "encoding and decoding"
+    it_behaves_like "encoding and decoding with the schema from schema store"
 
     it "uses the provided schema store" do
       allow(schema_store).to receive(:find).and_call_original

data/spec/schema_store_spec.rb

@@ -197,6 +197,42 @@ describe AvroTurf::SchemaStore do
       schema = store.find("person")
       expect(schema.fullname).to eq "person"
     end
+
+    it "is thread safe" do
+      define_schema "address.avsc", <<-AVSC
+        {
+          "type": "record",
+          "name": "address",
+          "fields": []
+        }
+      AVSC
+
+      # Set a Thread breakpoint right in the core place of race condition
+      expect(Avro::Name)
+        .to receive(:add_name)
+        .and_wrap_original { |m, *args|
+          Thread.stop
+          m.call(*args)
+        }
+
+      # Run two concurring threads which both will trigger the same schema loading
+      threads = 2.times.map { Thread.new { store.find("address") } }
+      # Wait for the moment when both threads will reach the breakpoint
+      sleep 0.001 until threads.all?(&:stop?)
+
+      expect {
+        # Resume the threads evaluation, one after one
+        threads.each do |thread|
+          next unless thread.status == 'sleep'
+
+          thread.run
+          sleep 0.001 until thread.stop?
+        end
+
+        # Ensure that threads are finished
+        threads.each(&:join)
+      }.to_not raise_error
+    end
   end
 
   describe "#load_schemas!" do

data/spec/spec_helper.rb

@@ -12,6 +12,14 @@ module Helpers
       f.write(content)
     end
   end
+
+  def store_cache(path, hash)
+    File.write(File.join("spec/cache", path), JSON.generate(hash))
+  end
+
+  def load_cache(path)
+    JSON.parse(File.read(File.join("spec/cache", path)))
+  end
 end
 
 RSpec.configure do |config|

data/spec/support/confluent_schema_registry_context.rb

@@ -88,16 +88,18 @@ shared_examples_for "a confluent schema registry client" do
   end
 
   describe "#subject_version" do
-    before do
-      2.times do |n|
-        registry.register(subject_name,
-                          { type: :record, name: "r#{n}", fields: [] }.to_json)
-      end
+    let!(:schema_id1) do
+      registry.register(subject_name, { type: :record, name: "r0", fields: [] }.to_json)
+    end
+    let!(:schema_id2) do
+      registry.register(subject_name, { type: :record, name: "r1", fields: [] }.to_json)
     end
+
     let(:expected) do
       {
         name: subject_name,
         version: 1,
+        id: schema_id1,
         schema: { type: :record, name: "r0", fields: [] }.to_json
       }.to_json
     end
@@ -112,6 +114,7 @@ shared_examples_for "a confluent schema registry client" do
         {
           name: subject_name,
           version: 2,
+          id: schema_id2,
           schema: { type: :record, name: "r1", fields: [] }.to_json
         }.to_json
       end

data/spec/test/fake_confluent_schema_registry_server_spec.rb

@@ -0,0 +1,40 @@
+require 'rack/test'
+require 'avro_turf/test/fake_confluent_schema_registry_server'
+
+describe FakeConfluentSchemaRegistryServer do
+  include Rack::Test::Methods
+
+  def app; described_class; end
+
+  let(:schema) do
+    {
+      type: "record",
+      name: "person",
+      fields: [
+        { name: "name", type: "string" }
+      ]
+    }.to_json
+  end
+
+  describe 'POST /subjects/:subject/versions' do
+    it 'returns the same schema ID when invoked with same schema and same subject' do
+      post '/subjects/person/versions', { schema: schema }.to_json, 'CONTENT_TYPE' => 'application/vnd.schemaregistry+json'
+
+      expected_id = JSON.parse(last_response.body).fetch('id')
+
+      post '/subjects/person/versions', { schema: schema }.to_json, 'CONTENT_TYPE' => 'application/vnd.schemaregistry+json'
+
+      expect(JSON.parse(last_response.body).fetch('id')).to eq expected_id
+    end
+
+    it 'returns a different schema ID when invoked with same schema and different subject' do
+      post '/subjects/person/versions', { schema: schema }.to_json, 'CONTENT_TYPE' => 'application/vnd.schemaregistry+json'
+
+      original_id = JSON.parse(last_response.body).fetch('id')
+
+      post '/subjects/happy-person/versions', { schema: schema }.to_json, 'CONTENT_TYPE' => 'application/vnd.schemaregistry+json'
+
+      expect(JSON.parse(last_response.body).fetch('id')).not_to eq original_id
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: avro_turf
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: 0.9.0
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-15 00:00:00.000000000 Z
+date: 2019-07-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro
@@ -19,7 +19,7 @@ dependencies:
         version: 1.7.7
     - - "<"
       - !ruby/object:Gem::Version
-        version: '1.9'
+        version: '1.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -29,7 +29,7 @@ dependencies:
         version: 1.7.7
     - - "<"
       - !ruby/object:Gem::Version
-        version: '1.9'
+        version: '1.10'
 - !ruby/object:Gem::Dependency
   name: excon
   requirement: !ruby/object:Gem::Requirement
@@ -142,6 +142,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rack-test
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - dasch@zendesk.com
@@ -149,6 +163,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".circleci/config.yml"
 - ".gitignore"
 - ".rspec"
 - CHANGELOG.md
@@ -157,7 +172,6 @@ files:
 - README.md
 - Rakefile
 - avro_turf.gemspec
-- circle.yml
 - lib/avro_turf.rb
 - lib/avro_turf/cached_confluent_schema_registry.rb
 - lib/avro_turf/cached_schema_registry.rb
@@ -173,6 +187,8 @@ files:
 - lib/avro_turf/core_ext/symbol.rb
 - lib/avro_turf/core_ext/time.rb
 - lib/avro_turf/core_ext/true_class.rb
+- lib/avro_turf/disk_cache.rb
+- lib/avro_turf/in_memory_cache.rb
 - lib/avro_turf/messaging.rb
 - lib/avro_turf/mutable_schema_store.rb
 - lib/avro_turf/schema_registry.rb
@@ -198,11 +214,13 @@ files:
 - spec/core_ext/symbol_spec.rb
 - spec/core_ext/time_spec.rb
 - spec/core_ext/true_class_spec.rb
+- spec/disk_cached_confluent_schema_registry_spec.rb
 - spec/messaging_spec.rb
 - spec/schema_store_spec.rb
 - spec/schema_to_avro_patch_spec.rb
 - spec/spec_helper.rb
 - spec/support/confluent_schema_registry_context.rb
+- spec/test/fake_confluent_schema_registry_server_spec.rb
 homepage: https://github.com/dasch/avro_turf
 licenses:
 - MIT
@@ -230,8 +248,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: A library that makes it easier to use the Avro serialization format from
@@ -250,8 +267,10 @@ test_files:
 - spec/core_ext/symbol_spec.rb
 - spec/core_ext/time_spec.rb
 - spec/core_ext/true_class_spec.rb
+- spec/disk_cached_confluent_schema_registry_spec.rb
 - spec/messaging_spec.rb
 - spec/schema_store_spec.rb
 - spec/schema_to_avro_patch_spec.rb
 - spec/spec_helper.rb
 - spec/support/confluent_schema_registry_context.rb
+- spec/test/fake_confluent_schema_registry_server_spec.rb

data/circle.yml

@@ -1,4 +0,0 @@
-machine:
-  ruby:
-    version: 2.2.0
-    version: 2.0.0