avro_turf 0.11.0 → 1.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1e2ee4d9598bcaa2ec5738a1130fae0b19be7b5e7250f27540313b7471f97e23
-  data.tar.gz: 0b441cb30a153958c2ea283300a1a05c26218e2a97cb807fce36f8ad9d0240da
+  metadata.gz: a4e9638703b961c348d06adcfa3e34ac2bd908e68f9cf7762a550ae4ef453b8a
+  data.tar.gz: 6d53b1f9b5e4b9e5a3aaeb13207bf350e2b56c8db138cb499643180a94a684b7
 SHA512:
-  metadata.gz: 12779eac5c325752cfa1be34da94ef5f332490cda4ff0aef29529a00557008cecf39592396a3f3525a2a12cb67a46744800781e0da69d4bf02511f5a2284e5e7
-  data.tar.gz: a2e4c84fb338d62296aefb8ae8c206c262d756475ec11bd9cead17bfe6015ea069ff36a891df3381385fa650cb4b5ea1584855fccc5294f5910ab34563ad973a
+  metadata.gz: 7045f852f20d3ddeca3256724c918ed255722a1556f662d159b805921670d0a2af3e574e3c967bee7622d5d3e05f5348d6023b250006c3651cb7b1f9cec059bf
+  data.tar.gz: 50ca19fb058246ba19b28472e03bedcfbc09ddcd336d46deb29dec4b406d81a65c2b0f6ca84ec36d5227da035906976af133cf757cbfc2f20abda116c8e17d8c

data/.github/workflows/ruby.yml

@@ -1,11 +1,15 @@
 name: Ruby
 
-on: [push]
+on: [push, pull_request]
 
 jobs:
   build:
 
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version: [1.8.x, 1.9.x, 2.0.x, 2.1.x, 2.2.x,
+          2.3.x, 2.4.x, 2.5.x, 2.6.x, 2.7.x, 3.0.x]
 
     steps:
     - uses: actions/checkout@v1

data/.github/workflows/stale.yml

@@ -0,0 +1,19 @@
+name: Mark stale issues and pull requests
+
+on:
+  schedule:
+  - cron: "0 0 * * *"
+
+jobs:
+  stale:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/stale@v1
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        stale-issue-message: 'Stale issue message'
+        stale-pr-message: 'Stale pull request message'
+        stale-issue-label: 'no-issue-activity'
+        stale-pr-label: 'no-pr-activity'

data/CHANGELOG.md

@@ -1,7 +1,29 @@
-# avro_turf
+# AvroTurf
 
 ## Unreleased
 
+## v1.3.1
+
+- Prevent CachedConfluentSchemaRegistry from caching the 'latest' version (#140)
+- Fix issue with zero length schema cache file (#138)
+
+## v1.3.0
+
+- Add support for plain user/password auth to ConfluentSchemaRegistry (#120)
+
+## v1.2.0
+
+- Expose `fetch_schema`, `fetch_schema_by_id` and `register_schema` schema in `Messaging` interface (#117, #119)
+- Add ability to validate message before encoding in `Messaging#encode` interface (#116, #118)
+
+## v1.1.0
+
+- Compatibility with Avro v1.10.x.
+
+## v1.0.0
+
+- Stop caching nested sub-schemas (#111)
+
 ## v0.11.0
 
 - Add proxy support (#107)

data/README.md

@@ -16,6 +16,48 @@ These classes have been renamed to `AvroTurf::ConfluentSchemaRegistry`,
 
 The aliases for the original names will be removed in a future release.
 
+## Note about finding nested schemas
+
+As of AvroTurf version 0.12.0, only top-level schemas that have their own .avsc file will be loaded and resolvable by the `AvroTurf::SchemaStore#find` method. This change will likely not affect most users. However, if you use `AvroTurf::SchemaStore#load_schemas!` to pre-cache all your schemas and then rely on `AvroTurf::SchemaStore#find` to access nested schemas that are not defined by their own .avsc files, your code may stop working when you upgrade to v0.12.0.
+
+As an example, if you have a `person` schema (defined in `my/schemas/contacts/person.avsc`) that defines a nested `address` schema like this:
+
+```json
+{
+  "name": "person",
+  "namespace": "contacts",
+  "type": "record",
+  "fields": [
+    {
+      "name": "address",
+      "type": {
+        "name": "address",
+        "type": "record",
+        "fields": [
+          { "name": "addr1", "type": "string" },
+          { "name": "addr2", "type": "string" },
+          { "name": "city", "type": "string" },
+          { "name": "zip", "type": "string" }
+        ]
+      }
+    }
+  ]
+}
+```
+...this will no longer work in v0.12.0:
+```ruby
+store = AvroTurf::SchemaStore.new(path: 'my/schemas')
+store.load_schemas!
+
+# Accessing 'person' is correct and works fine.
+person = store.find('person', 'contacts') # my/schemas/contacts/person.avsc exists
+
+# Trying to access 'address' raises AvroTurf::SchemaNotFoundError
+address = store.find('address', 'contacts') # my/schemas/contacts/address.avsc is not found
+```
+
+For details and context, see [this pull request](https://github.com/dasch/avro_turf/pull/111).
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -136,6 +178,10 @@ data = avro.encode({ "title" => "hello, world" }, subject: 'greeting', version:
 # of the same schema version will be served by the cache.
 data = avro.encode({ "title" => "hello, world" }, schema_id: 2)
 
+# Message can be validated before encoding to get a description of problem through
+# Avro::SchemaValidator::ValidationError exception
+data = avro.encode({ "titl" => "hello, world" }, schema_name: "greeting", validate: true)
+
 # When decoding, the schema will be fetched from the registry and cached. Subsequent
 # instances of the same schema id will be served by the cache.
 avro.decode(data) #=> { "title" => "hello, world" }
@@ -147,6 +193,20 @@ result.message       #=> { "title" => "hello, world" }
 result.schema_id     #=> 3
 result.writer_schema #=> #<Avro::Schema: ...>
 result.reader_schema #=> nil
+
+# You can also work with schema through this interface:
+# Fetch latest schema for subject from registry
+schema, schema_id = avro.fetch_schema(subject: 'greeting')
+# Fetch specific version
+schema, schema_id = avro.fetch_schema(subject: 'greeting', version: 1)
+# Fetch schema by id
+schema, schema_id = avro.fetch_schema_by_id(3)
+# Register schema fetched from store by name
+schema, schema_id = avro.register_schema(schema_name: 'greeting')
+# Specify namespace (same as schema_name: 'somewhere.greeting')
+schema, schema_id = avro.register_schema(schema_name: 'greeting', namespace: 'somewhere')
+# Customize subject under which to register schema
+schema, schema_id = avro.register_schema(schema_name: 'greeting', namespace: 'somewhere', subject: 'test')
 ```
 
 ### Confluent Schema Registry Client

data/avro_turf.gemspec

@@ -17,8 +17,8 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "avro", ">= 1.7.7", "< 1.10"
-  spec.add_dependency "excon", "~> 0.45"
+  spec.add_dependency "avro", ">= 1.7.7", "< 1.11"
+  spec.add_dependency "excon", "~> 0.71"
 
   spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "rake", "~> 13.0"

data/lib/avro_turf/cached_confluent_schema_registry.rb

@@ -33,6 +33,8 @@ class AvroTurf::CachedConfluentSchemaRegistry
   end
 
   def subject_version(subject, version = 'latest')
+    return @upstream.subject_version(subject, version) if version == 'latest'
+    
     @cache.lookup_by_version(subject, version) ||
       @cache.store_by_version(subject, version, @upstream.subject_version(subject, version))
   end

data/lib/avro_turf/confluent_schema_registry.rb

@@ -7,6 +7,8 @@ class AvroTurf::ConfluentSchemaRegistry
     url,
     logger: Logger.new($stdout),
     proxy: nil,
+    user: nil,
+    password: nil,
     client_cert: nil,
     client_key: nil,
     client_key_pass: nil,
@@ -17,10 +19,12 @@ class AvroTurf::ConfluentSchemaRegistry
     headers = {
       "Content-Type" => CONTENT_TYPE
     }
-    headers[:proxy] = proxy if proxy&.present?
+    headers[:proxy] = proxy unless proxy.nil?
     @connection = Excon.new(
       url,
       headers: headers,
+      user: user,
+      password: password,
       client_cert: client_cert,
       client_key: client_key,
       client_key_pass: client_key_pass,

data/lib/avro_turf/disk_cache.rb

@@ -2,15 +2,19 @@
 # Extends the InMemoryCache to provide a write-thru to disk for persistent cache.
 class AvroTurf::DiskCache < AvroTurf::InMemoryCache
 
-  def initialize(disk_path)
+  def initialize(disk_path, logger: Logger.new($stdout))
     super()
 
+    @logger = logger
+
     # load the write-thru cache on startup, if it exists
     @schemas_by_id_path = File.join(disk_path, 'schemas_by_id.json')
-    @schemas_by_id = JSON.parse(File.read(@schemas_by_id_path)) if File.exist?(@schemas_by_id_path)
+    hash = read_from_disk_cache(@schemas_by_id_path)
+    @schemas_by_id = hash if hash
 
     @ids_by_schema_path = File.join(disk_path, 'ids_by_schema.json')
-    @ids_by_schema = JSON.parse(File.read(@ids_by_schema_path)) if File.exist?(@ids_by_schema_path)
+    hash = read_from_disk_cache(@ids_by_schema_path)
+    @ids_by_schema = hash if hash
 
     @schemas_by_subject_version_path = File.join(disk_path, 'schemas_by_subject_version.json')
     @schemas_by_subject_version = {}
@@ -31,12 +35,18 @@ class AvroTurf::DiskCache < AvroTurf::InMemoryCache
     return value
   end
 
-  # override to include write-thru cache after storing result from upstream
+  # override to use a json serializable cache key
+  def lookup_by_schema(subject, schema)
+    key = "#{subject}#{schema}"
+    @ids_by_schema[key]
+  end
+
+  # override to use a json serializable cache key and update the file cache
   def store_by_schema(subject, schema, id)
-    # must return the value from storing the result (i.e. do not return result from file write)
-    value = super
+    key = "#{subject}#{schema}"
+    @ids_by_schema[key] = id
     File.write(@ids_by_schema_path, JSON.pretty_generate(@ids_by_schema))
-    return value
+    id
   end
 
   # checks instance var (in-memory cache) for schema
@@ -49,7 +59,7 @@ class AvroTurf::DiskCache < AvroTurf::InMemoryCache
 
     return schema unless schema.nil?
 
-    hash = JSON.parse(File.read(@schemas_by_subject_version_path)) if File.exist?(@schemas_by_subject_version_path)
+    hash = read_from_disk_cache(@schemas_by_subject_version_path)
     if hash
       @schemas_by_subject_version = hash
       @schemas_by_subject_version[key]
@@ -63,7 +73,7 @@ class AvroTurf::DiskCache < AvroTurf::InMemoryCache
   # update instance var (in memory-cache) to match
   def store_by_version(subject, version, schema)
     key = "#{subject}#{version}"
-    hash = JSON.parse(File.read(@schemas_by_subject_version_path)) if File.exist?(@schemas_by_subject_version_path)
+    hash = read_from_disk_cache(@schemas_by_subject_version_path)
     hash = if hash
              hash[key] = schema
              hash
@@ -77,6 +87,19 @@ class AvroTurf::DiskCache < AvroTurf::InMemoryCache
     @schemas_by_subject_version[key]
   end
 
+  # Parse the file from disk, if it exists and is not zero length
+  private def read_from_disk_cache(path)
+    if File.exist?(path)
+      if File.size(path)!=0
+        return JSON.parse(File.read(path))
+      else
+        # just log a message if skipping zero length file
+        @logger.warn "skipping JSON.parse of zero length file at #{path}"
+      end
+    end
+    return nil
+  end
+
   private def write_to_disk_cache(path, hash)
     File.write(path, JSON.pretty_generate(hash))
   end

data/lib/avro_turf/in_memory_cache.rb

@@ -17,12 +17,12 @@ class AvroTurf::InMemoryCache
   end
 
   def lookup_by_schema(subject, schema)
-    key = subject + schema.to_s
+    key = [subject, schema.to_s]
     @ids_by_schema[key]
   end
 
   def store_by_schema(subject, schema, id)
-    key = subject + schema.to_s
+    key = [subject, schema.to_s]
     @ids_by_schema[key] = id
   end
 

data/lib/avro_turf/messaging.rb

@@ -34,6 +34,8 @@ class AvroTurf
     # namespace         - The String default schema namespace.
     # logger            - The Logger that should be used to log information (optional).
     # proxy             - Forward the request via  proxy (optional).
+    # user              - User for basic auth (optional).
+    # password          - Password for basic auth (optional).
     # client_cert       - Name of file containing client certificate (optional).
     # client_key        - Name of file containing client private key to go with client_cert (optional).
     # client_key_pass   - Password to go with client_key (optional).
@@ -47,6 +49,8 @@ class AvroTurf
       namespace: nil,
       logger: nil,
       proxy: nil,
+      user: nil,
+      password: nil,
       client_cert: nil,
       client_key: nil,
       client_key_pass: nil,
@@ -61,6 +65,8 @@ class AvroTurf
           registry_url,
           logger: @logger,
           proxy: proxy,
+          user: user,
+          password: password,
           client_cert: client_cert,
           client_key: client_key,
           client_key_pass: client_key_pass,
@@ -84,19 +90,26 @@ class AvroTurf
     #               the data. Must match the schema used when encoding (optional).
     # schema_id   - The integer id of the schema that should be used to encode
     #               the data.
+    # validate    - The boolean for performing complete message validation before
+    #               encoding it, Avro::SchemaValidator::ValidationError with
+    #               a descriptive message will be raised in case of invalid message.
     #
     # Returns the encoded data as a String.
-    def encode(message, schema_name: nil, namespace: @namespace, subject: nil, version: nil, schema_id: nil)
-      schema_id, schema = if schema_id
+    def encode(message, schema_name: nil, namespace: @namespace, subject: nil, version: nil, schema_id: nil, validate: false)
+      schema, schema_id = if schema_id
         fetch_schema_by_id(schema_id)
       elsif subject && version
-        fetch_schema(subject, version)
+        fetch_schema(subject: subject, version: version)
       elsif schema_name
-        register_schema(subject, schema_name, namespace)
+        register_schema(subject: subject, schema_name: schema_name, namespace: namespace)
       else
         raise ArgumentError.new('Neither schema_name nor schema_id nor subject + version provided to determine the schema.')
       end
 
+      if validate
+        Avro::SchemaValidator.validate!(schema, message, recursive: true, encoded: false, fail_on_extra_fields: true)
+      end
+
       stream = StringIO.new
       writer = Avro::IO::DatumWriter.new(schema)
       encoder = Avro::IO::BinaryEncoder.new(stream)
@@ -111,7 +124,7 @@ class AvroTurf
       writer.write(message, encoder)
 
       stream.string
-    rescue Excon::Error::NotFound
+    rescue Excon::Errors::NotFound
       if schema_id
         raise SchemaNotFoundError.new("Schema with id: #{schema_id} is not found on registry")
       else
@@ -169,31 +182,31 @@ class AvroTurf
       raise SchemaNotFoundError.new("Schema with id: #{schema_id} is not found on registry")
     end
 
-    private
-
     # Providing subject and version to determine the schema,
     # which skips the auto registeration of schema on the schema registry.
     # Fetch the schema from registry with the provided subject name and version.
-    def fetch_schema(subject, version)
+    def fetch_schema(subject:, version: 'latest')
       schema_data = @registry.subject_version(subject, version)
       schema_id = schema_data.fetch('id')
       schema = Avro::Schema.parse(schema_data.fetch('schema'))
-      [schema_id, schema]
+      [schema, schema_id]
     end
 
     # Fetch the schema from registry with the provided schema_id.
     def fetch_schema_by_id(schema_id)
-      schema_json = @registry.fetch(schema_id)
-      schema = Avro::Schema.parse(schema_json)
-      [schema_id, schema]
+      schema = @schemas_by_id.fetch(schema_id) do
+        schema_json = @registry.fetch(schema_id)
+        Avro::Schema.parse(schema_json)
+      end
+      [schema, schema_id]
     end
 
     # Schemas are registered under the full name of the top level Avro record
     # type, or `subject` if it's provided.
-    def register_schema(subject, schema_name, namespace)
+    def register_schema(schema_name:, subject: nil, namespace: nil)
       schema = @schema_store.find(schema_name, namespace)
       schema_id = @registry.register(subject || schema.fullname, schema)
-      [schema_id, schema]
+      [schema, schema_id]
     end
   end
 end

data/lib/avro_turf/schema_store.rb

@@ -22,7 +22,7 @@ class AvroTurf::SchemaStore
       # Still need to check is the schema already loaded
       return @schemas[fullname] if @schemas.key?(fullname)
 
-      load_schema!(fullname, namespace)
+      load_schema!(fullname)
     end
   end
 
@@ -42,34 +42,57 @@ class AvroTurf::SchemaStore
     end
   end
 
-  private
+  protected
 
   # Loads single schema
   # Such method is not thread-safe, do not call it of from mutex synchronization routine
-  def load_schema!(fullname, namespace = nil)
-    *namespace, schema_name = fullname.split(".")
-    schema_path = File.join(@path, *namespace, schema_name + ".avsc")
+  def load_schema!(fullname, local_schemas_cache = {})
+    schema_path = build_schema_path(fullname)
     schema_json = JSON.parse(File.read(schema_path))
-    schema = Avro::Schema.real_parse(schema_json, @schemas)
 
+    schema = Avro::Schema.real_parse(schema_json, local_schemas_cache)
+
+    # Don't cache the parsed schema until after its fullname is validated
     if schema.respond_to?(:fullname) && schema.fullname != fullname
       raise AvroTurf::SchemaError, "expected schema `#{schema_path}' to define type `#{fullname}'"
     end
 
+    # Cache only this new top-level schema by its fullname. It's critical
+    # not to make every sub-schema resolvable at the top level here because
+    # multiple different avsc files may define the same sub-schema, and
+    # if we share the @schemas cache across all parsing contexts, the Avro
+    # gem will raise an Avro::SchemaParseError when parsing another avsc
+    # file that contains a subschema with the same fullname as one
+    # encountered previously in a different file:
+    # <Avro::SchemaParseError: The name "foo.bar" is already in use.>
+    # Essentially, the only schemas that should be resolvable in @schemas
+    # are those that have their own .avsc files on disk.
+    @schemas[fullname] = schema
+
     schema
   rescue ::Avro::SchemaParseError => e
     # This is a hack in order to figure out exactly which type was missing. The
     # Avro gem ought to provide this data directly.
     if e.to_s =~ /"([\w\.]+)" is not a schema we know about/
-      load_schema!($1)
+      # Try to first resolve a referenced schema from disk.
+      # If this is successful, the Avro gem will have mutated the
+      # local_schemas_cache, adding all the new schemas it found.
+      load_schema!($1, local_schemas_cache)
 
-      # Re-resolve the original schema now that the dependency has been resolved.
-      @schemas.delete(fullname)
-      load_schema!(fullname)
+      # Attempt to re-parse the original schema now that the dependency
+      # has been resolved and use the now-updated local_schemas_cache to
+      # pick up where we left off.
+      local_schemas_cache.delete(fullname)
+      load_schema!(fullname, local_schemas_cache)
     else
       raise
     end
   rescue Errno::ENOENT, Errno::ENAMETOOLONG
     raise AvroTurf::SchemaNotFoundError, "could not find Avro schema at `#{schema_path}'"
   end
+
+  def build_schema_path(fullname)
+    *namespace, schema_name = fullname.split(".")
+    schema_path = File.join(@path, *namespace, schema_name + ".avsc")
+  end
 end

data/lib/avro_turf/version.rb

@@ -1,3 +1,3 @@
 class AvroTurf
-  VERSION = "0.11.0"
+  VERSION = "1.3.1"
 end

data/spec/confluent_schema_registry_spec.rb

@@ -3,6 +3,8 @@ require 'avro_turf/confluent_schema_registry'
 require 'avro_turf/test/fake_confluent_schema_registry_server'
 
 describe AvroTurf::ConfluentSchemaRegistry do
+  let(:user) { "abc" }
+  let(:password) { "xxyyzz" }
   let(:client_cert) { "test client cert" }
   let(:client_key) { "test client key" }
   let(:client_key_pass) { "test client key password" }
@@ -18,4 +20,14 @@ describe AvroTurf::ConfluentSchemaRegistry do
       )
     }
   end
+
+  it_behaves_like "a confluent schema registry client" do
+    let(:registry) {
+      described_class.new(
+        registry_url,
+        user: user,
+        password: password,
+      )
+    }
+  end
 end

data/spec/disk_cached_confluent_schema_registry_spec.rb

@@ -4,7 +4,8 @@ require 'avro_turf/test/fake_confluent_schema_registry_server'
 
 describe AvroTurf::CachedConfluentSchemaRegistry do
   let(:upstream) { instance_double(AvroTurf::ConfluentSchemaRegistry) }
-  let(:cache)    { AvroTurf::DiskCache.new("spec/cache")}
+  let(:logger_io) { StringIO.new }
+  let(:cache)    { AvroTurf::DiskCache.new("spec/cache", logger: Logger.new(logger_io))}
   let(:registry) { described_class.new(upstream, cache: cache) }
   let(:id) { rand(999) }
   let(:schema) do
@@ -80,6 +81,40 @@ describe AvroTurf::CachedConfluentSchemaRegistry do
     end
   end
 
+  describe "#fetch (zero length cache file)" do
+    let(:cache_after) do
+      {
+        "#{id}" => "#{schema}"
+      }
+    end
+
+    before do
+      # setup the disk cache with a zero length file
+      File.write(File.join("spec/cache", "schemas_by_id.json"), '')
+    end
+
+    it "skips zero length disk cache" do
+      # multiple calls return same result, with only one upstream call
+      allow(upstream).to receive(:fetch).with(id).and_return(schema)
+      expect(registry.fetch(id)).to eq(schema)
+      expect(registry.fetch(id)).to eq(schema)
+      expect(upstream).to have_received(:fetch).exactly(1).times
+      expect(load_cache("schemas_by_id.json")).to eq cache_after
+      expect(logger_io.string).to include("zero length file at spec/cache/schemas_by_id.json")
+    end
+  end
+
+  describe "#fetch (corrupt cache file)" do
+    before do
+      # setup the disk cache with a corrupt file (i.e. not json)
+      File.write(File.join("spec/cache", "schemas_by_id.json"), 'NOTJSON')
+    end
+
+    it "raises error on corrupt cache file" do
+      expect{registry.fetch(id)}.to raise_error(JSON::ParserError, /unexpected token/)
+    end
+  end
+
   describe "#register" do
     let(:subject_name) { "a_subject" }
     let(:cache_before) do
@@ -120,6 +155,41 @@ describe AvroTurf::CachedConfluentSchemaRegistry do
     end
   end
 
+  describe "#register (zero length cache file)" do
+    let(:subject_name) { "a_subject" }
+    let(:cache_after) do
+      {
+        "#{subject_name}#{schema}" => id
+      }
+    end
+
+    before do
+      # setup the disk cache with a zero length file
+      File.write(File.join("spec/cache", "ids_by_schema.json"), '')
+    end
+
+    it "skips zero length disk cache" do
+      # multiple calls return same result, with only one upstream call
+      allow(upstream).to receive(:register).with(subject_name, schema).and_return(id)
+      expect(registry.register(subject_name, schema)).to eq(id)
+      expect(registry.register(subject_name, schema)).to eq(id)
+      expect(upstream).to have_received(:register).exactly(1).times
+      expect(load_cache("ids_by_schema.json")).to eq cache_after
+      expect(logger_io.string).to include("zero length file at spec/cache/ids_by_schema.json")
+    end
+  end
+
+  describe "#register (corrupt cache file)" do
+    before do
+      # setup the disk cache with a corrupt file (i.e. not json)
+      File.write(File.join("spec/cache", "ids_by_schema.json"), 'NOTJSON')
+    end
+
+    it "raises error on corrupt cache file" do
+      expect{registry.register(subject_name, schema)}.to raise_error(JSON::ParserError, /unexpected token/)
+    end
+  end
+
   describe "#subject_version" do
     it "writes thru to disk cache" do
       # multiple calls return same result, with zero upstream calls

data/spec/messaging_spec.rb

@@ -297,4 +297,106 @@ describe AvroTurf::Messaging do
       end
     end
   end
+
+  context "validating" do
+    subject(:encode){ avro.encode(message, schema_name: "person", validate: true) }
+
+    context "for correct message" do
+      it { expect { encode }.not_to raise_error }
+    end
+
+    context "when message has wrong type" do
+      let(:message) { { "full_name" => 123 } }
+
+      it { expect { encode }.to raise_error(Avro::SchemaValidator::ValidationError, /\.full_name expected type string, got int/) }
+    end
+
+    context "when message contains extra fields (typo in key)" do
+      let(:message) { { "fulll_name" => "John Doe" } }
+
+      it { expect { encode }.to raise_error(Avro::SchemaValidator::ValidationError, /extra field 'fulll_name'/) }
+    end
+  end
+
+  context 'fetching and registering schema' do
+    let(:schema_store) { AvroTurf::SchemaStore.new(path: "spec/schemas") }
+
+    let(:registry) { AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger) }
+
+    let(:avro) do
+      AvroTurf::Messaging.new(
+        registry: registry,
+        schema_store: schema_store,
+        logger: logger
+      )
+    end
+
+    let(:schema_id) { 234 }
+
+    context 'using fetch_schema' do
+      subject { avro.fetch_schema(subject: subj, version: version) }
+
+      let(:subj) { 'subject' }
+
+      let(:version) { 'version' }
+
+      let(:response) { {'id' => schema_id, 'schema' => schema_json} }
+
+      before do
+        allow(registry).to receive(:subject_version).with(subj, version).and_return(response)
+      end
+
+      it 'gets schema from registry' do
+        expect(subject).to eq([schema, schema_id])
+      end
+    end
+
+    context 'using fetch_schema_by_id' do
+      subject { avro.fetch_schema_by_id(schema_id) }
+
+      before do
+        allow(registry).to receive(:fetch).with(schema_id).and_return(schema_json)
+      end
+
+      it 'gets schema from registry' do
+        expect(subject).to eq([schema, schema_id])
+      end
+    end
+
+    context 'using register_schema' do
+      let(:schema_name) { 'schema_name' }
+
+      let(:namespace) { 'namespace' }
+
+      before do
+        allow(schema_store).to receive(:find).with(schema_name, namespace).and_return(schema)
+      end
+
+      context 'when subject is not set' do
+        subject { avro.register_schema(schema_name: schema_name, namespace: namespace) }
+
+        before do
+          allow(registry).to receive(:register).with(schema.fullname, schema).and_return(schema_id)
+        end
+
+        it 'registers schema in registry' do
+          expect(subject).to eq([schema, schema_id])
+        end
+      end
+
+      context 'when subject is set' do
+        subject { avro.register_schema(schema_name: schema_name, namespace: namespace, subject: subj) }
+
+        let(:subj) { 'subject' }
+
+        before do
+          allow(registry).to receive(:register).with(subj, schema).and_return(schema_id)
+        end
+
+        it 'registers schema in registry' do
+          expect(subject).to eq([schema, schema_id])
+        end
+      end
+    end
+  end
 end

data/spec/schema_store_spec.rb

@@ -198,6 +198,104 @@ describe AvroTurf::SchemaStore do
       expect(schema.fullname).to eq "person"
     end
 
+    # This test would fail under avro_turf <= v0.11.0
+    it "does NOT cache *nested* schemas in memory" do
+      FileUtils.mkdir_p("spec/schemas/test")
+
+      define_schema "test/person.avsc", <<-AVSC
+        {
+          "name": "person",
+          "namespace": "test",
+          "type": "record",
+          "fields": [
+            {
+              "name": "address",
+              "type": {
+                "name": "address",
+                "type": "record",
+                "fields": [
+                  { "name": "addr1", "type": "string" },
+                  { "name": "addr2", "type": "string" },
+                  { "name": "city", "type": "string" },
+                  { "name": "zip", "type": "string" }
+                ]
+              }
+            }
+          ]
+        }
+      AVSC
+
+      schema = store.find('person', 'test')
+      expect(schema.fullname).to eq "test.person"
+
+      expect { store.find('address', 'test') }.
+        to raise_error(AvroTurf::SchemaNotFoundError)
+    end
+
+    # This test would fail under avro_turf <= v0.11.0
+    it "allows two different avsc files to define nested sub-schemas with the same fullname" do
+      FileUtils.mkdir_p("spec/schemas/test")
+
+      define_schema "test/person.avsc", <<-AVSC
+        {
+          "name": "person",
+          "namespace": "test",
+          "type": "record",
+          "fields": [
+            {
+              "name": "location",
+              "type": {
+                "name": "location",
+                "type": "record",
+                "fields": [
+                  { "name": "city", "type": "string" },
+                  { "name": "zipcode", "type": "string" }
+                ]
+              }
+            }
+          ]
+        }
+      AVSC
+
+      define_schema "test/company.avsc", <<-AVSC
+        {
+          "name": "company",
+          "namespace": "test",
+          "type": "record",
+          "fields": [
+            {
+              "name": "headquarters",
+              "type": {
+                "name": "location",
+                "type": "record",
+                "fields": [
+                  { "name": "city", "type": "string" },
+                  { "name": "postcode", "type": "string" }
+                ]
+              }
+            }
+          ]
+        }
+      AVSC
+
+      company = nil
+      person = store.find('person', 'test')
+
+      # This should *NOT* raise the error:
+      # #<Avro::SchemaParseError: The name "test.location" is already in use.>
+      expect { company = store.find('company', 'test') }.not_to raise_error
+
+      person_location_field = person.fields_hash['location']
+      expect(person_location_field.type.name).to eq('location')
+      expect(person_location_field.type.fields_hash).to include('zipcode')
+      expect(person_location_field.type.fields_hash).not_to include('postcode')
+
+      company_headquarters_field = company.fields_hash['headquarters']
+      expect(company_headquarters_field.type.name).to eq('location')
+      expect(company_headquarters_field.type.fields_hash).to include('postcode')
+      expect(company_headquarters_field.type.fields_hash).not_to include('zipcode')
+    end
+
     it "is thread safe" do
       define_schema "address.avsc", <<-AVSC
         {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: avro_turf
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 1.3.1
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-25 00:00:00.000000000 Z
+date: 2021-02-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro
@@ -19,7 +19,7 @@ dependencies:
         version: 1.7.7
     - - "<"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '1.11'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -29,21 +29,21 @@ dependencies:
         version: 1.7.7
     - - "<"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '1.11'
 - !ruby/object:Gem::Dependency
   name: excon
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.45'
+        version: '0.71'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.45'
+        version: '0.71'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -163,8 +163,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".circleci/config.yml"
 - ".github/workflows/ruby.yml"
+- ".github/workflows/stale.yml"
 - ".gitignore"
 - ".rspec"
 - CHANGELOG.md
@@ -249,8 +249,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: A library that makes it easier to use the Avro serialization format from

data/.circleci/config.yml

@@ -1,36 +0,0 @@
-version: 2
-jobs:
-  build:
-    environment:
-      CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
-      CIRCLE_TEST_REPORTS: /tmp/circleci-test-results
-    docker:
-    - image: circleci/ruby:2.6.2
-    steps:
-    - checkout
-    - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS
-    - restore_cache:
-        keys:
-        # This branch if available
-        - v1-dep-{{ .Branch }}-
-        # Default branch if not
-        - v1-dep-master-
-        # Any branch if there are none on the default branch - this should be unnecessary if you have your default branch configured correctly
-        - v1-dep-
-    - run: gem install bundler --no-document
-    - run: 'bundle check --path=vendor/bundle || bundle install --path=vendor/bundle --jobs=4 --retry=3'
-    # Save dependency cache
-    - save_cache:
-        key: v1-dep-{{ .Branch }}-{{ epoch }}
-        paths:
-        - vendor/bundle
-        - ~/.bundle
-    - run: mkdir -p $CIRCLE_TEST_REPORTS/rspec
-    - run:
-        command: bundle exec rspec --color --require spec_helper --format progress
-    - store_test_results:
-        path: /tmp/circleci-test-results
-    - store_artifacts:
-        path: /tmp/circleci-artifacts
-    - store_artifacts:
-        path: /tmp/circleci-test-results