avro_turf 0.9.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 01a2bdf42a996a7a89bab1769672adc11888d973e5c0c70df42d07b63e87e2bf
-  data.tar.gz: c80b96eace3ae4fdb254f046335e7aaf21939cde9da86b671d7450e3d1d2ceb4
+  metadata.gz: 8fc2d29f1112e649cc2cd5ec96f2126a0ea5fa8f46ebee01d908cdceeac8da0d
+  data.tar.gz: e1989abd9d9b0e5db24f360e96e7bce7ce18e5f488cc794d25a2b3fae6102e66
 SHA512:
-  metadata.gz: 978d5323dd68d2a1518dadcaa90c1871b065e184ffd06ca7a9a86ad1402c16a46f557ba5847e0a841a269b38bfdcb0769a66ec961eddd6b40dd4f025c6eb2c9e
-  data.tar.gz: c2abf819298e5c925f60e7d5e6536aced127e1bee0ef4a66893ea25c16f1e8ea9905de9e142e3f7e0a5989901e7708a9ce000cd85384ef2a703d6e77eb27ce7d
+  metadata.gz: f562a66ba746d2c0e4bee8b8f3a0e7fcf65900f261eca444986c8ff6c68231e67aa9c4dc659213721d0d09c65fd9273e12e158938c0773baa48cf86598d6bef9
+  data.tar.gz: 53e3640900a3b64038fe063f5dadab2049417490be1833bb5f17124307abe628af64ea0c68b5bdc2bd4ef2f6e58bbfac287696eaf5ece9a35ddb7c2ab8e5127e

data/.circleci/config.yml

@@ -17,6 +17,7 @@ jobs:
         - v1-dep-master-
         # Any branch if there are none on the default branch - this should be unnecessary if you have your default branch configured correctly
         - v1-dep-
+    - run: gem install bundler --no-document
     - run: 'bundle check --path=vendor/bundle || bundle install --path=vendor/bundle --jobs=4 --retry=3'
     # Save dependency cache
     - save_cache:
@@ -26,7 +27,7 @@ jobs:
         - ~/.bundle
     - run: mkdir -p $CIRCLE_TEST_REPORTS/rspec
     - run:
-        command: bundle exec rspec --color --require spec_helper --format RspecJunitFormatter --out $CIRCLE_TEST_REPORTS/rspec/rspec.xml --format progress spec
+        command: bundle exec rspec --color --require spec_helper --format progress
     - store_test_results:
         path: /tmp/circleci-test-results
     - store_artifacts:

data/.github/workflows/ruby.yml

@@ -0,0 +1,20 @@
+name: Ruby
+
+on: [push, pull_request]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Ruby 2.6
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.6.x
+    - name: Build and test with RSpec
+      run: |
+        gem install bundler
+        bundle install --jobs 4 --retry 3
+        bundle exec rspec

data/.github/workflows/stale.yml

@@ -0,0 +1,19 @@
+name: Mark stale issues and pull requests
+
+on:
+  schedule:
+  - cron: "0 0 * * *"
+
+jobs:
+  stale:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/stale@v1
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        stale-issue-message: 'Stale issue message'
+        stale-pr-message: 'Stale pull request message'
+        stale-issue-label: 'no-issue-activity'
+        stale-pr-label: 'no-pr-activity'

data/CHANGELOG.md

@@ -1,7 +1,30 @@
-# avro_turf
+# AvroTurf
 
 ## Unreleased
 
+## v1.2.0
+
+- Expose `fetch_schema`, `fetch_schema_by_id` and `register_schema` schema in `Messaging` interface (#117, #119)
+- Add ability to validate message before encoding in `Messaging#encode` interface (#116, #118)
+
+## v1.1.0
+
+- Compatibility with Avro v1.10.x.
+
+## v1.0.0
+
+- Stop caching nested sub-schemas (#111)
+
+## v0.11.0
+
+- Add proxy support (#107)
+- Adding support for client certs (#109)
+
+## v0.10.0
+
+- Add more disk caching (#103)
+- Include schema information when decoding (#100, #101, #104)
+
 ## v0.9.0
 
 - Compatibility with Avro v1.9.0 (#94)

data/Gemfile

@@ -2,6 +2,3 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in avro_turf.gemspec
 gemspec
-
-# Used by CircleCI to format RSpec results.
-gem 'rspec_junit_formatter', :git => 'git@github.com:circleci/rspec_junit_formatter.git'

data/README.md

@@ -16,6 +16,48 @@ These classes have been renamed to `AvroTurf::ConfluentSchemaRegistry`,
 
 The aliases for the original names will be removed in a future release.
 
+## Note about finding nested schemas
+
+As of AvroTurf version 0.12.0, only top-level schemas that have their own .avsc file will be loaded and resolvable by the `AvroTurf::SchemaStore#find` method. This change will likely not affect most users. However, if you use `AvroTurf::SchemaStore#load_schemas!` to pre-cache all your schemas and then rely on `AvroTurf::SchemaStore#find` to access nested schemas that are not defined by their own .avsc files, your code may stop working when you upgrade to v0.12.0.
+
+As an example, if you have a `person` schema (defined in `my/schemas/contacts/person.avsc`) that defines a nested `address` schema like this:
+
+```json
+{
+  "name": "person",
+  "namespace": "contacts",
+  "type": "record",
+  "fields": [
+    {
+      "name": "address",
+      "type": {
+        "name": "address",
+        "type": "record",
+        "fields": [
+          { "name": "addr1", "type": "string" },
+          { "name": "addr2", "type": "string" },
+          { "name": "city", "type": "string" },
+          { "name": "zip", "type": "string" }
+        ]
+      }
+    }
+  ]
+}
+```
+...this will no longer work in v0.12.0:
+```ruby
+store = AvroTurf::SchemaStore.new(path: 'my/schemas')
+store.load_schemas!
+
+# Accessing 'person' is correct and works fine.
+person = store.find('person', 'contacts') # my/schemas/contacts/person.avsc exists
+
+# Trying to access 'address' raises AvroTurf::SchemaNotFoundError
+address = store.find('address', 'contacts') # my/schemas/contacts/address.avsc is not found
+```
+
+For details and context, see [this pull request](https://github.com/dasch/avro_turf/pull/111).
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -126,13 +168,45 @@ data = avro.encode({ "title" => "hello, world" }, schema_name: "greeting")
 
 # If you don't want to automatically register new schemas, you can pass explicitly
 # subject and version to specify which schema should be used for encoding.
-# It will fetch that schema from the registry and cache it. Subsequent instances 
+# It will fetch that schema from the registry and cache it. Subsequent instances
 # of the same schema version will be served by the cache.
 data = avro.encode({ "title" => "hello, world" }, subject: 'greeting', version: 1)
 
+# You can also pass explicitly schema_id to specify which schema
+# should be used for encoding.
+# It will fetch that schema from the registry and cache it. Subsequent instances
+# of the same schema version will be served by the cache.
+data = avro.encode({ "title" => "hello, world" }, schema_id: 2)
+
+# Message can be validated before encoding to get a description of problem through
+# Avro::SchemaValidator::ValidationError exception
+data = avro.encode({ "titl" => "hello, world" }, schema_name: "greeting", validate: true)
+
 # When decoding, the schema will be fetched from the registry and cached. Subsequent
 # instances of the same schema id will be served by the cache.
 avro.decode(data) #=> { "title" => "hello, world" }
+
+# If you want to get decoded message as well as the schema used to encode the message,
+# you can use `#decode_message` method.
+result = avro.decode_message(data)
+result.message       #=> { "title" => "hello, world" }
+result.schema_id     #=> 3
+result.writer_schema #=> #<Avro::Schema: ...>
+result.reader_schema #=> nil
+
+# You can also work with schema through this interface:
+# Fetch latest schema for subject from registry
+schema, schema_id = avro.fetch_schema(subject: 'greeting')
+# Fetch specific version
+schema, schema_id = avro.fetch_schema(subject: 'greeting', version: 1)
+# Fetch schema by id
+schema, schema_id = avro.fetch_schema_by_id(3)
+# Register schema fetched from store by name
+schema, schema_id = avro.register_schema(schema_name: 'greeting')
+# Specify namespace (same as schema_name: 'somewhere.greeting')
+schema, schema_id = avro.register_schema(schema_name: 'greeting', namespace: 'somewhere')
+# Customize subject under which to register schema
+schema, schema_id = avro.register_schema(schema_name: 'greeting', namespace: 'somewhere', subject: 'test')
 ```
 
 ### Confluent Schema Registry Client

data/avro_turf.gemspec

@@ -17,13 +17,13 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "avro", ">= 1.7.7", "< 1.10"
-  spec.add_dependency "excon", "~> 0.45"
+  spec.add_dependency "avro", ">= 1.7.7", "< 1.11"
+  spec.add_dependency "excon", "~> 0.71"
 
-  spec.add_development_dependency "bundler", "~> 1.7"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rspec", "~> 3.2.0"
-  spec.add_development_dependency "fakefs", "~> 0.6.7"
+  spec.add_development_dependency "bundler", "~> 2.0"
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "rspec", "~> 3.2"
+  spec.add_development_dependency "fakefs", "~> 0.20.0"
   spec.add_development_dependency "webmock"
   spec.add_development_dependency "sinatra"
   spec.add_development_dependency "json_spec"

data/lib/avro_turf/confluent_schema_registry.rb

@@ -3,11 +3,30 @@ require 'excon'
 class AvroTurf::ConfluentSchemaRegistry
   CONTENT_TYPE = "application/vnd.schemaregistry.v1+json".freeze
 
-  def initialize(url, logger: Logger.new($stdout))
+  def initialize(
+    url,
+    logger: Logger.new($stdout),
+    proxy: nil,
+    client_cert: nil,
+    client_key: nil,
+    client_key_pass: nil,
+    client_cert_data: nil,
+    client_key_data: nil
+  )
     @logger = logger
-    @connection = Excon.new(url, headers: {
-      "Content-Type" => CONTENT_TYPE,
-    })
+    headers = {
+      "Content-Type" => CONTENT_TYPE
+    }
+    headers[:proxy] = proxy if proxy&.present?
+    @connection = Excon.new(
+      url,
+      headers: headers,
+      client_cert: client_cert,
+      client_key: client_key,
+      client_key_pass: client_key_pass,
+      client_cert_data: client_cert_data,
+      client_key_data: client_key_data
+    )
   end
 
   def fetch(id)

data/lib/avro_turf/disk_cache.rb

@@ -11,6 +11,9 @@ class AvroTurf::DiskCache < AvroTurf::InMemoryCache
 
     @ids_by_schema_path = File.join(disk_path, 'ids_by_schema.json')
     @ids_by_schema = JSON.parse(File.read(@ids_by_schema_path)) if File.exist?(@ids_by_schema_path)
+
+    @schemas_by_subject_version_path = File.join(disk_path, 'schemas_by_subject_version.json')
+    @schemas_by_subject_version = {}
   end
 
   # override
@@ -35,4 +38,46 @@ class AvroTurf::DiskCache < AvroTurf::InMemoryCache
     File.write(@ids_by_schema_path, JSON.pretty_generate(@ids_by_schema))
     return value
   end
+
+  # checks instance var (in-memory cache) for schema
+  # checks disk cache if in-memory cache doesn't exists
+  # if file exists but no in-memory cache, read from file and sync in-memory cache
+  # finally, if file doesn't exist return nil
+  def lookup_by_version(subject, version)
+    key = "#{subject}#{version}"
+    schema = @schemas_by_subject_version[key]
+
+    return schema unless schema.nil?
+
+    hash = JSON.parse(File.read(@schemas_by_subject_version_path)) if File.exist?(@schemas_by_subject_version_path)
+    if hash
+      @schemas_by_subject_version = hash
+      @schemas_by_subject_version[key]
+    end
+  end
+
+  # check if file exists and parse json into a hash
+  # if file exists take json and overwite/insert schema at key
+  # if file doesn't exist create new hash
+  # write the new/updated hash to file
+  # update instance var (in memory-cache) to match
+  def store_by_version(subject, version, schema)
+    key = "#{subject}#{version}"
+    hash = JSON.parse(File.read(@schemas_by_subject_version_path)) if File.exist?(@schemas_by_subject_version_path)
+    hash = if hash
+             hash[key] = schema
+             hash
+           else
+             {key => schema}
+           end
+
+    write_to_disk_cache(@schemas_by_subject_version_path, hash)
+
+    @schemas_by_subject_version = hash
+    @schemas_by_subject_version[key]
+  end
+
+  private def write_to_disk_cache(path, hash)
+    File.write(path, JSON.pretty_generate(hash))
+  end
 end

data/lib/avro_turf/messaging.rb

@@ -21,21 +21,53 @@ class AvroTurf
   # 1: https://github.com/confluentinc/schema-registry
   class Messaging
     MAGIC_BYTE = [0].pack("C").freeze
+    DecodedMessage = Struct.new(:schema_id, :writer_schema, :reader_schema, :message)
+    private_constant(:DecodedMessage)
 
     # Instantiate a new Messaging instance with the given configuration.
     #
-    # registry     - A schema registry object that responds to all methods in the
-    #                AvroTurf::ConfluentSchemaRegistry interface.
-    # registry_url - The String URL of the schema registry that should be used.
-    # schema_store - A schema store object that responds to #find(schema_name, namespace).
-    # schemas_path - The String file system path where local schemas are stored.
-    # namespace    - The String default schema namespace.
-    # logger       - The Logger that should be used to log information (optional).
-    def initialize(registry: nil, registry_url: nil, schema_store: nil, schemas_path: nil, namespace: nil, logger: nil)
+    # registry          - A schema registry object that responds to all methods in the
+    #                     AvroTurf::ConfluentSchemaRegistry interface.
+    # registry_url      - The String URL of the schema registry that should be used.
+    # schema_store      - A schema store object that responds to #find(schema_name, namespace).
+    # schemas_path      - The String file system path where local schemas are stored.
+    # namespace         - The String default schema namespace.
+    # logger            - The Logger that should be used to log information (optional).
+    # proxy             - Forward the request via  proxy (optional).
+    # client_cert       - Name of file containing client certificate (optional).
+    # client_key        - Name of file containing client private key to go with client_cert (optional).
+    # client_key_pass   - Password to go with client_key (optional).
+    # client_cert_data  - In-memory client certificate (optional).
+    # client_key_data   - In-memory client private key to go with client_cert_data (optional).
+    def initialize(
+      registry: nil,
+      registry_url: nil,
+      schema_store: nil,
+      schemas_path: nil,
+      namespace: nil,
+      logger: nil,
+      proxy: nil,
+      client_cert: nil,
+      client_key: nil,
+      client_key_pass: nil,
+      client_cert_data: nil,
+      client_key_data: nil
+    )
       @logger = logger || Logger.new($stderr)
       @namespace = namespace
       @schema_store = schema_store || SchemaStore.new(path: schemas_path || DEFAULT_SCHEMAS_PATH)
-      @registry = registry || CachedConfluentSchemaRegistry.new(ConfluentSchemaRegistry.new(registry_url, logger: @logger))
+      @registry = registry || CachedConfluentSchemaRegistry.new(
+        ConfluentSchemaRegistry.new(
+          registry_url,
+          logger: @logger,
+          proxy: proxy,
+          client_cert: client_cert,
+          client_key: client_key,
+          client_key_pass: client_key_pass,
+          client_cert_data: client_cert_data,
+          client_key_data: client_key_data
+        )
+      )
       @schemas_by_id = {}
     end
 
@@ -50,15 +82,26 @@ class AvroTurf
     #               the schema registry (optional).
     # version     - The integer version of the schema that should be used to decode
     #               the data. Must match the schema used when encoding (optional).
+    # schema_id   - The integer id of the schema that should be used to encode
+    #               the data.
+    # validate    - The boolean for performing complete message validation before
+    #               encoding it, Avro::SchemaValidator::ValidationError with
+    #               a descriptive message will be raised in case of invalid message.
     #
     # Returns the encoded data as a String.
-    def encode(message, schema_name: nil, namespace: @namespace, subject: nil, version: nil)
-      schema_id, schema = if subject && version
-        fetch_schema(subject, version)
+    def encode(message, schema_name: nil, namespace: @namespace, subject: nil, version: nil, schema_id: nil, validate: false)
+      schema, schema_id = if schema_id
+        fetch_schema_by_id(schema_id)
+      elsif subject && version
+        fetch_schema(subject: subject, version: version)
       elsif schema_name
-        register_schema(subject, schema_name, namespace)
+        register_schema(subject: subject, schema_name: schema_name, namespace: namespace)
       else
-        raise ArgumentError.new('Neither schema_name nor subject + version provided to determine the schema.')
+        raise ArgumentError.new('Neither schema_name nor schema_id nor subject + version provided to determine the schema.')
+      end
+
+      if validate
+        Avro::SchemaValidator.validate!(schema, message, recursive: true, encoded: false, fail_on_extra_fields: true)
       end
 
       stream = StringIO.new
@@ -76,7 +119,11 @@ class AvroTurf
 
       stream.string
     rescue Excon::Error::NotFound
-      raise SchemaNotFoundError.new("Schema with subject: `#{subject}` version: `#{version}` is not found on registry")
+      if schema_id
+        raise SchemaNotFoundError.new("Schema with id: #{schema_id} is not found on registry")
+      else
+        raise SchemaNotFoundError.new("Schema with subject: `#{subject}` version: `#{version}` is not found on registry")
+      end
     end
 
     # Decodes data into the original message.
@@ -88,6 +135,20 @@ class AvroTurf
     #
     # Returns the decoded message.
     def decode(data, schema_name: nil, namespace: @namespace)
+      decode_message(data, schema_name: schema_name, namespace: namespace).message
+    end
+
+    # Decodes data into the original message.
+    #
+    # data        - A String containing encoded data.
+    # schema_name - The String name of the schema that should be used to decode
+    #               the data. Must match the schema used when encoding (optional).
+    # namespace   - The namespace of the schema (optional).
+    #
+    # Returns Struct with the next attributes:
+    #   schema_id  - The integer id of schema used to encode the message
+    #   message    - The decoded message
+    def decode_message(data, schema_name: nil, namespace: @namespace)
       readers_schema = schema_name && @schema_store.find(schema_name, namespace)
       stream = StringIO.new(data)
       decoder = Avro::IO::BinaryDecoder.new(stream)
@@ -108,29 +169,36 @@ class AvroTurf
       end
 
       reader = Avro::IO::DatumReader.new(writers_schema, readers_schema)
-      reader.read(decoder)
+      message = reader.read(decoder)
+
+      DecodedMessage.new(schema_id, writers_schema, readers_schema, message)
     rescue Excon::Error::NotFound
       raise SchemaNotFoundError.new("Schema with id: #{schema_id} is not found on registry")
     end
 
-    private
-
     # Providing subject and version to determine the schema,
     # which skips the auto registeration of schema on the schema registry.
     # Fetch the schema from registry with the provided subject name and version.
-    def fetch_schema(subject, version)
+    def fetch_schema(subject:, version: 'latest')
       schema_data = @registry.subject_version(subject, version)
       schema_id = schema_data.fetch('id')
       schema = Avro::Schema.parse(schema_data.fetch('schema'))
-      [schema_id, schema]
+      [schema, schema_id]
+    end
+
+    # Fetch the schema from registry with the provided schema_id.
+    def fetch_schema_by_id(schema_id)
+      schema_json = @registry.fetch(schema_id)
+      schema = Avro::Schema.parse(schema_json)
+      [schema, schema_id]
     end
 
     # Schemas are registered under the full name of the top level Avro record
     # type, or `subject` if it's provided.
-    def register_schema(subject, schema_name, namespace)
+    def register_schema(schema_name:, subject: nil, namespace: nil)
       schema = @schema_store.find(schema_name, namespace)
       schema_id = @registry.register(subject || schema.fullname, schema)
-      [schema_id, schema]
+      [schema, schema_id]
     end
   end
 end

data/lib/avro_turf/schema_store.rb

@@ -46,26 +46,45 @@ class AvroTurf::SchemaStore
 
   # Loads single schema
   # Such method is not thread-safe, do not call it of from mutex synchronization routine
-  def load_schema!(fullname, namespace = nil)
+  def load_schema!(fullname, namespace = nil, local_schemas_cache = {})
     *namespace, schema_name = fullname.split(".")
     schema_path = File.join(@path, *namespace, schema_name + ".avsc")
     schema_json = JSON.parse(File.read(schema_path))
-    schema = Avro::Schema.real_parse(schema_json, @schemas)
 
+    schema = Avro::Schema.real_parse(schema_json, local_schemas_cache)
+
+    # Don't cache the parsed schema until after its fullname is validated
     if schema.respond_to?(:fullname) && schema.fullname != fullname
       raise AvroTurf::SchemaError, "expected schema `#{schema_path}' to define type `#{fullname}'"
     end
 
+    # Cache only this new top-level schema by its fullname. It's critical
+    # not to make every sub-schema resolvable at the top level here because
+    # multiple different avsc files may define the same sub-schema, and
+    # if we share the @schemas cache across all parsing contexts, the Avro
+    # gem will raise an Avro::SchemaParseError when parsing another avsc
+    # file that contains a subschema with the same fullname as one
+    # encountered previously in a different file:
+    # <Avro::SchemaParseError: The name "foo.bar" is already in use.>
+    # Essentially, the only schemas that should be resolvable in @schemas
+    # are those that have their own .avsc files on disk.
+    @schemas[fullname] = schema
+
     schema
   rescue ::Avro::SchemaParseError => e
     # This is a hack in order to figure out exactly which type was missing. The
     # Avro gem ought to provide this data directly.
     if e.to_s =~ /"([\w\.]+)" is not a schema we know about/
-      load_schema!($1)
+      # Try to first resolve a referenced schema from disk.
+      # If this is successful, the Avro gem will have mutated the
+      # local_schemas_cache, adding all the new schemas it found.
+      load_schema!($1, nil, local_schemas_cache)
 
-      # Re-resolve the original schema now that the dependency has been resolved.
-      @schemas.delete(fullname)
-      load_schema!(fullname)
+      # Attempt to re-parse the original schema now that the dependency
+      # has been resolved and use the now-updated local_schemas_cache to
+      # pick up where we left off.
+      local_schemas_cache.delete(fullname)
+      load_schema!(fullname, nil, local_schemas_cache)
     else
       raise
     end

data/lib/avro_turf/version.rb

@@ -1,3 +1,3 @@
 class AvroTurf
-  VERSION = "0.9.0"
+  VERSION = "1.2.0"
 end

data/spec/confluent_schema_registry_spec.rb

@@ -3,7 +3,19 @@ require 'avro_turf/confluent_schema_registry'
 require 'avro_turf/test/fake_confluent_schema_registry_server'
 
 describe AvroTurf::ConfluentSchemaRegistry do
+  let(:client_cert) { "test client cert" }
+  let(:client_key) { "test client key" }
+  let(:client_key_pass) { "test client key password" }
+
   it_behaves_like "a confluent schema registry client" do
-    let(:registry) { described_class.new(registry_url, logger: logger) }
+    let(:registry) {
+      described_class.new(
+        registry_url,
+        logger: logger,
+        client_cert: client_cert,
+        client_key: client_key,
+        client_key_pass: client_key_pass
+      )
+    }
   end
 end

data/spec/disk_cached_confluent_schema_registry_spec.rb

@@ -24,6 +24,21 @@ describe AvroTurf::CachedConfluentSchemaRegistry do
     }.to_json
   end
 
+  let(:subject) { 'subject' }
+  let(:version) { rand(999) }
+  let(:subject_version_schema) do
+    {
+      subject: subject,
+      version: version,
+      id: id,
+      schema:  {
+        type: "record",
+        name: "city",
+        fields: { name: "name", type: "string" }
+      }
+    }.to_json
+  end
+
   before do
     FileUtils.mkdir_p("spec/cache")
   end
@@ -105,6 +120,38 @@ describe AvroTurf::CachedConfluentSchemaRegistry do
     end
   end
 
+  describe "#subject_version" do
+    it "writes thru to disk cache" do
+      # multiple calls return same result, with zero upstream calls
+      allow(upstream).to receive(:subject_version).with(subject, version).and_return(subject_version_schema)
+      expect(File).not_to exist("./spec/cache/schemas_by_subject_version.json")
+
+      expect(registry.subject_version(subject, version)).to eq(subject_version_schema)
+
+      json = JSON.parse(File.read("./spec/cache/schemas_by_subject_version.json"))["#{subject}#{version}"]
+      expect(json).to eq(subject_version_schema)
+
+      expect(registry.subject_version(subject, version)).to eq(subject_version_schema)
+      expect(upstream).to have_received(:subject_version).exactly(1).times
+    end
+
+    it "reads from disk cache and populates mem cache" do
+      allow(upstream).to receive(:subject_version).with(subject, version).and_return(subject_version_schema)
+      key = "#{subject}#{version}"
+      hash = {key => subject_version_schema}
+      cache.send(:write_to_disk_cache, "./spec/cache/schemas_by_subject_version.json", hash)
+
+      cached_schema = cache.instance_variable_get(:@schemas_by_subject_version)
+      expect(cached_schema).to eq({})
+
+      expect(registry.subject_version(subject, version)).to eq(subject_version_schema)
+      expect(upstream).to have_received(:subject_version).exactly(0).times
+
+      cached_schema = cache.instance_variable_get(:@schemas_by_subject_version)
+      expect(cached_schema).to eq({key => subject_version_schema})
+    end
+  end
+
   it_behaves_like "a confluent schema registry client" do
     let(:upstream) { AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger) }
     let(:registry) { described_class.new(upstream) }

data/spec/messaging_spec.rb

@@ -4,13 +4,19 @@ require 'avro_turf/test/fake_confluent_schema_registry_server'
 
 describe AvroTurf::Messaging do
   let(:registry_url) { "http://registry.example.com" }
+  let(:client_cert) { "test client cert" }
+  let(:client_key) { "test client key" }
+  let(:client_key_pass) { "test client key password" }
   let(:logger) { Logger.new(StringIO.new) }
 
   let(:avro) {
     AvroTurf::Messaging.new(
       registry_url: registry_url,
       schemas_path: "spec/schemas",
-      logger: logger
+      logger: logger,
+      client_cert: client_cert,
+      client_key: client_key,
+      client_key_pass: client_key_pass
     )
   }
 
@@ -29,6 +35,7 @@ describe AvroTurf::Messaging do
       }
     AVSC
   end
+  let(:schema) { Avro::Schema.parse(schema_json) }
 
   before do
     FileUtils.mkdir_p("spec/schemas")
@@ -66,8 +73,8 @@ describe AvroTurf::Messaging do
   shared_examples_for 'encoding and decoding with the schema from registry' do
     before do
       registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger)
-      registry.register('person', Avro::Schema.parse(schema_json))
-      registry.register('people', Avro::Schema.parse(schema_json))
+      registry.register('person', schema)
+      registry.register('people', schema)
     end
 
     it 'encodes and decodes messages' do
@@ -93,10 +100,37 @@ describe AvroTurf::Messaging do
     end
   end
 
+  shared_examples_for 'encoding and decoding with the schema_id from registry' do
+    before do
+      registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger)
+      registry.register('person', schema)
+      registry.register('people', schema)
+    end
+
+    it 'encodes and decodes messages' do
+      data = avro.encode(message, schema_id: 1)
+      expect(avro.decode(data)).to eq message
+    end
+
+    it 'raises AvroTurf::SchemaNotFoundError when the schema does not exist on registry' do
+      expect { avro.encode(message, schema_id: 5) }.to raise_error(AvroTurf::SchemaNotFoundError)
+    end
+
+    it 'caches parsed schemas for decoding' do
+      data = avro.encode(message, schema_id: 1)
+      avro.decode(data)
+      allow(Avro::Schema).to receive(:parse).and_call_original
+      expect(avro.decode(data)).to eq message
+      expect(Avro::Schema).not_to have_received(:parse)
+    end
+  end
+
   it_behaves_like "encoding and decoding with the schema from schema store"
 
   it_behaves_like 'encoding and decoding with the schema from registry'
 
+  it_behaves_like 'encoding and decoding with the schema_id from registry'
+
   context "with a provided registry" do
     let(:registry) { AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger) }
 
@@ -112,6 +146,8 @@ describe AvroTurf::Messaging do
 
     it_behaves_like 'encoding and decoding with the schema from registry'
 
+    it_behaves_like 'encoding and decoding with the schema_id from registry'
+
     it "uses the provided registry" do
       allow(registry).to receive(:register).and_call_original
       message = { "full_name" => "John Doe" }
@@ -146,4 +182,221 @@ describe AvroTurf::Messaging do
       expect(schema_store).to have_received(:find).with("person", nil)
     end
   end
+
+  describe 'decoding with #decode_message' do
+    shared_examples_for "encoding and decoding with the schema from schema store" do
+      it "encodes and decodes messages" do
+        data = avro.encode(message, schema_name: "person")
+        result = avro.decode_message(data)
+        expect(result.message).to eq message
+        expect(result.schema_id).to eq 0
+        expect(result.writer_schema).to eq schema
+        expect(result.reader_schema).to eq nil
+      end
+
+      it "allows specifying a reader's schema" do
+        data = avro.encode(message, schema_name: "person")
+        result = avro.decode_message(data, schema_name: "person")
+        expect(result.message).to eq message
+        expect(result.writer_schema).to eq schema
+        expect(result.reader_schema).to eq schema
+      end
+
+      it "caches parsed schemas for decoding" do
+        data = avro.encode(message, schema_name: "person")
+        avro.decode_message(data)
+        allow(Avro::Schema).to receive(:parse).and_call_original
+        expect(avro.decode_message(data).message).to eq message
+        expect(Avro::Schema).not_to have_received(:parse)
+      end
+    end
+
+    shared_examples_for 'encoding and decoding with the schema from registry' do
+      before do
+        registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger)
+        registry.register('person', schema)
+        registry.register('people', schema)
+      end
+
+      it 'encodes and decodes messages' do
+        data = avro.encode(message, subject: 'person', version: 1)
+        result = avro.decode_message(data)
+        expect(result.message).to eq message
+        expect(result.schema_id).to eq 0
+      end
+
+      it "allows specifying a reader's schema by subject and version" do
+        data = avro.encode(message, subject: 'person', version: 1)
+        expect(avro.decode_message(data, schema_name: 'person').message).to eq message
+      end
+
+      it 'raises AvroTurf::SchemaNotFoundError when the schema does not exist on registry' do
+        expect { avro.encode(message, subject: 'missing', version: 1) }.to raise_error(AvroTurf::SchemaNotFoundError)
+      end
+
+      it 'caches parsed schemas for decoding' do
+        data = avro.encode(message, subject: 'person', version: 1)
+        avro.decode_message(data)
+        allow(Avro::Schema).to receive(:parse).and_call_original
+        expect(avro.decode_message(data).message).to eq message
+        expect(Avro::Schema).not_to have_received(:parse)
+      end
+    end
+
+    it_behaves_like "encoding and decoding with the schema from schema store"
+
+    it_behaves_like 'encoding and decoding with the schema from registry'
+
+    context "with a provided registry" do
+      let(:registry) { AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger) }
+
+      let(:avro) do
+        AvroTurf::Messaging.new(
+          registry: registry,
+          schemas_path: "spec/schemas",
+          logger: logger
+        )
+      end
+
+      it_behaves_like "encoding and decoding with the schema from schema store"
+
+      it_behaves_like 'encoding and decoding with the schema from registry'
+
+      it "uses the provided registry" do
+        allow(registry).to receive(:register).and_call_original
+        message = { "full_name" => "John Doe" }
+        avro.encode(message, schema_name: "person")
+        expect(registry).to have_received(:register).with("person", anything)
+      end
+
+      it "allows specifying a schema registry subject" do
+        allow(registry).to receive(:register).and_call_original
+        message = { "full_name" => "John Doe" }
+        avro.encode(message, schema_name: "person", subject: "people")
+        expect(registry).to have_received(:register).with("people", anything)
+      end
+    end
+
+    context "with a provided schema store" do
+      let(:schema_store) { AvroTurf::SchemaStore.new(path: "spec/schemas") }
+
+      let(:avro) do
+        AvroTurf::Messaging.new(
+          registry_url: registry_url,
+          schema_store: schema_store,
+          logger: logger
+        )
+      end
+
+      it_behaves_like "encoding and decoding with the schema from schema store"
+
+      it "uses the provided schema store" do
+        allow(schema_store).to receive(:find).and_call_original
+        avro.encode(message, schema_name: "person")
+        expect(schema_store).to have_received(:find).with("person", nil)
+      end
+    end
+  end
+
+  context "validating" do
+    subject(:encode){ avro.encode(message, schema_name: "person", validate: true) }
+
+    context "for correct message" do
+      it { expect { encode }.not_to raise_error }
+    end
+
+    context "when message has wrong type" do
+      let(:message) { { "full_name" => 123 } }
+
+      it { expect { encode }.to raise_error(Avro::SchemaValidator::ValidationError, /\.full_name expected type string, got int/) }
+    end
+
+    context "when message contains extra fields (typo in key)" do
+      let(:message) { { "fulll_name" => "John Doe" } }
+
+      it { expect { encode }.to raise_error(Avro::SchemaValidator::ValidationError, /extra field 'fulll_name'/) }
+    end
+  end
+
+  context 'fetching and registering schema' do
+    let(:schema_store) { AvroTurf::SchemaStore.new(path: "spec/schemas") }
+
+    let(:registry) { AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger) }
+
+    let(:avro) do
+      AvroTurf::Messaging.new(
+        registry: registry,
+        schema_store: schema_store,
+        logger: logger
+      )
+    end
+
+    let(:schema_id) { 234 }
+
+    context 'using fetch_schema' do
+      subject { avro.fetch_schema(subject: subj, version: version) }
+
+      let(:subj) { 'subject' }
+
+      let(:version) { 'version' }
+
+      let(:response) { {'id' => schema_id, 'schema' => schema_json} }
+
+      before do
+        allow(registry).to receive(:subject_version).with(subj, version).and_return(response)
+      end
+
+      it 'gets schema from registry' do
+        expect(subject).to eq([schema, schema_id])
+      end
+    end
+
+    context 'using fetch_schema_by_id' do
+      subject { avro.fetch_schema_by_id(schema_id) }
+
+      before do
+        allow(registry).to receive(:fetch).with(schema_id).and_return(schema_json)
+      end
+
+      it 'gets schema from registry' do
+        expect(subject).to eq([schema, schema_id])
+      end
+    end
+
+    context 'using register_schema' do
+      let(:schema_name) { 'schema_name' }
+
+      let(:namespace) { 'namespace' }
+
+      before do
+        allow(schema_store).to receive(:find).with(schema_name, namespace).and_return(schema)
+      end
+
+      context 'when subject is not set' do
+        subject { avro.register_schema(schema_name: schema_name, namespace: namespace) }
+
+        before do
+          allow(registry).to receive(:register).with(schema.fullname, schema).and_return(schema_id)
+        end
+
+        it 'registers schema in registry' do
+          expect(subject).to eq([schema, schema_id])
+        end
+      end
+
+      context 'when subject is set' do
+        subject { avro.register_schema(schema_name: schema_name, namespace: namespace, subject: subj) }
+
+        let(:subj) { 'subject' }
+
+        before do
+          allow(registry).to receive(:register).with(subj, schema).and_return(schema_id)
+        end
+
+        it 'registers schema in registry' do
+          expect(subject).to eq([schema, schema_id])
+        end
+      end
+    end
+  end
 end

data/spec/schema_store_spec.rb

@@ -198,6 +198,104 @@ describe AvroTurf::SchemaStore do
       expect(schema.fullname).to eq "person"
     end
 
+    # This test would fail under avro_turf <= v0.11.0
+    it "does NOT cache *nested* schemas in memory" do
+      FileUtils.mkdir_p("spec/schemas/test")
+
+      define_schema "test/person.avsc", <<-AVSC
+        {
+          "name": "person",
+          "namespace": "test",
+          "type": "record",
+          "fields": [
+            {
+              "name": "address",
+              "type": {
+                "name": "address",
+                "type": "record",
+                "fields": [
+                  { "name": "addr1", "type": "string" },
+                  { "name": "addr2", "type": "string" },
+                  { "name": "city", "type": "string" },
+                  { "name": "zip", "type": "string" }
+                ]
+              }
+            }
+          ]
+        }
+      AVSC
+
+      schema = store.find('person', 'test')
+      expect(schema.fullname).to eq "test.person"
+
+      expect { store.find('address', 'test') }.
+        to raise_error(AvroTurf::SchemaNotFoundError)
+    end
+
+    # This test would fail under avro_turf <= v0.11.0
+    it "allows two different avsc files to define nested sub-schemas with the same fullname" do
+      FileUtils.mkdir_p("spec/schemas/test")
+
+      define_schema "test/person.avsc", <<-AVSC
+        {
+          "name": "person",
+          "namespace": "test",
+          "type": "record",
+          "fields": [
+            {
+              "name": "location",
+              "type": {
+                "name": "location",
+                "type": "record",
+                "fields": [
+                  { "name": "city", "type": "string" },
+                  { "name": "zipcode", "type": "string" }
+                ]
+              }
+            }
+          ]
+        }
+      AVSC
+
+      define_schema "test/company.avsc", <<-AVSC
+        {
+          "name": "company",
+          "namespace": "test",
+          "type": "record",
+          "fields": [
+            {
+              "name": "headquarters",
+              "type": {
+                "name": "location",
+                "type": "record",
+                "fields": [
+                  { "name": "city", "type": "string" },
+                  { "name": "postcode", "type": "string" }
+                ]
+              }
+            }
+          ]
+        }
+      AVSC
+
+      company = nil
+      person = store.find('person', 'test')
+
+      # This should *NOT* raise the error:
+      # #<Avro::SchemaParseError: The name "test.location" is already in use.>
+      expect { company = store.find('company', 'test') }.not_to raise_error
+
+      person_location_field = person.fields_hash['location']
+      expect(person_location_field.type.name).to eq('location')
+      expect(person_location_field.type.fields_hash).to include('zipcode')
+      expect(person_location_field.type.fields_hash).not_to include('postcode')
+
+      company_headquarters_field = company.fields_hash['headquarters']
+      expect(company_headquarters_field.type.name).to eq('location')
+      expect(company_headquarters_field.type.fields_hash).to include('postcode')
+      expect(company_headquarters_field.type.fields_hash).not_to include('zipcode')
+    end
+
     it "is thread safe" do
       define_schema "address.avsc", <<-AVSC
         {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: avro_turf
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-07-15 00:00:00.000000000 Z
+date: 2020-08-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro
@@ -19,7 +19,7 @@ dependencies:
         version: 1.7.7
     - - "<"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '1.11'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -29,77 +29,77 @@ dependencies:
         version: 1.7.7
     - - "<"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '1.11'
 - !ruby/object:Gem::Dependency
   name: excon
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.45'
+        version: '0.71'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.45'
+        version: '0.71'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.2.0
+        version: '3.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.2.0
+        version: '3.2'
 - !ruby/object:Gem::Dependency
   name: fakefs
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.6.7
+        version: 0.20.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.6.7
+        version: 0.20.0
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -164,6 +164,8 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".circleci/config.yml"
+- ".github/workflows/ruby.yml"
+- ".github/workflows/stale.yml"
 - ".gitignore"
 - ".rspec"
 - CHANGELOG.md
@@ -248,7 +250,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubyforge_project: 
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: A library that makes it easier to use the Avro serialization format from