avro_turf 0.11.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1e2ee4d9598bcaa2ec5738a1130fae0b19be7b5e7250f27540313b7471f97e23
-  data.tar.gz: 0b441cb30a153958c2ea283300a1a05c26218e2a97cb807fce36f8ad9d0240da
+  metadata.gz: 692b0814cd5b8fcaaf55e6d607687714ae9bc496461fe0b838253d5fbeb2d218
+  data.tar.gz: 506a5d9bbe91a9386b1b92eb5c7e0366ad06856615442424725a82d4834cc6a6
 SHA512:
-  metadata.gz: 12779eac5c325752cfa1be34da94ef5f332490cda4ff0aef29529a00557008cecf39592396a3f3525a2a12cb67a46744800781e0da69d4bf02511f5a2284e5e7
-  data.tar.gz: a2e4c84fb338d62296aefb8ae8c206c262d756475ec11bd9cead17bfe6015ea069ff36a891df3381385fa650cb4b5ea1584855fccc5294f5910ab34563ad973a
+  metadata.gz: 0db4e3e78577224cdb07ca4747c759bcdaae620dc03bbcac03194941a3bdf52bc69871cfad88dc14fc8ee6fd798a34d5acab289f8c841a97581290f258213f06
+  data.tar.gz: 672aec874705185e68dce6c8806ccf93b18ec8c3d0f34eda074e8fca6c372d4761d682ff316d3ba3d08c4b69b61a853bdb3c31ff68d4da2f0e37463a407ca656

data/.github/workflows/ruby.yml

@@ -1,6 +1,6 @@
 name: Ruby
 
-on: [push]
+on: [push, pull_request]
 
 jobs:
   build:

data/.github/workflows/stale.yml

@@ -0,0 +1,19 @@
+name: Mark stale issues and pull requests
+
+on:
+  schedule:
+  - cron: "0 0 * * *"
+
+jobs:
+  stale:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/stale@v1
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        stale-issue-message: 'Stale issue message'
+        stale-pr-message: 'Stale pull request message'
+        stale-issue-label: 'no-issue-activity'
+        stale-pr-label: 'no-pr-activity'

data/CHANGELOG.md

@@ -1,7 +1,11 @@
-# avro_turf
+# AvroTurf
 
 ## Unreleased
 
+## v1.0.0
+
+- Stop caching nested sub-schemas (#111)
+
 ## v0.11.0
 
 - Add proxy support (#107)

data/README.md

@@ -16,6 +16,48 @@ These classes have been renamed to `AvroTurf::ConfluentSchemaRegistry`,
 
 The aliases for the original names will be removed in a future release.
 
+## Note about finding nested schemas
+
+As of AvroTurf version 0.12.0, only top-level schemas that have their own .avsc file will be loaded and resolvable by the `AvroTurf::SchemaStore#find` method. This change will likely not affect most users. However, if you use `AvroTurf::SchemaStore#load_schemas!` to pre-cache all your schemas and then rely on `AvroTurf::SchemaStore#find` to access nested schemas that are not defined by their own .avsc files, your code may stop working when you upgrade to v0.12.0.
+
+As an example, if you have a `person` schema (defined in `my/schemas/contacts/person.avsc`) that defines a nested `address` schema like this:
+
+```json
+{
+  "name": "person",
+  "namespace": "contacts",
+  "type": "record",
+  "fields": [
+    {
+      "name": "address",
+      "type": {
+        "name": "address",
+        "type": "record",
+        "fields": [
+          { "name": "addr1", "type": "string" },
+          { "name": "addr2", "type": "string" },
+          { "name": "city", "type": "string" },
+          { "name": "zip", "type": "string" }
+        ]
+      }
+    }
+  ]
+}
+```
+...this will no longer work in v0.12.0:
+```ruby
+store = AvroTurf::SchemaStore.new(path: 'my/schemas')
+store.load_schemas!
+
+# Accessing 'person' is correct and works fine.
+person = store.find('person', 'contacts') # my/schemas/contacts/person.avsc exists
+
+# Trying to access 'address' raises AvroTurf::SchemaNotFoundError
+address = store.find('address', 'contacts') # my/schemas/contacts/address.avsc is not found
+```
+
+For details and context, see [this pull request](https://github.com/dasch/avro_turf/pull/111).
+
 ## Installation
 
 Add this line to your application's Gemfile:

data/avro_turf.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "avro", ">= 1.7.7", "< 1.10"
-  spec.add_dependency "excon", "~> 0.45"
+  spec.add_dependency "excon", "~> 0.71"
 
   spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "rake", "~> 13.0"

data/lib/avro_turf/schema_store.rb

@@ -46,26 +46,45 @@ class AvroTurf::SchemaStore
 
   # Loads single schema
   # Such method is not thread-safe, do not call it of from mutex synchronization routine
-  def load_schema!(fullname, namespace = nil)
+  def load_schema!(fullname, namespace = nil, local_schemas_cache = {})
     *namespace, schema_name = fullname.split(".")
     schema_path = File.join(@path, *namespace, schema_name + ".avsc")
     schema_json = JSON.parse(File.read(schema_path))
-    schema = Avro::Schema.real_parse(schema_json, @schemas)
 
+    schema = Avro::Schema.real_parse(schema_json, local_schemas_cache)
+
+    # Don't cache the parsed schema until after its fullname is validated
     if schema.respond_to?(:fullname) && schema.fullname != fullname
       raise AvroTurf::SchemaError, "expected schema `#{schema_path}' to define type `#{fullname}'"
     end
 
+    # Cache only this new top-level schema by its fullname. It's critical
+    # not to make every sub-schema resolvable at the top level here because
+    # multiple different avsc files may define the same sub-schema, and
+    # if we share the @schemas cache across all parsing contexts, the Avro
+    # gem will raise an Avro::SchemaParseError when parsing another avsc
+    # file that contains a subschema with the same fullname as one
+    # encountered previously in a different file:
+    # <Avro::SchemaParseError: The name "foo.bar" is already in use.>
+    # Essentially, the only schemas that should be resolvable in @schemas
+    # are those that have their own .avsc files on disk.
+    @schemas[fullname] = schema
+
     schema
   rescue ::Avro::SchemaParseError => e
     # This is a hack in order to figure out exactly which type was missing. The
     # Avro gem ought to provide this data directly.
     if e.to_s =~ /"([\w\.]+)" is not a schema we know about/
-      load_schema!($1)
+      # Try to first resolve a referenced schema from disk.
+      # If this is successful, the Avro gem will have mutated the
+      # local_schemas_cache, adding all the new schemas it found.
+      load_schema!($1, nil, local_schemas_cache)
 
-      # Re-resolve the original schema now that the dependency has been resolved.
-      @schemas.delete(fullname)
-      load_schema!(fullname)
+      # Attempt to re-parse the original schema now that the dependency
+      # has been resolved and use the now-updated local_schemas_cache to
+      # pick up where we left off.
+      local_schemas_cache.delete(fullname)
+      load_schema!(fullname, nil, local_schemas_cache)
     else
       raise
     end

data/lib/avro_turf/version.rb

@@ -1,3 +1,3 @@
 class AvroTurf
-  VERSION = "0.11.0"
+  VERSION = "1.0.0"
 end

data/spec/schema_store_spec.rb

@@ -198,6 +198,104 @@ describe AvroTurf::SchemaStore do
       expect(schema.fullname).to eq "person"
     end
 
+    # This test would fail under avro_turf <= v0.11.0
+    it "does NOT cache *nested* schemas in memory" do
+      FileUtils.mkdir_p("spec/schemas/test")
+
+      define_schema "test/person.avsc", <<-AVSC
+        {
+          "name": "person",
+          "namespace": "test",
+          "type": "record",
+          "fields": [
+            {
+              "name": "address",
+              "type": {
+                "name": "address",
+                "type": "record",
+                "fields": [
+                  { "name": "addr1", "type": "string" },
+                  { "name": "addr2", "type": "string" },
+                  { "name": "city", "type": "string" },
+                  { "name": "zip", "type": "string" }
+                ]
+              }
+            }
+          ]
+        }
+      AVSC
+
+      schema = store.find('person', 'test')
+      expect(schema.fullname).to eq "test.person"
+
+      expect { store.find('address', 'test') }.
+        to raise_error(AvroTurf::SchemaNotFoundError)
+    end
+
+    # This test would fail under avro_turf <= v0.11.0
+    it "allows two different avsc files to define nested sub-schemas with the same fullname" do
+      FileUtils.mkdir_p("spec/schemas/test")
+
+      define_schema "test/person.avsc", <<-AVSC
+        {
+          "name": "person",
+          "namespace": "test",
+          "type": "record",
+          "fields": [
+            {
+              "name": "location",
+              "type": {
+                "name": "location",
+                "type": "record",
+                "fields": [
+                  { "name": "city", "type": "string" },
+                  { "name": "zipcode", "type": "string" }
+                ]
+              }
+            }
+          ]
+        }
+      AVSC
+
+      define_schema "test/company.avsc", <<-AVSC
+        {
+          "name": "company",
+          "namespace": "test",
+          "type": "record",
+          "fields": [
+            {
+              "name": "headquarters",
+              "type": {
+                "name": "location",
+                "type": "record",
+                "fields": [
+                  { "name": "city", "type": "string" },
+                  { "name": "postcode", "type": "string" }
+                ]
+              }
+            }
+          ]
+        }
+      AVSC
+
+      company = nil
+      person = store.find('person', 'test')
+
+      # This should *NOT* raise the error:
+      # #<Avro::SchemaParseError: The name "test.location" is already in use.>
+      expect { company = store.find('company', 'test') }.not_to raise_error
+
+      person_location_field = person.fields_hash['location']
+      expect(person_location_field.type.name).to eq('location')
+      expect(person_location_field.type.fields_hash).to include('zipcode')
+      expect(person_location_field.type.fields_hash).not_to include('postcode')
+
+      company_headquarters_field = company.fields_hash['headquarters']
+      expect(company_headquarters_field.type.name).to eq('location')
+      expect(company_headquarters_field.type.fields_hash).to include('postcode')
+      expect(company_headquarters_field.type.fields_hash).not_to include('zipcode')
+    end
+
     it "is thread safe" do
       define_schema "address.avsc", <<-AVSC
         {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: avro_turf
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-25 00:00:00.000000000 Z
+date: 2020-05-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro
@@ -36,14 +36,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.45'
+        version: '0.71'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.45'
+        version: '0.71'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -165,6 +165,7 @@ extra_rdoc_files: []
 files:
 - ".circleci/config.yml"
 - ".github/workflows/ruby.yml"
+- ".github/workflows/stale.yml"
 - ".gitignore"
 - ".rspec"
 - CHANGELOG.md