avro_turf 1.8.0 → 1.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 67ff8ee9b578227cd54a33d76a95fed8bd2c9a396aa360ad9df9f52d01e5c25a
-  data.tar.gz: 8bca24236ce42f19f6db457ad13d76d33e89b18703dc0eb74a8adca53b2c7b6e
+  metadata.gz: c0907531d40226b59cd545baf7341f3e41613565d50de98684b266b6b33a081f
+  data.tar.gz: e472794d6e7ecbd4a32b1a08f4cc0eb6b0ae810d849483d86ab16a7da929a9fb
 SHA512:
-  metadata.gz: 859e8fa6938679007704d35b1b4a54616da7637c83f41c58cbd10828216e0af198846e755334f465613ceb9086ec0c63a823dc2cbf3ddbdc191a97423d2e3147
-  data.tar.gz: 7900a85abaf9dd55ac3b89bba2d04935ddbafc850a92fb1941ff8984df864cfaa4bb5527ca4d48a74ef3715915d598044b7c486363909f2705ece83b0ffca50f
+  metadata.gz: ac1324149cb3299c3e67ffbd287879da88a09c338b967e2098fed63836c9fba5f775623ee6d6ae2ac0ac5e6f6a4ad28ea4c7bb7ece2d1e5979cee2eb379ac43a
+  data.tar.gz: 84709d06f4b3d97a23c1f0a1b94f4757c8b7024d17354ec65c7068be8cca52cdeed832096d7ea43f646d3f52d09ebd0f2286375456503d9da14cf6cb7a37772b

data/.github/workflows/ruby.yml

@@ -7,17 +7,17 @@ jobs:
 
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        ruby: [2.3, 2.4, 2.5, 2.6, 2.7, 3.0]
+        ruby: [2.6, 2.7, "3.0", 3.1, 3.2]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Ruby ${{ matrix.ruby }}
       uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}
+        bundler-cache: true
     - name: Build and test with RSpec
       run: |
-        gem install bundler
-        bundle install --jobs 4 --retry 3
         bundle exec rspec

data/CHANGELOG.md

@@ -2,9 +2,18 @@
 
 ## Unreleased
 
+## v1.10.0
+
+- Add `schema_subject_versions` to `ConfluentSchemaRegistry` to retrieve all subject versions for a schema id. (#189)
+- `FakeConfluentSchemaRegistryServer` now returns same id if identical schema is created for a different subject (#188)
+
+## v1.9.0
+
+- Send Accept and User-Agent headers on every request (#184)
+
 ## v1.8.0
 
-- add support for `Date` via appropriate logicalType defintion.  This is a backwards incompatible change  (#177)
+- Add support for `Date` via appropriate logicalType defintion.  This is a backwards incompatible change  (#177)
 - Fixed schema file cache truncation on multiple running instances and parallel access to the cache files.
 
 ## v1.7.0

data/avro_turf.gemspec

@@ -25,7 +25,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.2"
-  spec.add_development_dependency "fakefs", "~> 0.20.0"
+  spec.add_development_dependency "fakefs", "< 3"
   spec.add_development_dependency "webmock"
   spec.add_development_dependency "sinatra"
   spec.add_development_dependency "json_spec"

data/lib/avro_turf/cached_confluent_schema_registry.rb

@@ -17,7 +17,7 @@ class AvroTurf::CachedConfluentSchemaRegistry
   end
 
   # Delegate the following methods to the upstream
-  %i(subjects subject_versions check compatible?
+  %i(subjects subject_versions schema_subject_versions check compatible?
      global_config update_global_config subject_config update_subject_config).each do |name|
     define_method(name) do |*args|
       instance_variable_get(:@upstream).send(name, *args)

data/lib/avro_turf/confluent_schema_registry.rb

@@ -19,9 +19,9 @@ class AvroTurf::ConfluentSchemaRegistry
   )
     @path_prefix = path_prefix
     @logger = logger
-    headers = {
+    headers = Excon.defaults[:headers].merge(
       "Content-Type" => CONTENT_TYPE
-    }
+    )
     headers[:proxy] = proxy unless proxy.nil?
     @connection = Excon.new(
       url,
@@ -68,6 +68,11 @@ class AvroTurf::ConfluentSchemaRegistry
     get("/subjects/#{subject}/versions/#{version}")
   end
 
+  # Get the subject and version for a schema id
+  def schema_subject_versions(schema_id)
+    get("/schemas/ids/#{schema_id}/versions")
+  end
+
   # Check if a schema exists. Returns nil if not found.
   def check(subject, schema)
     data = post("/subjects/#{subject}",

data/lib/avro_turf/in_memory_cache.rb

@@ -17,12 +17,12 @@ class AvroTurf::InMemoryCache
   end
 
   def lookup_by_schema(subject, schema)
-    key = [subject, schema.to_s]
+    key = [subject, schema]
     @ids_by_schema[key]
   end
 
   def store_by_schema(subject, schema, id)
-    key = [subject, schema.to_s]
+    key = [subject, schema]
     @ids_by_schema[key] = id
   end
 

data/lib/avro_turf/schema_store.rb

@@ -22,7 +22,7 @@ class AvroTurf::SchemaStore
       # Still need to check is the schema already loaded
       return @schemas[fullname] if @schemas.key?(fullname)
 
-      load_schema!(fullname)
+      load_schema!(fullname, @schemas.dup)
     end
   end
 

data/lib/avro_turf/test/fake_confluent_schema_registry_server.rb

@@ -35,23 +35,35 @@ class FakeConfluentSchemaRegistryServer < Sinatra::Base
 
   post "/subjects/:subject/versions" do
     schema = parse_schema
-    ids_for_subject = SUBJECTS[params[:subject]]
-
-    schemas_for_subject =
-      SCHEMAS.select
-             .with_index { |_, i| ids_for_subject.include?(i) }
-
-    if schemas_for_subject.include?(schema)
-      schema_id = SCHEMAS.index(schema)
-    else
+    schema_id = SCHEMAS.index(schema)
+    if schema_id.nil?
       SCHEMAS << schema
       schema_id = SCHEMAS.size - 1
-      SUBJECTS[params[:subject]] = SUBJECTS[params[:subject]] << schema_id
+    end
+
+    subject = params[:subject]
+    unless SUBJECTS[subject].include?(schema_id)
+      SUBJECTS[subject] = SUBJECTS[subject] << schema_id
     end
 
     { id: schema_id }.to_json
   end
 
+  get "/schemas/ids/:schema_id/versions" do
+    schema_id = params[:schema_id].to_i
+    schema = SCHEMAS.at(schema_id)
+    halt(404, SCHEMA_NOT_FOUND) unless schema
+
+    related_subjects = SUBJECTS.select {|_, vs| vs.include? schema_id }
+
+    related_subjects.map do |subject, versions|
+      {
+        subject: subject,
+        version: versions.find_index(schema_id) + 1
+      }
+    end.to_json
+  end
+
   get "/schemas/ids/:schema_id" do
     schema = SCHEMAS.at(params[:schema_id].to_i)
     halt(404, SCHEMA_NOT_FOUND) unless schema

data/lib/avro_turf/version.rb

@@ -1,3 +1,3 @@
 class AvroTurf
-  VERSION = "1.8.0"
+  VERSION = "1.10.0"
 end

data/lib/avro_turf.rb

@@ -121,10 +121,9 @@ class AvroTurf
   # namespace - The namespace of the Avro schema (optional).
   #
   # Returns true if the data is valid, false otherwise.
-  def valid?(data, schema_name: nil, namespace: @namespace)
+  def valid?(data, schema_name: nil, namespace: @namespace, validate_options: {})
     schema = schema_name && @schema_store.find(schema_name, namespace)
-
-    Avro::Schema.validate(schema, data.as_avro)
+    Avro::Schema.validate(schema, data.as_avro, **validate_options)
   end
 
   # Loads all schema definition files in the `schemas_dir`.

data/spec/avro_turf_spec.rb

@@ -360,11 +360,42 @@ describe AvroTurf do
         }
       AVSC
 
-      datum = { 
+      datum = {
         message: "hello",
         sent_date: Date.new(2022, 9, 11)
       }
       expect(avro.valid?(datum, schema_name: "postcard")).to eq true
     end
+
+    context "when message contains extra fields (typo in key)" do
+      let(:message) { { "fulll_name" => "John Doe" } }
+
+      before do
+        define_schema "message.avsc", <<-AVSC
+          {
+            "name": "message",
+            "type": "record",
+            "fields": [
+              { "name": "full_name", "type": "string" }
+            ]
+          }
+        AVSC
+      end
+
+      it "is valid" do
+        datum = { "full_name" => "John Doe", "extra" => "extra" }
+        expect(avro.valid?(datum, schema_name: "message")).to eq true
+      end
+
+      it "is invalid when passing fail_on_extra_fields" do
+        datum = { "full_name" => "John Doe", "extra" => "extra" }
+        validate_options = {
+          recursive: true,
+          encoded: false,
+          fail_on_extra_fields: true }
+        valid = avro.valid?(datum, schema_name: "message", validate_options: validate_options)
+        expect(valid).to eq false
+      end
+    end
   end
 end

data/spec/messaging_spec.rb

@@ -109,7 +109,7 @@ describe AvroTurf::Messaging do
     end
 
     it 'encodes and decodes messages' do
-      data = avro.encode(message, schema_id: 1)
+      data = avro.encode(message, schema_id: 0)
       expect(avro.decode(data)).to eq message
     end
 
@@ -118,7 +118,7 @@ describe AvroTurf::Messaging do
     end
 
     it 'caches parsed schemas for decoding' do
-      data = avro.encode(message, schema_id: 1)
+      data = avro.encode(message, schema_id: 0)
       avro.decode(data)
       allow(Avro::Schema).to receive(:parse).and_call_original
       expect(avro.decode(data)).to eq message

data/spec/schema_store_spec.rb

@@ -361,6 +361,49 @@ describe AvroTurf::SchemaStore do
   end
 
   describe "#load_schemas!" do
+    it "do not try to reload known schemas" do
+      define_schema "first.avsc", <<-AVSC
+        {
+          "name": "first",
+          "type": "record",
+          "fields": [
+            {
+              "type": "string",
+              "name": "a_value"
+            }
+          ]
+        }
+      AVSC
+
+      define_schema "second.avsc", <<-AVSC
+        {
+          "name": "second",
+          "type": "record",
+          "fields": [
+            {
+              "type": "first",
+              "name": "full_name"
+            }
+          ]
+        }
+      AVSC
+
+      define_schema "third.avsc", <<-AVSC
+        {
+          "name": "third",
+          "type": "record",
+          "fields": [
+            {
+              "type": "second",
+              "name": "embedded_first"
+            }
+          ]
+        }
+      AVSC
+
+      expect { store.load_schemas! }.not_to raise_error
+    end
+
     it "loads schemas defined in the `schemas_path` directory" do
       define_schema "person.avsc", <<-AVSC
         {

data/spec/spec_helper.rb

@@ -1,6 +1,7 @@
 require 'bundler/setup'
 require 'logger'
 require 'json_spec'
+require 'pp' # Require pp before fakefs to fix TypeError: superclass mismatch for class File
 require 'fakefs/spec_helpers'
 require 'avro_turf'
 

data/spec/support/confluent_schema_registry_context.rb

@@ -13,9 +13,20 @@ shared_examples_for "a confluent schema registry client" do
       ]
     }.to_json
   end
+  let(:headers) do
+    {
+      'Accept'=>'*/*',
+      'Content-Type'=> AvroTurf::ConfluentSchemaRegistry::CONTENT_TYPE,
+      'Host'=> "#{URI.parse(registry_url).host}:80",
+      'User-Agent'=> "excon/#{Excon::VERSION}"
+    }
+  end
 
   before do
-    stub_request(:any, /^#{registry_url}/).to_rack(FakeConfluentSchemaRegistryServer)
+    stub_request(:any, /^#{registry_url}/)
+      .with(headers: headers)
+      .to_rack(FakeConfluentSchemaRegistryServer)
+
     FakeConfluentSchemaRegistryServer.clear
   end
 
@@ -66,6 +77,33 @@ shared_examples_for "a confluent schema registry client" do
     end
   end
 
+  describe "#schema_subject_versions" do
+    it "returns subject and version for a schema id" do
+      schema_id1 = registry.register(subject_name, { type: :record, name: "r1", fields: [] }.to_json)
+      registry.register(subject_name, { type: :record, name: "r2", fields: [] }.to_json)
+      schema_id2 = registry.register("other#{subject_name}", { type: :record, name: "r2", fields: [] }.to_json)
+      expect(registry.schema_subject_versions(schema_id1)).to eq([
+        'subject' => subject_name,
+        'version' => 1
+      ])
+      expect(registry.schema_subject_versions(schema_id2)).to include({
+        'subject' => subject_name,
+        'version' => 2
+      },{
+        'subject' => "other#{subject_name}",
+        'version' => 1
+      } )
+    end
+
+    context "when the schema does not exist" do
+      it "raises an error" do
+        expect do
+          registry.schema_subject_versions(-1)
+        end.to raise_error(Excon::Errors::NotFound)
+      end
+    end
+  end
+
   describe "#subject_versions" do
     it "lists all the versions for the subject" do
       2.times do |n|

data/spec/test/fake_confluent_schema_registry_server_spec.rb

@@ -27,14 +27,75 @@ describe FakeConfluentSchemaRegistryServer do
       expect(JSON.parse(last_response.body).fetch('id')).to eq expected_id
     end
 
-    it 'returns a different schema ID when invoked with same schema and different subject' do
+    it 'returns the same schema ID when invoked with same schema and different subject' do
       post '/subjects/person/versions', { schema: schema }.to_json, 'CONTENT_TYPE' => 'application/vnd.schemaregistry+json'
 
       original_id = JSON.parse(last_response.body).fetch('id')
 
       post '/subjects/happy-person/versions', { schema: schema }.to_json, 'CONTENT_TYPE' => 'application/vnd.schemaregistry+json'
 
-      expect(JSON.parse(last_response.body).fetch('id')).not_to eq original_id
+      expect(JSON.parse(last_response.body).fetch('id')).to eq original_id
+    end
+
+    it 'returns a different schema ID when invoked with a different schema' do
+      post '/subjects/person/versions', { schema: schema }.to_json, 'CONTENT_TYPE' => 'application/vnd.schemaregistry+json'
+
+      original_id = JSON.parse(last_response.body).fetch('id')
+
+      other_schema = {
+        type: "record",
+        name: "other",
+        fields: [
+          { name: "name", type: "string" }
+        ]
+      }.to_json
+
+      post '/subjects/person/versions', { schema: other_schema }.to_json, 'CONTENT_TYPE' => 'application/vnd.schemaregistry+json'
+
+      expect(JSON.parse(last_response.body).fetch('id')).to_not eq original_id
+    end
+  end
+
+  describe 'GET /schemas/ids/:id/versions' do
+    def schema(name:)
+      {
+        type: "record",
+        name: name,
+        fields: [
+          { name: "name", type: "string" },
+        ]
+      }.to_json
+    end
+
+    it "returns array containing subjects and versions for given schema id" do
+      schema1 = schema(name: "name1")
+      schema2 = schema(name: "name2")
+
+      post "/subjects/cats/versions", { schema: schema1 }.to_json, 'CONTENT_TYPE' => 'application/vnd.schemaregistry+json'
+      schema1_id = JSON.parse(last_response.body).fetch('id') # Original cats schema
+
+      post "/subjects/dogs/versions", { schema: schema2 }.to_json, 'CONTENT_TYPE' => 'application/vnd.schemaregistry+json'
+      post "/subjects/cats/versions", { schema: schema2 }.to_json, 'CONTENT_TYPE' => 'application/vnd.schemaregistry+json'
+      schema2_id = JSON.parse(last_response.body).fetch('id') # Changed cats schema == Original Dogs schema
+
+      get "/schemas/ids/#{schema1_id}/versions"
+      result = JSON.parse(last_response.body)
+
+      expect(result).to eq [{
+        'subject' => 'cats',
+        'version' => 1
+      }]
+
+      get "/schemas/ids/#{schema2_id}/versions"
+      result = JSON.parse(last_response.body)
+
+      expect(result).to include( {
+        'subject' => 'cats',
+        'version' => 2
+      }, {
+        'subject' => 'dogs',
+        'version' => 1
+      })
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: avro_turf
 version: !ruby/object:Gem::Version
-  version: 1.8.0
+  version: 1.10.0
 platform: ruby
 authors:
 - Daniel Schierbeck
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-11-14 00:00:00.000000000 Z
+date: 2023-08-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro
@@ -90,16 +90,16 @@ dependencies:
   name: fakefs
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.20.0
+        version: '3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.20.0
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -156,7 +156,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description:
+description: 
 email:
 - dasch@zendesk.com
 executables: []
@@ -251,8 +251,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.3
-signing_key:
+rubygems_version: 3.0.3.1
+signing_key: 
 specification_version: 4
 summary: A library that makes it easier to use the Avro serialization format from
   Ruby