avro_turf 1.9.0 → 1.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3cb36f9b1395a1a6210d1178ca0823636f30ad194caf57663e8fb5cc13273d68
-  data.tar.gz: 2fb489fe330b39b88ed585d6b622936e573b525c1f4c5c4fe50704b16b11fd39
+  metadata.gz: 31248ad33908f238baea49fc106bc3e6292f45a1fe7fce0c0ae091dc10a9ea17
+  data.tar.gz: 25af934023a269b28acf004a1ec13c052380f5a664696ce881ef27efb209a7e5
 SHA512:
-  metadata.gz: be796c4b42daf30a79dd699d23842e61c61e8f8c70b1645f269ec25e4a4f4fb1e87148b0ffd69a68b24382e9abc01048cb3ba082bd3a3dd2162be714ecdcd57b
-  data.tar.gz: 332486d45cd611da3906247f4fb55386c7e1617043070dd44fff06b05c50c44237af6dad62f1fd087ca9c67580aadd53fc342f76509b69b67375532b60637bf9
+  metadata.gz: 6e0796ce28027a654fa189f561f8179567fbebf50ba6c225be9b8623999e7ef424b58b05d1282a151474aaf48057b0000ce7999f6f8aed1c5f56f53f2d1f8cab
+  data.tar.gz: 70fb1dc693589e71691ab8bf18cef28ebe533921efe401c6c0f7c3f1ace8ac880e5e3bd1161bd3235148f7f469c837d5224f90ea96a3867aa0fe24a16de91200

data/CHANGELOG.md

@@ -2,6 +2,17 @@
 
 ## Unreleased
 
+## v1.11.0
+
+- Add `decode_all` and `decode_all_from_stream` methods to return all entries in a data file (#194)
+- Improve the way schemas are automatically loaded (#190)
+- Increment dependency on `avro` gem to v1.8.
+
+## v1.10.0
+
+- Add `schema_subject_versions` to `ConfluentSchemaRegistry` to retrieve all subject versions for a schema id. (#189)
+- `FakeConfluentSchemaRegistryServer` now returns same id if identical schema is created for a different subject (#188)
+
 ## v1.9.0
 
 - Send Accept and User-Agent headers on every request (#184)

data/avro_turf.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "avro", ">= 1.7.7", "< 1.12"
+  spec.add_dependency "avro", ">= 1.8.0", "< 1.12"
   spec.add_dependency "excon", "~> 0.71"
 
   spec.add_development_dependency "bundler", "~> 2.0"

data/lib/avro_turf/cached_confluent_schema_registry.rb

@@ -17,7 +17,7 @@ class AvroTurf::CachedConfluentSchemaRegistry
   end
 
   # Delegate the following methods to the upstream
-  %i(subjects subject_versions check compatible?
+  %i(subjects subject_versions schema_subject_versions check compatible?
      global_config update_global_config subject_config update_subject_config).each do |name|
     define_method(name) do |*args|
       instance_variable_get(:@upstream).send(name, *args)

data/lib/avro_turf/confluent_schema_registry.rb

@@ -68,6 +68,11 @@ class AvroTurf::ConfluentSchemaRegistry
     get("/subjects/#{subject}/versions/#{version}")
   end
 
+  # Get the subject and version for a schema id
+  def schema_subject_versions(schema_id)
+    get("/schemas/ids/#{schema_id}/versions")
+  end
+
   # Check if a schema exists. Returns nil if not found.
   def check(subject, schema)
     data = post("/subjects/#{subject}",

data/lib/avro_turf/schema_store.rb

@@ -22,7 +22,7 @@ class AvroTurf::SchemaStore
       # Still need to check is the schema already loaded
       return @schemas[fullname] if @schemas.key?(fullname)
 
-      load_schema!(fullname)
+      load_schema!(fullname, @schemas.dup)
     end
   end
 
@@ -70,28 +70,22 @@ class AvroTurf::SchemaStore
     @schemas[fullname] = schema
 
     schema
-  rescue ::Avro::SchemaParseError => e
-    # This is a hack in order to figure out exactly which type was missing. The
-    # Avro gem ought to provide this data directly.
-    if e.to_s =~ /"([\w\.]+)" is not a schema we know about/
-      # Try to first resolve a referenced schema from disk.
-      # If this is successful, the Avro gem will have mutated the
-      # local_schemas_cache, adding all the new schemas it found.
-      load_schema!($1, local_schemas_cache)
-
-      # Attempt to re-parse the original schema now that the dependency
-      # has been resolved and use the now-updated local_schemas_cache to
-      # pick up where we left off.
-      local_schemas_cache.delete(fullname)
-      # Ensure all sub-schemas are cleaned up to avoid conflicts when re-parsing
-      # schema.
-      local_schemas_cache.each do |schema_name, schema|
-        local_schemas_cache.delete(schema_name) unless File.exist?(build_schema_path(schema_name))
-      end
-      load_schema!(fullname, @schemas.dup)
-    else
-      raise
+  rescue ::Avro::UnknownSchemaError => e
+    # Try to first resolve a referenced schema from disk.
+    # If this is successful, the Avro gem will have mutated the
+    # local_schemas_cache, adding all the new schemas it found.
+    load_schema!(e.type_name, local_schemas_cache)
+
+    # Attempt to re-parse the original schema now that the dependency
+    # has been resolved and use the now-updated local_schemas_cache to
+    # pick up where we left off.
+    local_schemas_cache.delete(fullname)
+    # Ensure all sub-schemas are cleaned up to avoid conflicts when re-parsing
+    # schema.
+    local_schemas_cache.each_key do |schema_name|
+      local_schemas_cache.delete(schema_name) unless File.exist?(build_schema_path(schema_name))
     end
+    load_schema!(fullname, @schemas.dup)
   rescue Errno::ENOENT, Errno::ENAMETOOLONG
     raise AvroTurf::SchemaNotFoundError, "could not find Avro schema at `#{schema_path}'"
   end

data/lib/avro_turf/test/fake_confluent_schema_registry_server.rb

@@ -35,23 +35,35 @@ class FakeConfluentSchemaRegistryServer < Sinatra::Base
 
   post "/subjects/:subject/versions" do
     schema = parse_schema
-    ids_for_subject = SUBJECTS[params[:subject]]
-
-    schemas_for_subject =
-      SCHEMAS.select
-             .with_index { |_, i| ids_for_subject.include?(i) }
-
-    if schemas_for_subject.include?(schema)
-      schema_id = SCHEMAS.index(schema)
-    else
+    schema_id = SCHEMAS.index(schema)
+    if schema_id.nil?
       SCHEMAS << schema
       schema_id = SCHEMAS.size - 1
-      SUBJECTS[params[:subject]] = SUBJECTS[params[:subject]] << schema_id
+    end
+
+    subject = params[:subject]
+    unless SUBJECTS[subject].include?(schema_id)
+      SUBJECTS[subject] = SUBJECTS[subject] << schema_id
     end
 
     { id: schema_id }.to_json
   end
 
+  get "/schemas/ids/:schema_id/versions" do
+    schema_id = params[:schema_id].to_i
+    schema = SCHEMAS.at(schema_id)
+    halt(404, SCHEMA_NOT_FOUND) unless schema
+
+    related_subjects = SUBJECTS.select {|_, vs| vs.include? schema_id }
+
+    related_subjects.map do |subject, versions|
+      {
+        subject: subject,
+        version: versions.find_index(schema_id) + 1
+      }
+    end.to_json
+  end
+
   get "/schemas/ids/:schema_id" do
     schema = SCHEMAS.at(params[:schema_id].to_i)
     halt(404, SCHEMA_NOT_FOUND) unless schema

data/lib/avro_turf/version.rb

@@ -1,3 +1,3 @@
 class AvroTurf
-  VERSION = "1.9.0"
+  VERSION = "1.11.0"
 end

data/lib/avro_turf.rb

@@ -93,24 +93,39 @@ class AvroTurf
   # namespace    - The namespace of the Avro schema used to decode the data.
   #
   # Returns whatever is encoded in the data.
-  def decode(encoded_data, schema_name: nil, namespace: @namespace)
+  def decode_first(encoded_data, schema_name: nil, namespace: @namespace)
     stream = StringIO.new(encoded_data)
     decode_stream(stream, schema_name: schema_name, namespace: namespace)
   end
 
-  # Decodes Avro data from an IO stream.
+  alias decode decode_first
+
+  # Returns all entries encoded in the data.
+  def decode_all(encoded_data, schema_name: nil, namespace: @namespace)
+    stream = StringIO.new(encoded_data)
+    decode_all_from_stream(stream, schema_name: schema_name, namespace: namespace)
+  end
+
+  # Decodes the first entry from an IO stream containing Avro data.
   #
   # stream       - An IO object containing Avro data.
   # schema_name  - The String name of the schema that should be used to read
   #                the data. If nil, the writer schema will be used.
   # namespace    - The namespace of the Avro schema used to decode the data.
   #
-  # Returns whatever is encoded in the stream.
-  def decode_stream(stream, schema_name: nil, namespace: @namespace)
+  # Returns first entry encoded in the stream.
+  def decode_first_from_stream(stream, schema_name: nil, namespace: @namespace)
+    data = decode_all_from_stream(stream, schema_name: schema_name, namespace: namespace)
+    data.first
+  end
+
+  alias decode_stream decode_first_from_stream
+
+  # Returns all entries encoded in the stream.
+  def decode_all_from_stream(stream, schema_name: nil, namespace: @namespace)
     schema = schema_name && @schema_store.find(schema_name, namespace)
     reader = Avro::IO::DatumReader.new(nil, schema)
-    dr = Avro::DataFile::Reader.new(stream, reader)
-    dr.first
+    Avro::DataFile::Reader.new(stream, reader)
   end
 
   # Validates data against an Avro schema.

data/spec/avro_turf_spec.rb

@@ -186,6 +186,47 @@ describe AvroTurf do
     end
   end
 
+  describe "#decode_all" do
+    context "when data contains multiple entries" do
+      let(:encoded_data) {  "Obj\u0001\u0004\u0014avro.codec\bnull\u0016avro.schema\xB6\u0004[{\"type\": \"record\", \"name\": \"address\", \"fields\": [{\"type\": \"string\", \"name\": \"street\"}, {\"type\": \"string\", \"name\": \"city\"}]}, {\"type\": \"record\", \"name\": \"person\", \"fields\": [{\"type\": \"string\", \"name\": \"name\"}, {\"type\": \"int\", \"name\": \"age\"}, {\"type\": \"address\", \"name\": \"address\"}]}]\u0000\xF9u\x84\xA1c\u0010\x82B\xE2\xCF\xF1\x98\xF7\xF1JH\u0004\x96\u0001\u0002\u0014Python🐍\x80\u0004\u0018Green Street\u001ASan Francisco\u0002\u0010Mojo🐍\u0002\u0016Blue Street\u0014Saturn🪐\xF9u\x84\xA1c\u0010\x82B\xE2\xCF\xF1\x98\xF7\xF1JH" } 
+
+      it "returns array of entries decoded using the inlined writer's schema " do
+        expect(avro.decode_all(encoded_data).entries).to eq(
+          [
+            {"name"=>"Python🐍", "age"=>256, "address"=>{"street"=>"Green Street", "city"=>"San Francisco"}},
+            {"name"=>"Mojo🐍", "age"=>1, "address"=>{"street"=>"Blue Street", "city"=>"Saturn🪐"}}
+          ]
+        )
+      end
+
+      it "returns array of entries decoded using the specified reader's schema " do
+        FileUtils.mkdir_p("spec/schemas/reader")
+
+        define_schema "reader/person.avsc", <<-AVSC
+          {
+            "name": "person",
+            "type": "record",
+            "fields": [
+              { "name": "fav_color", "type": "string", "default": "red🟥" },
+              { "name": "name", "type": "string" },
+              { "name": "age", "type": "int" }
+            ]
+          }
+        AVSC
+
+        expect(
+          AvroTurf.new(schemas_path: "spec/schemas/reader")
+                  .decode_all(encoded_data, schema_name: "person").entries
+        ).to eq(
+          [
+            {"name"=>"Python🐍", "age"=>256, "fav_color"=>"red🟥"},
+            {"name"=>"Mojo🐍", "age"=>1, "fav_color"=>"red🟥"}
+          ]
+        )
+      end
+    end
+  end
+
   describe "#encode_to_stream" do
     it "writes encoded data to an existing stream" do
       define_schema "message.avsc", <<-AVSC
@@ -304,6 +345,20 @@ describe AvroTurf do
     end
   end
 
+  describe "#decode_all_from_stream" do
+    it "returns all entries when decodes Avro data from a stream containing multiple entries" do
+      encoded_data = "Obj\u0001\u0004\u0014avro.codec\bnull\u0016avro.schema\xB6\u0004[{\"type\": \"record\", \"name\": \"address\", \"fields\": [{\"type\": \"string\", \"name\": \"street\"}, {\"type\": \"string\", \"name\": \"city\"}]}, {\"type\": \"record\", \"name\": \"person\", \"fields\": [{\"type\": \"string\", \"name\": \"name\"}, {\"type\": \"int\", \"name\": \"age\"}, {\"type\": \"address\", \"name\": \"address\"}]}]\u0000\xF9u\x84\xA1c\u0010\x82B\xE2\xCF\xF1\x98\xF7\xF1JH\u0004\x96\u0001\u0002\u0014Python🐍\x80\u0004\u0018Green Street\u001ASan Francisco\u0002\u0010Mojo🐍\u0002\u0016Blue Street\u0014Saturn🪐\xF9u\x84\xA1c\u0010\x82B\xE2\xCF\xF1\x98\xF7\xF1JH"
+      stream = StringIO.new(encoded_data)
+
+      expect(avro.decode_all_from_stream(stream).entries).to eq(
+        [
+          {"name"=>"Python🐍", "age"=>256, "address"=>{"street"=>"Green Street", "city"=>"San Francisco"}},
+          {"name"=>"Mojo🐍", "age"=>1, "address"=>{"street"=>"Blue Street", "city"=>"Saturn🪐"}}
+        ]
+      )
+    end
+  end
+
   describe "#valid?" do
     before do
       define_schema "message.avsc", <<-AVSC

data/spec/messaging_spec.rb

@@ -109,7 +109,7 @@ describe AvroTurf::Messaging do
     end
 
     it 'encodes and decodes messages' do
-      data = avro.encode(message, schema_id: 1)
+      data = avro.encode(message, schema_id: 0)
       expect(avro.decode(data)).to eq message
     end
 
@@ -118,7 +118,7 @@ describe AvroTurf::Messaging do
     end
 
     it 'caches parsed schemas for decoding' do
-      data = avro.encode(message, schema_id: 1)
+      data = avro.encode(message, schema_id: 0)
       avro.decode(data)
       allow(Avro::Schema).to receive(:parse).and_call_original
       expect(avro.decode(data)).to eq message

data/spec/schema_store_spec.rb

@@ -361,6 +361,49 @@ describe AvroTurf::SchemaStore do
   end
 
   describe "#load_schemas!" do
+    it "do not try to reload known schemas" do
+      define_schema "first.avsc", <<-AVSC
+        {
+          "name": "first",
+          "type": "record",
+          "fields": [
+            {
+              "type": "string",
+              "name": "a_value"
+            }
+          ]
+        }
+      AVSC
+
+      define_schema "second.avsc", <<-AVSC
+        {
+          "name": "second",
+          "type": "record",
+          "fields": [
+            {
+              "type": "first",
+              "name": "full_name"
+            }
+          ]
+        }
+      AVSC
+
+      define_schema "third.avsc", <<-AVSC
+        {
+          "name": "third",
+          "type": "record",
+          "fields": [
+            {
+              "type": "second",
+              "name": "embedded_first"
+            }
+          ]
+        }
+      AVSC
+
+      expect { store.load_schemas! }.not_to raise_error
+    end
+
     it "loads schemas defined in the `schemas_path` directory" do
       define_schema "person.avsc", <<-AVSC
         {

data/spec/spec_helper.rb

@@ -1,6 +1,7 @@
 require 'bundler/setup'
 require 'logger'
 require 'json_spec'
+require 'pp' # Require pp before fakefs to fix TypeError: superclass mismatch for class File
 require 'fakefs/spec_helpers'
 require 'avro_turf'
 

data/spec/support/confluent_schema_registry_context.rb

@@ -77,6 +77,33 @@ shared_examples_for "a confluent schema registry client" do
     end
   end
 
+  describe "#schema_subject_versions" do
+    it "returns subject and version for a schema id" do
+      schema_id1 = registry.register(subject_name, { type: :record, name: "r1", fields: [] }.to_json)
+      registry.register(subject_name, { type: :record, name: "r2", fields: [] }.to_json)
+      schema_id2 = registry.register("other#{subject_name}", { type: :record, name: "r2", fields: [] }.to_json)
+      expect(registry.schema_subject_versions(schema_id1)).to eq([
+        'subject' => subject_name,
+        'version' => 1
+      ])
+      expect(registry.schema_subject_versions(schema_id2)).to include({
+        'subject' => subject_name,
+        'version' => 2
+      },{
+        'subject' => "other#{subject_name}",
+        'version' => 1
+      } )
+    end
+
+    context "when the schema does not exist" do
+      it "raises an error" do
+        expect do
+          registry.schema_subject_versions(-1)
+        end.to raise_error(Excon::Errors::NotFound)
+      end
+    end
+  end
+
   describe "#subject_versions" do
     it "lists all the versions for the subject" do
       2.times do |n|

data/spec/test/fake_confluent_schema_registry_server_spec.rb

@@ -27,14 +27,75 @@ describe FakeConfluentSchemaRegistryServer do
       expect(JSON.parse(last_response.body).fetch('id')).to eq expected_id
     end
 
-    it 'returns a different schema ID when invoked with same schema and different subject' do
+    it 'returns the same schema ID when invoked with same schema and different subject' do
       post '/subjects/person/versions', { schema: schema }.to_json, 'CONTENT_TYPE' => 'application/vnd.schemaregistry+json'
 
       original_id = JSON.parse(last_response.body).fetch('id')
 
       post '/subjects/happy-person/versions', { schema: schema }.to_json, 'CONTENT_TYPE' => 'application/vnd.schemaregistry+json'
 
-      expect(JSON.parse(last_response.body).fetch('id')).not_to eq original_id
+      expect(JSON.parse(last_response.body).fetch('id')).to eq original_id
+    end
+
+    it 'returns a different schema ID when invoked with a different schema' do
+      post '/subjects/person/versions', { schema: schema }.to_json, 'CONTENT_TYPE' => 'application/vnd.schemaregistry+json'
+
+      original_id = JSON.parse(last_response.body).fetch('id')
+
+      other_schema = {
+        type: "record",
+        name: "other",
+        fields: [
+          { name: "name", type: "string" }
+        ]
+      }.to_json
+
+      post '/subjects/person/versions', { schema: other_schema }.to_json, 'CONTENT_TYPE' => 'application/vnd.schemaregistry+json'
+
+      expect(JSON.parse(last_response.body).fetch('id')).to_not eq original_id
+    end
+  end
+
+  describe 'GET /schemas/ids/:id/versions' do
+    def schema(name:)
+      {
+        type: "record",
+        name: name,
+        fields: [
+          { name: "name", type: "string" },
+        ]
+      }.to_json
+    end
+
+    it "returns array containing subjects and versions for given schema id" do
+      schema1 = schema(name: "name1")
+      schema2 = schema(name: "name2")
+
+      post "/subjects/cats/versions", { schema: schema1 }.to_json, 'CONTENT_TYPE' => 'application/vnd.schemaregistry+json'
+      schema1_id = JSON.parse(last_response.body).fetch('id') # Original cats schema
+
+      post "/subjects/dogs/versions", { schema: schema2 }.to_json, 'CONTENT_TYPE' => 'application/vnd.schemaregistry+json'
+      post "/subjects/cats/versions", { schema: schema2 }.to_json, 'CONTENT_TYPE' => 'application/vnd.schemaregistry+json'
+      schema2_id = JSON.parse(last_response.body).fetch('id') # Changed cats schema == Original Dogs schema
+
+      get "/schemas/ids/#{schema1_id}/versions"
+      result = JSON.parse(last_response.body)
+
+      expect(result).to eq [{
+        'subject' => 'cats',
+        'version' => 1
+      }]
+
+      get "/schemas/ids/#{schema2_id}/versions"
+      result = JSON.parse(last_response.body)
+
+      expect(result).to include( {
+        'subject' => 'cats',
+        'version' => 2
+      }, {
+        'subject' => 'dogs',
+        'version' => 1
+      })
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: avro_turf
 version: !ruby/object:Gem::Version
-  version: 1.9.0
+  version: 1.11.0
 platform: ruby
 authors:
 - Daniel Schierbeck
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-06-06 00:00:00.000000000 Z
+date: 2023-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.7.7
+        version: 1.8.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '1.12'
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.7.7
+        version: 1.8.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '1.12'
@@ -156,7 +156,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
+description:
 email:
 - dasch@zendesk.com
 executables: []
@@ -251,8 +251,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3.1
-signing_key: 
+rubygems_version: 3.4.10
+signing_key:
 specification_version: 4
 summary: A library that makes it easier to use the Avro serialization format from
   Ruby