avro_turf 1.10.0 → 1.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c0907531d40226b59cd545baf7341f3e41613565d50de98684b266b6b33a081f
-  data.tar.gz: e472794d6e7ecbd4a32b1a08f4cc0eb6b0ae810d849483d86ab16a7da929a9fb
+  metadata.gz: 31248ad33908f238baea49fc106bc3e6292f45a1fe7fce0c0ae091dc10a9ea17
+  data.tar.gz: 25af934023a269b28acf004a1ec13c052380f5a664696ce881ef27efb209a7e5
 SHA512:
-  metadata.gz: ac1324149cb3299c3e67ffbd287879da88a09c338b967e2098fed63836c9fba5f775623ee6d6ae2ac0ac5e6f6a4ad28ea4c7bb7ece2d1e5979cee2eb379ac43a
-  data.tar.gz: 84709d06f4b3d97a23c1f0a1b94f4757c8b7024d17354ec65c7068be8cca52cdeed832096d7ea43f646d3f52d09ebd0f2286375456503d9da14cf6cb7a37772b
+  metadata.gz: 6e0796ce28027a654fa189f561f8179567fbebf50ba6c225be9b8623999e7ef424b58b05d1282a151474aaf48057b0000ce7999f6f8aed1c5f56f53f2d1f8cab
+  data.tar.gz: 70fb1dc693589e71691ab8bf18cef28ebe533921efe401c6c0f7c3f1ace8ac880e5e3bd1161bd3235148f7f469c837d5224f90ea96a3867aa0fe24a16de91200

data/CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## Unreleased
 
+## v1.11.0
+
+- Add `decode_all` and `decode_all_from_stream` methods to return all entries in a data file (#194)
+- Improve the way schemas are automatically loaded (#190)
+- Increment dependency on `avro` gem to v1.8.
+
 ## v1.10.0
 
 - Add `schema_subject_versions` to `ConfluentSchemaRegistry` to retrieve all subject versions for a schema id. (#189)

data/avro_turf.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "avro", ">= 1.7.7", "< 1.12"
+  spec.add_dependency "avro", ">= 1.8.0", "< 1.12"
   spec.add_dependency "excon", "~> 0.71"
 
   spec.add_development_dependency "bundler", "~> 2.0"

data/lib/avro_turf/schema_store.rb

@@ -70,28 +70,22 @@ class AvroTurf::SchemaStore
     @schemas[fullname] = schema
 
     schema
-  rescue ::Avro::SchemaParseError => e
-    # This is a hack in order to figure out exactly which type was missing. The
-    # Avro gem ought to provide this data directly.
-    if e.to_s =~ /"([\w\.]+)" is not a schema we know about/
-      # Try to first resolve a referenced schema from disk.
-      # If this is successful, the Avro gem will have mutated the
-      # local_schemas_cache, adding all the new schemas it found.
-      load_schema!($1, local_schemas_cache)
-
-      # Attempt to re-parse the original schema now that the dependency
-      # has been resolved and use the now-updated local_schemas_cache to
-      # pick up where we left off.
-      local_schemas_cache.delete(fullname)
-      # Ensure all sub-schemas are cleaned up to avoid conflicts when re-parsing
-      # schema.
-      local_schemas_cache.each do |schema_name, schema|
-        local_schemas_cache.delete(schema_name) unless File.exist?(build_schema_path(schema_name))
-      end
-      load_schema!(fullname, @schemas.dup)
-    else
-      raise
+  rescue ::Avro::UnknownSchemaError => e
+    # Try to first resolve a referenced schema from disk.
+    # If this is successful, the Avro gem will have mutated the
+    # local_schemas_cache, adding all the new schemas it found.
+    load_schema!(e.type_name, local_schemas_cache)
+
+    # Attempt to re-parse the original schema now that the dependency
+    # has been resolved and use the now-updated local_schemas_cache to
+    # pick up where we left off.
+    local_schemas_cache.delete(fullname)
+    # Ensure all sub-schemas are cleaned up to avoid conflicts when re-parsing
+    # schema.
+    local_schemas_cache.each_key do |schema_name|
+      local_schemas_cache.delete(schema_name) unless File.exist?(build_schema_path(schema_name))
     end
+    load_schema!(fullname, @schemas.dup)
   rescue Errno::ENOENT, Errno::ENAMETOOLONG
     raise AvroTurf::SchemaNotFoundError, "could not find Avro schema at `#{schema_path}'"
   end

data/lib/avro_turf/version.rb

@@ -1,3 +1,3 @@
 class AvroTurf
-  VERSION = "1.10.0"
+  VERSION = "1.11.0"
 end

data/lib/avro_turf.rb

@@ -93,24 +93,39 @@ class AvroTurf
   # namespace    - The namespace of the Avro schema used to decode the data.
   #
   # Returns whatever is encoded in the data.
-  def decode(encoded_data, schema_name: nil, namespace: @namespace)
+  def decode_first(encoded_data, schema_name: nil, namespace: @namespace)
     stream = StringIO.new(encoded_data)
     decode_stream(stream, schema_name: schema_name, namespace: namespace)
   end
 
-  # Decodes Avro data from an IO stream.
+  alias decode decode_first
+
+  # Returns all entries encoded in the data.
+  def decode_all(encoded_data, schema_name: nil, namespace: @namespace)
+    stream = StringIO.new(encoded_data)
+    decode_all_from_stream(stream, schema_name: schema_name, namespace: namespace)
+  end
+
+  # Decodes the first entry from an IO stream containing Avro data.
   #
   # stream       - An IO object containing Avro data.
   # schema_name  - The String name of the schema that should be used to read
   #                the data. If nil, the writer schema will be used.
   # namespace    - The namespace of the Avro schema used to decode the data.
   #
-  # Returns whatever is encoded in the stream.
-  def decode_stream(stream, schema_name: nil, namespace: @namespace)
+  # Returns first entry encoded in the stream.
+  def decode_first_from_stream(stream, schema_name: nil, namespace: @namespace)
+    data = decode_all_from_stream(stream, schema_name: schema_name, namespace: namespace)
+    data.first
+  end
+
+  alias decode_stream decode_first_from_stream
+
+  # Returns all entries encoded in the stream.
+  def decode_all_from_stream(stream, schema_name: nil, namespace: @namespace)
     schema = schema_name && @schema_store.find(schema_name, namespace)
     reader = Avro::IO::DatumReader.new(nil, schema)
-    dr = Avro::DataFile::Reader.new(stream, reader)
-    dr.first
+    Avro::DataFile::Reader.new(stream, reader)
   end
 
   # Validates data against an Avro schema.

data/spec/avro_turf_spec.rb

@@ -186,6 +186,47 @@ describe AvroTurf do
     end
   end
 
+  describe "#decode_all" do
+    context "when data contains multiple entries" do
+      let(:encoded_data) {  "Obj\u0001\u0004\u0014avro.codec\bnull\u0016avro.schema\xB6\u0004[{\"type\": \"record\", \"name\": \"address\", \"fields\": [{\"type\": \"string\", \"name\": \"street\"}, {\"type\": \"string\", \"name\": \"city\"}]}, {\"type\": \"record\", \"name\": \"person\", \"fields\": [{\"type\": \"string\", \"name\": \"name\"}, {\"type\": \"int\", \"name\": \"age\"}, {\"type\": \"address\", \"name\": \"address\"}]}]\u0000\xF9u\x84\xA1c\u0010\x82B\xE2\xCF\xF1\x98\xF7\xF1JH\u0004\x96\u0001\u0002\u0014Python🐍\x80\u0004\u0018Green Street\u001ASan Francisco\u0002\u0010Mojo🐍\u0002\u0016Blue Street\u0014Saturn🪐\xF9u\x84\xA1c\u0010\x82B\xE2\xCF\xF1\x98\xF7\xF1JH" } 
+
+      it "returns array of entries decoded using the inlined writer's schema " do
+        expect(avro.decode_all(encoded_data).entries).to eq(
+          [
+            {"name"=>"Python🐍", "age"=>256, "address"=>{"street"=>"Green Street", "city"=>"San Francisco"}},
+            {"name"=>"Mojo🐍", "age"=>1, "address"=>{"street"=>"Blue Street", "city"=>"Saturn🪐"}}
+          ]
+        )
+      end
+
+      it "returns array of entries decoded using the specified reader's schema " do
+        FileUtils.mkdir_p("spec/schemas/reader")
+
+        define_schema "reader/person.avsc", <<-AVSC
+          {
+            "name": "person",
+            "type": "record",
+            "fields": [
+              { "name": "fav_color", "type": "string", "default": "red🟥" },
+              { "name": "name", "type": "string" },
+              { "name": "age", "type": "int" }
+            ]
+          }
+        AVSC
+
+        expect(
+          AvroTurf.new(schemas_path: "spec/schemas/reader")
+                  .decode_all(encoded_data, schema_name: "person").entries
+        ).to eq(
+          [
+            {"name"=>"Python🐍", "age"=>256, "fav_color"=>"red🟥"},
+            {"name"=>"Mojo🐍", "age"=>1, "fav_color"=>"red🟥"}
+          ]
+        )
+      end
+    end
+  end
+
   describe "#encode_to_stream" do
     it "writes encoded data to an existing stream" do
       define_schema "message.avsc", <<-AVSC
@@ -304,6 +345,20 @@ describe AvroTurf do
     end
   end
 
+  describe "#decode_all_from_stream" do
+    it "returns all entries when decodes Avro data from a stream containing multiple entries" do
+      encoded_data = "Obj\u0001\u0004\u0014avro.codec\bnull\u0016avro.schema\xB6\u0004[{\"type\": \"record\", \"name\": \"address\", \"fields\": [{\"type\": \"string\", \"name\": \"street\"}, {\"type\": \"string\", \"name\": \"city\"}]}, {\"type\": \"record\", \"name\": \"person\", \"fields\": [{\"type\": \"string\", \"name\": \"name\"}, {\"type\": \"int\", \"name\": \"age\"}, {\"type\": \"address\", \"name\": \"address\"}]}]\u0000\xF9u\x84\xA1c\u0010\x82B\xE2\xCF\xF1\x98\xF7\xF1JH\u0004\x96\u0001\u0002\u0014Python🐍\x80\u0004\u0018Green Street\u001ASan Francisco\u0002\u0010Mojo🐍\u0002\u0016Blue Street\u0014Saturn🪐\xF9u\x84\xA1c\u0010\x82B\xE2\xCF\xF1\x98\xF7\xF1JH"
+      stream = StringIO.new(encoded_data)
+
+      expect(avro.decode_all_from_stream(stream).entries).to eq(
+        [
+          {"name"=>"Python🐍", "age"=>256, "address"=>{"street"=>"Green Street", "city"=>"San Francisco"}},
+          {"name"=>"Mojo🐍", "age"=>1, "address"=>{"street"=>"Blue Street", "city"=>"Saturn🪐"}}
+        ]
+      )
+    end
+  end
+
   describe "#valid?" do
     before do
       define_schema "message.avsc", <<-AVSC

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: avro_turf
 version: !ruby/object:Gem::Version
-  version: 1.10.0
+  version: 1.11.0
 platform: ruby
 authors:
 - Daniel Schierbeck
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-08-25 00:00:00.000000000 Z
+date: 2023-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.7.7
+        version: 1.8.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '1.12'
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.7.7
+        version: 1.8.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '1.12'
@@ -156,7 +156,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
+description:
 email:
 - dasch@zendesk.com
 executables: []
@@ -251,8 +251,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3.1
-signing_key: 
+rubygems_version: 3.4.10
+signing_key:
 specification_version: 4
 summary: A library that makes it easier to use the Avro serialization format from
   Ruby