avro_turf 1.10.0 → 1.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c0907531d40226b59cd545baf7341f3e41613565d50de98684b266b6b33a081f
-  data.tar.gz: e472794d6e7ecbd4a32b1a08f4cc0eb6b0ae810d849483d86ab16a7da929a9fb
+  metadata.gz: 94e8abd646816e6e375365c50ed19eebb3bfd4722548aba30fca974740fd7788
+  data.tar.gz: 18bba7e20c8ac10b52177af05db7b4ea6b461f696f4230c7086d1b5e3a7479df
 SHA512:
-  metadata.gz: ac1324149cb3299c3e67ffbd287879da88a09c338b967e2098fed63836c9fba5f775623ee6d6ae2ac0ac5e6f6a4ad28ea4c7bb7ece2d1e5979cee2eb379ac43a
-  data.tar.gz: 84709d06f4b3d97a23c1f0a1b94f4757c8b7024d17354ec65c7068be8cca52cdeed832096d7ea43f646d3f52d09ebd0f2286375456503d9da14cf6cb7a37772b
+  metadata.gz: 8b2b977f70cdc2c497a2cf1449d9f84bf11a6d7ce62022fd9295a58ea1133fe10674431dbf765d8676ba404e54c73f5bc35016868fd6e3ce2cbef4ea5027de07
+  data.tar.gz: a3ad6d0518e93b12a5371ac338bcb99e7f7acd22728d2527c16a57888e8fe39567d1c8bc57060451cf2b536c653e8cb470809e76381e2ec07a0c3b8cbaefe734

data/CHANGELOG.md

@@ -2,6 +2,16 @@
 
 ## Unreleased
 
+## v1.12.0
+
+- Add `connect_timeout` parameter to `AvroTurf::Messaging` to set the timeout for the connection to the schema registry (#197)
+
+## v1.11.0
+
+- Add `decode_all` and `decode_all_from_stream` methods to return all entries in a data file (#194)
+- Improve the way schemas are automatically loaded (#190)
+- Increment dependency on `avro` gem to v1.8.
+
 ## v1.10.0
 
 - Add `schema_subject_versions` to `ConfluentSchemaRegistry` to retrieve all subject versions for a schema id. (#189)

data/avro_turf.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "avro", ">= 1.7.7", "< 1.12"
+  spec.add_dependency "avro", ">= 1.8.0", "< 1.12"
   spec.add_dependency "excon", "~> 0.71"
 
   spec.add_development_dependency "bundler", "~> 2.0"

data/lib/avro_turf/confluent_schema_registry.rb

@@ -15,7 +15,8 @@ class AvroTurf::ConfluentSchemaRegistry
     client_key_pass: nil,
     client_cert_data: nil,
     client_key_data: nil,
-    path_prefix: nil
+    path_prefix: nil,
+    connect_timeout: nil
   )
     @path_prefix = path_prefix
     @logger = logger
@@ -33,7 +34,8 @@ class AvroTurf::ConfluentSchemaRegistry
       client_key: client_key,
       client_key_pass: client_key_pass,
       client_cert_data: client_cert_data,
-      client_key_data: client_key_data
+      client_key_data: client_key_data,
+      connect_timeout: connect_timeout
     )
   end
 

data/lib/avro_turf/messaging.rb

@@ -55,6 +55,7 @@ class AvroTurf
     # client_key_pass      - Password to go with client_key (optional).
     # client_cert_data     - In-memory client certificate (optional).
     # client_key_data      - In-memory client private key to go with client_cert_data (optional).
+    # connect_timeout      - Timeout to use in the connection with the schema registry (optional).
     def initialize(
       registry: nil,
       registry_url: nil,
@@ -71,7 +72,8 @@ class AvroTurf
       client_key: nil,
       client_key_pass: nil,
       client_cert_data: nil,
-      client_key_data: nil
+      client_key_data: nil,
+      connect_timeout: nil
     )
       @logger = logger || Logger.new($stderr)
       @namespace = namespace
@@ -89,7 +91,8 @@ class AvroTurf
           client_key_pass: client_key_pass,
           client_cert_data: client_cert_data,
           client_key_data: client_key_data,
-          path_prefix: registry_path_prefix
+          path_prefix: registry_path_prefix,
+          connect_timeout: connect_timeout
         )
       )
       @schemas_by_id = {}

data/lib/avro_turf/schema_store.rb

@@ -70,28 +70,22 @@ class AvroTurf::SchemaStore
     @schemas[fullname] = schema
 
     schema
-  rescue ::Avro::SchemaParseError => e
-    # This is a hack in order to figure out exactly which type was missing. The
-    # Avro gem ought to provide this data directly.
-    if e.to_s =~ /"([\w\.]+)" is not a schema we know about/
-      # Try to first resolve a referenced schema from disk.
-      # If this is successful, the Avro gem will have mutated the
-      # local_schemas_cache, adding all the new schemas it found.
-      load_schema!($1, local_schemas_cache)
-
-      # Attempt to re-parse the original schema now that the dependency
-      # has been resolved and use the now-updated local_schemas_cache to
-      # pick up where we left off.
-      local_schemas_cache.delete(fullname)
-      # Ensure all sub-schemas are cleaned up to avoid conflicts when re-parsing
-      # schema.
-      local_schemas_cache.each do |schema_name, schema|
-        local_schemas_cache.delete(schema_name) unless File.exist?(build_schema_path(schema_name))
-      end
-      load_schema!(fullname, @schemas.dup)
-    else
-      raise
+  rescue ::Avro::UnknownSchemaError => e
+    # Try to first resolve a referenced schema from disk.
+    # If this is successful, the Avro gem will have mutated the
+    # local_schemas_cache, adding all the new schemas it found.
+    load_schema!(e.type_name, local_schemas_cache)
+
+    # Attempt to re-parse the original schema now that the dependency
+    # has been resolved and use the now-updated local_schemas_cache to
+    # pick up where we left off.
+    local_schemas_cache.delete(fullname)
+    # Ensure all sub-schemas are cleaned up to avoid conflicts when re-parsing
+    # schema.
+    local_schemas_cache.each_key do |schema_name|
+      local_schemas_cache.delete(schema_name) unless File.exist?(build_schema_path(schema_name))
     end
+    load_schema!(fullname, @schemas.dup)
   rescue Errno::ENOENT, Errno::ENAMETOOLONG
     raise AvroTurf::SchemaNotFoundError, "could not find Avro schema at `#{schema_path}'"
   end

data/lib/avro_turf/version.rb

@@ -1,3 +1,3 @@
 class AvroTurf
-  VERSION = "1.10.0"
+  VERSION = "1.12.0"
 end

data/lib/avro_turf.rb

@@ -93,24 +93,39 @@ class AvroTurf
   # namespace    - The namespace of the Avro schema used to decode the data.
   #
   # Returns whatever is encoded in the data.
-  def decode(encoded_data, schema_name: nil, namespace: @namespace)
+  def decode_first(encoded_data, schema_name: nil, namespace: @namespace)
     stream = StringIO.new(encoded_data)
     decode_stream(stream, schema_name: schema_name, namespace: namespace)
   end
 
-  # Decodes Avro data from an IO stream.
+  alias decode decode_first
+
+  # Returns all entries encoded in the data.
+  def decode_all(encoded_data, schema_name: nil, namespace: @namespace)
+    stream = StringIO.new(encoded_data)
+    decode_all_from_stream(stream, schema_name: schema_name, namespace: namespace)
+  end
+
+  # Decodes the first entry from an IO stream containing Avro data.
   #
   # stream       - An IO object containing Avro data.
   # schema_name  - The String name of the schema that should be used to read
   #                the data. If nil, the writer schema will be used.
   # namespace    - The namespace of the Avro schema used to decode the data.
   #
-  # Returns whatever is encoded in the stream.
-  def decode_stream(stream, schema_name: nil, namespace: @namespace)
+  # Returns first entry encoded in the stream.
+  def decode_first_from_stream(stream, schema_name: nil, namespace: @namespace)
+    data = decode_all_from_stream(stream, schema_name: schema_name, namespace: namespace)
+    data.first
+  end
+
+  alias decode_stream decode_first_from_stream
+
+  # Returns all entries encoded in the stream.
+  def decode_all_from_stream(stream, schema_name: nil, namespace: @namespace)
     schema = schema_name && @schema_store.find(schema_name, namespace)
     reader = Avro::IO::DatumReader.new(nil, schema)
-    dr = Avro::DataFile::Reader.new(stream, reader)
-    dr.first
+    Avro::DataFile::Reader.new(stream, reader)
   end
 
   # Validates data against an Avro schema.

data/spec/avro_turf_spec.rb

@@ -186,6 +186,47 @@ describe AvroTurf do
     end
   end
 
+  describe "#decode_all" do
+    context "when data contains multiple entries" do
+      let(:encoded_data) {  "Obj\u0001\u0004\u0014avro.codec\bnull\u0016avro.schema\xB6\u0004[{\"type\": \"record\", \"name\": \"address\", \"fields\": [{\"type\": \"string\", \"name\": \"street\"}, {\"type\": \"string\", \"name\": \"city\"}]}, {\"type\": \"record\", \"name\": \"person\", \"fields\": [{\"type\": \"string\", \"name\": \"name\"}, {\"type\": \"int\", \"name\": \"age\"}, {\"type\": \"address\", \"name\": \"address\"}]}]\u0000\xF9u\x84\xA1c\u0010\x82B\xE2\xCF\xF1\x98\xF7\xF1JH\u0004\x96\u0001\u0002\u0014Python🐍\x80\u0004\u0018Green Street\u001ASan Francisco\u0002\u0010Mojo🐍\u0002\u0016Blue Street\u0014Saturn🪐\xF9u\x84\xA1c\u0010\x82B\xE2\xCF\xF1\x98\xF7\xF1JH" } 
+
+      it "returns array of entries decoded using the inlined writer's schema " do
+        expect(avro.decode_all(encoded_data).entries).to eq(
+          [
+            {"name"=>"Python🐍", "age"=>256, "address"=>{"street"=>"Green Street", "city"=>"San Francisco"}},
+            {"name"=>"Mojo🐍", "age"=>1, "address"=>{"street"=>"Blue Street", "city"=>"Saturn🪐"}}
+          ]
+        )
+      end
+
+      it "returns array of entries decoded using the specified reader's schema " do
+        FileUtils.mkdir_p("spec/schemas/reader")
+
+        define_schema "reader/person.avsc", <<-AVSC
+          {
+            "name": "person",
+            "type": "record",
+            "fields": [
+              { "name": "fav_color", "type": "string", "default": "red🟥" },
+              { "name": "name", "type": "string" },
+              { "name": "age", "type": "int" }
+            ]
+          }
+        AVSC
+
+        expect(
+          AvroTurf.new(schemas_path: "spec/schemas/reader")
+                  .decode_all(encoded_data, schema_name: "person").entries
+        ).to eq(
+          [
+            {"name"=>"Python🐍", "age"=>256, "fav_color"=>"red🟥"},
+            {"name"=>"Mojo🐍", "age"=>1, "fav_color"=>"red🟥"}
+          ]
+        )
+      end
+    end
+  end
+
   describe "#encode_to_stream" do
     it "writes encoded data to an existing stream" do
       define_schema "message.avsc", <<-AVSC
@@ -304,6 +345,20 @@ describe AvroTurf do
     end
   end
 
+  describe "#decode_all_from_stream" do
+    it "returns all entries when decodes Avro data from a stream containing multiple entries" do
+      encoded_data = "Obj\u0001\u0004\u0014avro.codec\bnull\u0016avro.schema\xB6\u0004[{\"type\": \"record\", \"name\": \"address\", \"fields\": [{\"type\": \"string\", \"name\": \"street\"}, {\"type\": \"string\", \"name\": \"city\"}]}, {\"type\": \"record\", \"name\": \"person\", \"fields\": [{\"type\": \"string\", \"name\": \"name\"}, {\"type\": \"int\", \"name\": \"age\"}, {\"type\": \"address\", \"name\": \"address\"}]}]\u0000\xF9u\x84\xA1c\u0010\x82B\xE2\xCF\xF1\x98\xF7\xF1JH\u0004\x96\u0001\u0002\u0014Python🐍\x80\u0004\u0018Green Street\u001ASan Francisco\u0002\u0010Mojo🐍\u0002\u0016Blue Street\u0014Saturn🪐\xF9u\x84\xA1c\u0010\x82B\xE2\xCF\xF1\x98\xF7\xF1JH"
+      stream = StringIO.new(encoded_data)
+
+      expect(avro.decode_all_from_stream(stream).entries).to eq(
+        [
+          {"name"=>"Python🐍", "age"=>256, "address"=>{"street"=>"Green Street", "city"=>"San Francisco"}},
+          {"name"=>"Mojo🐍", "age"=>1, "address"=>{"street"=>"Blue Street", "city"=>"Saturn🪐"}}
+        ]
+      )
+    end
+  end
+
   describe "#valid?" do
     before do
       define_schema "message.avsc", <<-AVSC

data/spec/confluent_schema_registry_spec.rb

@@ -8,6 +8,7 @@ describe AvroTurf::ConfluentSchemaRegistry do
   let(:client_cert) { "test client cert" }
   let(:client_key) { "test client key" }
   let(:client_key_pass) { "test client key password" }
+  let(:connect_timeout) { 10 }
 
   it_behaves_like "a confluent schema registry client" do
     let(:registry) {
@@ -30,4 +31,15 @@ describe AvroTurf::ConfluentSchemaRegistry do
       )
     }
   end
+
+  it_behaves_like "a confluent schema registry client" do
+    let(:registry) {
+      described_class.new(
+        registry_url,
+        user: user,
+        password: password,
+        connect_timeout: connect_timeout
+      )
+    }
+  end
 end

data/spec/messaging_spec.rb

@@ -425,4 +425,27 @@ describe AvroTurf::Messaging do
     it_behaves_like 'encoding and decoding with the schema from registry'
     it_behaves_like 'encoding and decoding with the schema_id from registry'
   end
-end
+
+  context 'with a connect timeout' do
+    let(:avro) {
+      AvroTurf::Messaging.new(
+        registry_url: registry_url,
+        schemas_path: "spec/schemas",
+        logger: logger,
+        client_cert: client_cert,
+        client_key: client_key,
+        client_key_pass: client_key_pass,
+        connect_timeout: 10
+      )
+    }
+
+    it_behaves_like "encoding and decoding with the schema from schema store"
+    it_behaves_like 'encoding and decoding with the schema from registry'
+    it_behaves_like 'encoding and decoding with the schema_id from registry'
+
+    it 'passes the connect timeout setting to Excon' do
+      expect(Excon).to receive(:new).with(anything, hash_including(connect_timeout: 10)).and_call_original
+      avro
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: avro_turf
 version: !ruby/object:Gem::Version
-  version: 1.10.0
+  version: 1.12.0
 platform: ruby
 authors:
 - Daniel Schierbeck
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-08-25 00:00:00.000000000 Z
+date: 2023-09-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.7.7
+        version: 1.8.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '1.12'
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.7.7
+        version: 1.8.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '1.12'
@@ -156,7 +156,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
+description:
 email:
 - dasch@zendesk.com
 executables: []
@@ -251,8 +251,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3.1
-signing_key: 
+rubygems_version: 3.4.10
+signing_key:
 specification_version: 4
 summary: A library that makes it easier to use the Avro serialization format from
   Ruby