avro_turf 1.16.0 → 1.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2345cdad9c7472c5fd79aa3710150c40095ff2b970b28ea17b3db8b951fd143f
-  data.tar.gz: 264fbec47c582106134b51fc80823f93a82c9f1053a2d61e8899eac72a27256f
+  metadata.gz: 22e1ca443d93f4f1c38b9626ad1a83db6352a4aac397256a8a3bfae2ca462d9c
+  data.tar.gz: 2621ae9378b2511cee81522247c5f4df883ee9c25efe92839cdc5af22543bc1f
 SHA512:
-  metadata.gz: 2f17112a66c063514d56c7a4a5d785fa61e1c2079b7a02eede619b1b030997ba689b897ac8fc55f355fc781a1fd48dd77c545315478155b70853a0263fd571ac
-  data.tar.gz: 9d34639b83ee204e15143cc073aacd24157556eaede70e855384ba43ec769c6ed12505ffa2cf6d17ae6e216095c0ad097ace8af42579efaec4fc34ad92c482fe
+  metadata.gz: 56f2e3885be65423da7be65fe067151b3fd270ce13af76455a0f71cccbf6d68b09661e75f882c50ea9745f9cdc21e78ce8bd527d8123fa4b0c094dfaf6baa03a
+  data.tar.gz: 4b8e92503870f8c83c8af5457786da3c83823706d957a96986488f2800166ae02846141e9eeb28c261c6b0e0f5d7922244f65dfb47e39bbc96a9a29fea075e55

data/.github/workflows/push_gem.yml

@@ -0,0 +1,24 @@
+on:
+  push:
+    branches:
+    - master
+jobs:
+  push:
+    name: Push gem to RubyGems.org
+    runs-on: ubuntu-latest
+
+    permissions:
+      id-token: write # IMPORTANT: this permission is mandatory for trusted publishing
+      contents: write # IMPORTANT: this permission is required for `rake release` to push the release tag
+
+    steps:
+      # Set up
+      - uses: actions/checkout@v4
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+          ruby-version: ruby
+
+      # Release
+      - uses: rubygems/release-gem@v1

data/CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+## v1.17.0
+
+- Add `register_schemas` option to `encode` method [#210](https://github.com/dasch/avro_turf/pull/210)
+
 ## v1.16.0
 
 - Add compatibility with Avro v1.12.x.

data/README.md

@@ -92,6 +92,9 @@ avro.encode({ "name" => "Jane", "age" => 28 }, schema_name: "person")
 # Data can be validated before encoding to get a description of problem through
 # Avro::SchemaValidator::ValidationError exception
 avro.encode({ "titl" => "hello, world" }, schema_name: "person", validate: true)
+
+# If you do not want to register the schema in case it does not exist, you can pass the register_schemas option as false
+avro.encode({ "name" => "Jane", "age" => 28 }, schema_name: "person", register_schemas: false)
 ```
 
 ### Inter-schema references

data/lib/avro_turf/cached_confluent_schema_registry.rb

@@ -17,13 +17,17 @@ class AvroTurf::CachedConfluentSchemaRegistry
   end
 
   # Delegate the following methods to the upstream
-  %i(subjects subject_versions schema_subject_versions check compatible?
+  %i(subjects subject_versions schema_subject_versions compatible?
      global_config update_global_config subject_config update_subject_config).each do |name|
     define_method(name) do |*args|
       instance_variable_get(:@upstream).send(name, *args)
     end
   end
 
+  def check(subject, schema)
+    @cache.lookup_data_by_schema(subject, schema) || @cache.store_data_by_schema(subject, schema, @upstream.check(subject, schema))
+  end
+
   def fetch(id)
     @cache.lookup_by_id(id) || @cache.store_by_id(id, @upstream.fetch(id))
   end

data/lib/avro_turf/disk_cache.rb

@@ -16,6 +16,10 @@ class AvroTurf::DiskCache
 
     @schemas_by_subject_version_path = File.join(disk_path, 'schemas_by_subject_version.json')
     @schemas_by_subject_version = {}
+
+    @data_by_schema_path = File.join(disk_path, 'data_by_schema.json')
+    hash = read_from_disk_cache(@data_by_schema_path)
+    @data_by_schema = hash || {}
   end
 
   # override
@@ -40,6 +44,12 @@ class AvroTurf::DiskCache
     @ids_by_schema[key]
   end
 
+  # override to use a json serializable cache key
+  def lookup_data_by_schema(subject, schema)
+    key = "#{subject}#{schema}"
+    @data_by_schema[key]
+  end
+
   # override to use a json serializable cache key and update the file cache
   def store_by_schema(subject, schema, id)
     key = "#{subject}#{schema}"
@@ -49,6 +59,15 @@ class AvroTurf::DiskCache
     id
   end
 
+  def store_data_by_schema(subject, schema, data)
+    return unless data
+
+    key = "#{subject}#{schema}"
+    @data_by_schema[key] = data
+    write_to_disk_cache(@data_by_schema_path, @data_by_schema)
+    data
+  end
+
   # checks instance var (in-memory cache) for schema
   # checks disk cache if in-memory cache doesn't exists
   # if file exists but no in-memory cache, read from file and sync in-memory cache

data/lib/avro_turf/in_memory_cache.rb

@@ -1,11 +1,11 @@
 # A cache for the CachedConfluentSchemaRegistry.
 # Simply stores the schemas and ids in in-memory hashes.
 class AvroTurf::InMemoryCache
-
   def initialize
     @schemas_by_id = {}
     @ids_by_schema = {}
     @schema_by_subject_version = {}
+    @data_by_schema = {}
   end
 
   def lookup_by_id(id)
@@ -21,11 +21,23 @@ class AvroTurf::InMemoryCache
     @ids_by_schema[key]
   end
 
+  def lookup_data_by_schema(subject, schema)
+    key = [subject, schema]
+    @data_by_schema[key]
+  end
+
   def store_by_schema(subject, schema, id)
     key = [subject, schema]
     @ids_by_schema[key] = id
   end
 
+  def store_data_by_schema(subject, schema, data)
+    return unless data
+
+    key = [subject, schema]
+    @data_by_schema[key] = data
+  end
+
   def lookup_by_version(subject, version)
     key = "#{subject}#{version}"
     @schema_by_subject_version[key]

data/lib/avro_turf/messaging.rb

@@ -106,27 +106,33 @@ class AvroTurf
 
     # Encodes a message using the specified schema.
     #
-    # message     - The message that should be encoded. Must be compatible with
-    #               the schema.
-    # schema_name - The String name of the schema that should be used to encode
-    #               the data.
-    # namespace   - The namespace of the schema (optional).
-    # subject     - The subject name the schema should be registered under in
-    #               the schema registry (optional).
-    # version     - The integer version of the schema that should be used to decode
-    #               the data. Must match the schema used when encoding (optional).
-    # schema_id   - The integer id of the schema that should be used to encode
-    #               the data.
-    # validate    - The boolean for performing complete message validation before
-    #               encoding it, Avro::SchemaValidator::ValidationError with
-    #               a descriptive message will be raised in case of invalid message.
+    # message           - The message that should be encoded. Must be compatible with
+    #                     the schema.
+    # schema_name       - The String name of the schema that should be used to encode
+    #                     the data.
+    # namespace         - The namespace of the schema (optional).
+    # subject           - The subject name the schema should be registered under in
+    #                     the schema registry (optional).
+    # version           - The integer version of the schema that should be used to decode
+    #                     the data. Must match the schema used when encoding (optional).
+    # schema_id         - The integer id of the schema that should be used to encode
+    #                     the data.
+    # validate          - The boolean for performing complete message validation before
+    #                     encoding it, Avro::SchemaValidator::ValidationError with
+    #                     a descriptive message will be raised in case of invalid message.
+    # register_schemas  - The boolean that indicates whether or not the schema should be
+    #                     registered in case it does not exist, or if it should be fetched
+    #                     from the registry without registering it (register_schemas: false).
     #
     # Returns the encoded data as a String.
-    def encode(message, schema_name: nil, namespace: @namespace, subject: nil, version: nil, schema_id: nil, validate: false)
+    def encode(message, schema_name: nil, namespace: @namespace, subject: nil, version: nil, schema_id: nil, validate: false,
+               register_schemas: true)
       schema, schema_id = if schema_id
         fetch_schema_by_id(schema_id)
       elsif subject && version
         fetch_schema(subject: subject, version: version)
+      elsif schema_name && !register_schemas
+        fetch_schema_by_body(subject: subject, schema_name: schema_name, namespace: namespace)
       elsif schema_name
         register_schema(subject: subject, schema_name: schema_name, namespace: namespace)
       else
@@ -228,6 +234,14 @@ class AvroTurf
       [schema, schema_id]
     end
 
+    def fetch_schema_by_body(schema_name:, subject: nil, namespace: nil)
+      schema = @schema_store.find(schema_name, namespace)
+      schema_data = @registry.check(subject || schema.fullname, schema)
+      raise SchemaNotFoundError.new("Schema with structure: #{schema} not found on registry") unless schema_data
+
+      [schema, schema_data["id"]]
+    end
+
     # Schemas are registered under the full name of the top level Avro record
     # type, or `subject` if it's provided.
     def register_schema(schema_name:, subject: nil, namespace: nil)

data/lib/avro_turf/test/fake_prefixed_confluent_schema_registry_server.rb

@@ -64,7 +64,8 @@ class FakePrefixedConfluentSchemaRegistryServer < FakeConfluentSchemaRegistrySer
 
     # Note: this does not actually handle the same schema registered under
     # multiple subjects
-    schema_id = SCHEMAS.index(schema)
+    context, _subject = parse_qualified_subject(params[:subject])
+    schema_id = SCHEMAS[context].index(schema)
 
     halt(404, SCHEMA_NOT_FOUND) unless schema_id
 

data/lib/avro_turf/version.rb

@@ -1,3 +1,3 @@
 class AvroTurf
-  VERSION = "1.16.0"
+  VERSION = "1.17.0"
 end

data/spec/cached_confluent_schema_registry_spec.rb

@@ -6,6 +6,7 @@ describe AvroTurf::CachedConfluentSchemaRegistry do
   let(:upstream) { instance_double(AvroTurf::ConfluentSchemaRegistry) }
   let(:registry) { described_class.new(upstream) }
   let(:id) { rand(999) }
+  let(:subject_name) { 'a_subject' }
   let(:schema) do
     {
       type: "record",
@@ -25,8 +26,6 @@ describe AvroTurf::CachedConfluentSchemaRegistry do
   end
 
   describe "#register" do
-    let(:subject_name) { "a_subject" }
-
     it "caches the result of register" do
       # multiple calls return same result, with only one upstream call
       allow(upstream).to receive(:register).with(subject_name, schema).and_return(id)
@@ -36,8 +35,29 @@ describe AvroTurf::CachedConfluentSchemaRegistry do
     end
   end
 
+  describe "#check" do
+    let(:schema_data) do
+      {
+        "subject" => subject_name,
+        "version" => 123,
+        "id" => id,
+        "schema" => schema
+      }
+    end
+
+    before do
+      allow(upstream).to receive(:check).with(subject_name, schema).and_return(schema_data)
+    end
+
+    it "caches the result of check" do
+      # multiple calls return same result, with only one upstream call
+      expect(registry.check(subject_name, schema)).to eq(schema_data)
+      expect(registry.check(subject_name, schema)).to eq(schema_data)
+      expect(upstream).to have_received(:check).exactly(1).times
+    end
+  end
+
   describe '#subject_version' do
-    let(:subject_name) { 'a_subject' }
     let(:version) { 1 }
     let(:schema_with_meta) do
       {

data/spec/disk_cached_confluent_schema_registry_spec.rb

@@ -222,6 +222,66 @@ describe AvroTurf::CachedConfluentSchemaRegistry do
     end
   end
 
+  describe "#check" do
+    let(:city_name) { "a_city" }
+    let(:schema_data) do
+      {
+        "subject" => subject,
+        "version" => version,
+        "id" => id,
+        "schema" => schema
+      }
+    end
+
+    let(:city_schema_data) do
+      {
+        "subject" => city_name,
+        "version" => version,
+        "id" => city_id,
+        "schema" => city_schema
+      }
+    end
+
+    let(:cache_before) do
+      {
+        "#{subject}#{schema}" => schema_data
+      }
+    end
+
+    let(:cache_after) do
+      {
+        "#{subject}#{schema}" => schema_data,
+        "#{city_name}#{city_schema}" => city_schema_data
+      }
+    end
+
+    # setup the disk cache to avoid performing the upstream fetch
+    before do
+      store_cache("data_by_schema.json", cache_before)
+      allow(upstream).to receive(:check).with(subject, schema).and_return(schema_data)
+      allow(upstream).to receive(:check).with(city_name, city_schema).and_return(city_schema_data)
+    end
+
+    context "when the schema is not found in the cache" do
+      it "makes only one request using upstream" do
+        expect(registry.check(city_name, city_schema)).to eq(city_schema_data)
+        expect(registry.check(city_name, city_schema)).to eq(city_schema_data)
+        expect(upstream).to have_received(:check).with(city_name, city_schema).exactly(1).times
+        expect(load_cache("data_by_schema.json")).to eq cache_after
+      end
+    end
+
+    context "when schema is already in the cache" do
+      it "uses preloaded disk cache" do
+        # multiple calls return same result, with zero upstream calls
+        expect(registry.check(subject, schema)).to eq(schema_data)
+        expect(registry.check(subject, schema)).to eq(schema_data)
+        expect(upstream).to have_received(:check).exactly(0).times
+        expect(load_cache("data_by_schema.json")).to eq cache_before
+      end
+    end
+  end
+
   it_behaves_like "a confluent schema registry client" do
     let(:upstream) { AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger) }
     let(:registry) { described_class.new(upstream) }

data/spec/messaging_spec.rb

@@ -36,6 +36,24 @@ describe AvroTurf::Messaging do
       }
     AVSC
   end
+
+  let(:city_message) { { "name" => "Paris" } }
+  let(:city_schema_json) do
+    <<-AVSC
+      {
+        "name": "city",
+        "type": "record",
+        "fields": [
+          {
+            "type": "string",
+            "name": "name"
+          }
+        ]
+      }
+    AVSC
+  end
+
+  let(:city_schema) { Avro::Schema.parse(city_schema_json) }
   let(:schema) { Avro::Schema.parse(schema_json) }
 
   before do
@@ -49,6 +67,7 @@ describe AvroTurf::Messaging do
 
   before do
     define_schema "person.avsc", schema_json
+    define_schema "city.avsc", city_schema_json
   end
 
   shared_examples_for "encoding and decoding with the schema from schema store" do
@@ -92,6 +111,16 @@ describe AvroTurf::Messaging do
       expect { avro.encode(message, subject: 'missing', version: 1) }.to raise_error(AvroTurf::SchemaNotFoundError)
     end
 
+    it 'raises AvroTurf::SchemaNotFoundError when the schema does not exist on registry and register_schemas false' do
+      expect { avro.encode(city_message, schema_name: 'city', register_schemas: false) }.
+        to raise_error(AvroTurf::SchemaNotFoundError, "Schema with structure: #{city_schema} not found on registry")
+    end
+
+    it 'encodes with register_schemas false when the schema exists on the registry' do
+      data = avro.encode(message, schema_name: 'person', register_schemas: false)
+      expect(avro.decode(data, schema_name: 'person')).to eq message
+    end
+
     it 'caches parsed schemas for decoding' do
       data = avro.encode(message, subject: 'person', version: 1)
       avro.decode(data)
@@ -364,6 +393,34 @@ describe AvroTurf::Messaging do
       end
     end
 
+    context 'using fetch_schema_by_body' do
+      let(:subject_name) { 'city' }
+      let(:schema_name) { 'city' }
+      let(:namespace) { 'namespace' }
+      let(:city_schema_id) { 125 }
+      let(:city_schema_data) do
+        {
+          "subject" => subject_name,
+          "version" => 123,
+          "id" => city_schema_id,
+          "schema" => city_schema
+        }
+      end
+
+      subject(:fetch_schema_by_body) do
+        avro.fetch_schema_by_body(schema_name: schema_name, namespace: namespace, subject: subject_name)
+      end
+
+      before do
+        allow(schema_store).to receive(:find).with(schema_name, namespace).and_return(city_schema)
+        allow(registry).to receive(:check).with(subject_name, city_schema).and_return(city_schema_data)
+      end
+
+      it 'gets schema from registry' do
+        expect(fetch_schema_by_body).to eq([city_schema, city_schema_id])
+      end
+    end
+
     context 'using register_schema' do
       let(:schema_name) { 'schema_name' }
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: avro_turf
 version: !ruby/object:Gem::Version
-  version: 1.16.0
+  version: 1.17.0
 platform: ruby
 authors:
 - Daniel Schierbeck
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-08-12 00:00:00.000000000 Z
+date: 2024-08-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro
@@ -170,13 +170,14 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description:
+description: 
 email:
 - dasch@zendesk.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/push_gem.yml"
 - ".github/workflows/ruby.yml"
 - ".github/workflows/stale.yml"
 - ".gitignore"
@@ -265,8 +266,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
-signing_key:
+rubygems_version: 3.5.11
+signing_key: 
 specification_version: 4
 summary: A library that makes it easier to use the Avro serialization format from
   Ruby