avro_turf 1.4.1 → 1.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0d201b8d402f0d88f3ce125040de4e056d0125607b6988236ff076b3fd042569
-  data.tar.gz: 0c1cbad2263b35f56dc04ca2def7a9ec520a3c95952acbc27f544a1c96337de8
+  metadata.gz: 3eea429db02f7f6eba5f13f46b2bf3e82602371ef9306e374c1102561f508675
+  data.tar.gz: d798ef10b0e674f48ffdcc6bd3b87e59d9447c985d539f2ba01164e689fe4f8a
 SHA512:
-  metadata.gz: 3af9af40e690c6033afb5183d9438ff0785e09a8ac2288873df20f7f50cfd4c439d05dd227d5655e91e490630f4cb677b9ef167be66af072816a07bd2ef1d7a6
-  data.tar.gz: 0d440453c9c8ac12aa3cfc07b9348de674e29fbed65809cbf47956468d2c28b62223f213ffa1f1db2726a96cc0707be98e4cb3cc4bfbb0b4229a7982d529a235
+  metadata.gz: e7cacb60d4e7890a700ebaba9a0cf1d46606fb01157e0746a0d6a6c900895e9ca1268f0a04dd4d03f5d1a9cec9bf6795d9c5d70acb6884965353c72bb1de4095
+  data.tar.gz: c4dbd604b66916355a6cba44ec4e7225eb439c2d9bb0682e62414b6ffa3018bb1d2a84b78d943983c99d7af3897a67658db9cb2205f7d92d6c13661ae44ba602

data/CHANGELOG.md

@@ -2,6 +2,20 @@
 
 ## Unreleased
 
+## v1.7.0
+
+- Added extra params for the validation message schem before encode (#169)
+- Fix infinite retry when loading schema with nested primary type in separate file (#165)
+
+## v1.6.0
+
+- Schema registry path prefix (#162)
+
+## v1.5.0
+
+- Add CA cert file option (#157)
+- Add compatibility with Avro v1.11.x.
+
 ## v1.4.1
 
 - Purge sub-schemas from cache before re-parsing schema (#151)

data/avro_turf.gemspec

@@ -12,12 +12,14 @@ Gem::Specification.new do |spec|
   spec.homepage      = "https://github.com/dasch/avro_turf"
   spec.license       = "MIT"
 
+  spec.metadata["rubygems_mfa_required"] = "true"
+
   spec.files         = `git ls-files -z`.split("\x0")
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "avro", ">= 1.7.7", "< 1.11"
+  spec.add_dependency "avro", ">= 1.7.7", "< 1.12"
   spec.add_dependency "excon", "~> 0.71"
 
   spec.add_development_dependency "bundler", "~> 2.0"

data/lib/avro_turf/confluent_schema_registry.rb

@@ -9,12 +9,15 @@ class AvroTurf::ConfluentSchemaRegistry
     proxy: nil,
     user: nil,
     password: nil,
+    ssl_ca_file: nil,
     client_cert: nil,
     client_key: nil,
     client_key_pass: nil,
     client_cert_data: nil,
-    client_key_data: nil
+    client_key_data: nil,
+    path_prefix: nil
   )
+    @path_prefix = path_prefix
     @logger = logger
     headers = {
       "Content-Type" => CONTENT_TYPE
@@ -25,6 +28,7 @@ class AvroTurf::ConfluentSchemaRegistry
       headers: headers,
       user: user,
       password: password,
+      ssl_ca_file: ssl_ca_file,
       client_cert: client_cert,
       client_key: client_key,
       client_key_pass: client_key_pass,
@@ -120,6 +124,7 @@ class AvroTurf::ConfluentSchemaRegistry
 
   def request(path, **options)
     options = { expects: 200 }.merge!(options)
+    path = File.join(@path_prefix, path) unless @path_prefix.nil?
     response = @connection.request(path: path, **options)
     JSON.parse(response.body)
   end

data/lib/avro_turf/messaging.rb

@@ -26,31 +26,35 @@ class AvroTurf
 
     # Instantiate a new Messaging instance with the given configuration.
     #
-    # registry          - A schema registry object that responds to all methods in the
-    #                     AvroTurf::ConfluentSchemaRegistry interface.
-    # registry_url      - The String URL of the schema registry that should be used.
-    # schema_store      - A schema store object that responds to #find(schema_name, namespace).
-    # schemas_path      - The String file system path where local schemas are stored.
-    # namespace         - The String default schema namespace.
-    # logger            - The Logger that should be used to log information (optional).
-    # proxy             - Forward the request via  proxy (optional).
-    # user              - User for basic auth (optional).
-    # password          - Password for basic auth (optional).
-    # client_cert       - Name of file containing client certificate (optional).
-    # client_key        - Name of file containing client private key to go with client_cert (optional).
-    # client_key_pass   - Password to go with client_key (optional).
-    # client_cert_data  - In-memory client certificate (optional).
-    # client_key_data   - In-memory client private key to go with client_cert_data (optional).
+    # registry             - A schema registry object that responds to all methods in the
+    #                        AvroTurf::ConfluentSchemaRegistry interface.
+    # registry_url         - The String URL of the schema registry that should be used.
+    # schema_store         - A schema store object that responds to #find(schema_name, namespace).
+    # schemas_path         - The String file system path where local schemas are stored.
+    # namespace            - The String default schema namespace.
+    # registry_path_prefix - The String URL path prefix used to namespace schema registry requests (optional).
+    # logger               - The Logger that should be used to log information (optional).
+    # proxy                - Forward the request via  proxy (optional).
+    # user                 - User for basic auth (optional).
+    # password             - Password for basic auth (optional).
+    # ssl_ca_file          - Name of file containing CA certificate (optional).
+    # client_cert          - Name of file containing client certificate (optional).
+    # client_key           - Name of file containing client private key to go with client_cert (optional).
+    # client_key_pass      - Password to go with client_key (optional).
+    # client_cert_data     - In-memory client certificate (optional).
+    # client_key_data      - In-memory client private key to go with client_cert_data (optional).
     def initialize(
       registry: nil,
       registry_url: nil,
       schema_store: nil,
       schemas_path: nil,
       namespace: nil,
+      registry_path_prefix: nil,
       logger: nil,
       proxy: nil,
       user: nil,
       password: nil,
+      ssl_ca_file: nil,
       client_cert: nil,
       client_key: nil,
       client_key_pass: nil,
@@ -67,11 +71,13 @@ class AvroTurf
           proxy: proxy,
           user: user,
           password: password,
+          ssl_ca_file: ssl_ca_file,
           client_cert: client_cert,
           client_key: client_key,
           client_key_pass: client_key_pass,
           client_cert_data: client_cert_data,
-          client_key_data: client_key_data
+          client_key_data: client_key_data,
+          path_prefix: registry_path_prefix
         )
       )
       @schemas_by_id = {}

data/lib/avro_turf/schema_store.rb

@@ -88,7 +88,7 @@ class AvroTurf::SchemaStore
       local_schemas_cache.each do |schema_name, schema|
         local_schemas_cache.delete(schema_name) unless File.exist?(build_schema_path(schema_name))
       end
-      load_schema!(fullname, local_schemas_cache)
+      load_schema!(fullname, @schemas.dup)
     else
       raise
     end

data/lib/avro_turf/test/fake_prefixed_confluent_schema_registry_server.rb

@@ -0,0 +1,102 @@
+require 'sinatra/base'
+
+class FakePrefixedConfluentSchemaRegistryServer < FakeConfluentSchemaRegistryServer
+  post "/prefix/subjects/:subject/versions" do
+    schema = parse_schema
+    ids_for_subject = SUBJECTS[params[:subject]]
+
+    schemas_for_subject =
+      SCHEMAS.select
+             .with_index { |_, i| ids_for_subject.include?(i) }
+
+    if schemas_for_subject.include?(schema)
+      schema_id = SCHEMAS.index(schema)
+    else
+      SCHEMAS << schema
+      schema_id = SCHEMAS.size - 1
+      SUBJECTS[params[:subject]] = SUBJECTS[params[:subject]] << schema_id
+    end
+
+    { id: schema_id }.to_json
+  end
+
+  get "/prefix/schemas/ids/:schema_id" do
+    schema = SCHEMAS.at(params[:schema_id].to_i)
+    halt(404, SCHEMA_NOT_FOUND) unless schema
+    { schema: schema }.to_json
+  end
+
+  get "/prefix/subjects" do
+    SUBJECTS.keys.to_json
+  end
+
+  get "/prefix/subjects/:subject/versions" do
+    schema_ids = SUBJECTS[params[:subject]]
+    halt(404, SUBJECT_NOT_FOUND) if schema_ids.empty?
+    (1..schema_ids.size).to_a.to_json
+  end
+
+  get "/prefix/subjects/:subject/versions/:version" do
+    schema_ids = SUBJECTS[params[:subject]]
+    halt(404, SUBJECT_NOT_FOUND) if schema_ids.empty?
+
+    schema_id = if params[:version] == 'latest'
+                  schema_ids.last
+                else
+                  schema_ids.at(Integer(params[:version]) - 1)
+                end
+    halt(404, VERSION_NOT_FOUND) unless schema_id
+
+    schema = SCHEMAS.at(schema_id)
+
+    {
+      name: params[:subject],
+      version: schema_ids.index(schema_id) + 1,
+      id: schema_id,
+      schema: schema
+    }.to_json
+  end
+
+  post "/prefix/subjects/:subject" do
+    schema = parse_schema
+
+    # Note: this does not actually handle the same schema registered under
+    # multiple subjects
+    schema_id = SCHEMAS.index(schema)
+
+    halt(404, SCHEMA_NOT_FOUND) unless schema_id
+
+    {
+      subject: params[:subject],
+      id: schema_id,
+      version: SUBJECTS[params[:subject]].index(schema_id) + 1,
+      schema: schema
+    }.to_json
+  end
+
+  post "/prefix/compatibility/subjects/:subject/versions/:version" do
+    # The ruby avro gem does not yet include a compatibility check between schemas.
+    # See https://github.com/apache/avro/pull/170
+    raise NotImplementedError
+  end
+
+  get "/prefix/config" do
+    global_config.to_json
+  end
+
+  put "/prefix/config" do
+    global_config.merge!(parse_config).to_json
+  end
+
+  get "/prefix/config/:subject" do
+    CONFIGS.fetch(params[:subject], global_config).to_json
+  end
+
+  put "/prefix/config/:subject" do
+    config = parse_config
+    subject = params[:subject]
+    CONFIGS.fetch(subject) do
+      CONFIGS[subject] = {}
+    end.merge!(config).to_json
+  end
+end

data/lib/avro_turf/version.rb

@@ -1,3 +1,3 @@
 class AvroTurf
-  VERSION = "1.4.1"
+  VERSION = "1.7.0"
 end

data/lib/avro_turf.rb

@@ -16,7 +16,9 @@ end
 
 class AvroTurf
   class Error < StandardError; end
+
   class SchemaError < Error; end
+
   class SchemaNotFoundError < Error; end
 
   DEFAULT_SCHEMAS_PATH = "./schemas"
@@ -31,7 +33,7 @@ class AvroTurf
   # Currently, the only valid codec name is `deflate`.
   def initialize(schemas_path: nil, schema_store: nil, namespace: nil, codec: nil)
     @namespace = namespace
-    @schema_store = schema_store || 
+    @schema_store = schema_store ||
       SchemaStore.new(path: schemas_path || DEFAULT_SCHEMAS_PATH)
     @codec = codec
   end
@@ -62,14 +64,20 @@ class AvroTurf
   # validate    - The boolean for performing complete data validation before
   #               encoding it, Avro::SchemaValidator::ValidationError with
   #               a descriptive message will be raised in case of invalid message.
+  # validate_options - Hash for the Avro::SchemaValidator, default
+  #               {recursive: true, encoded: false, fail_on_extra_fields: true}
   #
   # Returns nothing.
-  def encode_to_stream(data, schema_name: nil, stream: nil, namespace: @namespace, validate: false)
+  def encode_to_stream(data, schema_name: nil, stream: nil, namespace: @namespace,
+                       validate: false,
+                       validate_options: { recursive: true,
+                                           encoded: false,
+                                           fail_on_extra_fields: true })
     schema = @schema_store.find(schema_name, namespace)
     writer = Avro::IO::DatumWriter.new(schema)
 
     if validate
-      Avro::SchemaValidator.validate!(schema, data, recursive: true, encoded: false, fail_on_extra_fields: true)
+      Avro::SchemaValidator.validate!(schema, data, **validate_options)
     end
 
     dw = Avro::DataFile::Writer.new(stream, writer, schema, @codec)

data/spec/avro_turf_spec.rb

@@ -251,6 +251,31 @@ describe AvroTurf do
           expect { encode_to_stream }.to raise_error(Avro::SchemaValidator::ValidationError, /extra field 'fulll_name'/)
         end
       end
+
+      context "when the `fail_on_extra_fields` validation option is disabled" do
+        let(:message) { { "full_name" => "John Doe", "first_name" => "John", "last_name" => "Doe" } }
+        subject(:encode_to_stream) do
+          stream = StringIO.new
+          avro.encode_to_stream(message, stream: stream, schema_name: "message",
+                                validate: true,
+                                validate_options: { recursive: true, encoded: false, fail_on_extra_fields: false }
+          )
+        end
+
+        it "should not raise Avro::SchemaValidator::ValidationError with a message about extra field" do
+          define_schema "message.avsc", <<-AVSC
+            {
+              "name": "message",
+              "type": "record",
+              "fields": [
+                { "name": "full_name", "type": "string" }
+              ]
+            }
+          AVSC
+
+          expect { encode_to_stream }.not_to raise_error
+        end
+      end
     end
   end
 

data/spec/messaging_spec.rb

@@ -1,6 +1,7 @@
 require 'webmock/rspec'
 require 'avro_turf/messaging'
 require 'avro_turf/test/fake_confluent_schema_registry_server'
+require 'avro_turf/test/fake_prefixed_confluent_schema_registry_server'
 
 describe AvroTurf::Messaging do
   let(:registry_url) { "http://registry.example.com" }
@@ -8,7 +9,7 @@ describe AvroTurf::Messaging do
   let(:client_key) { "test client key" }
   let(:client_key_pass) { "test client key password" }
   let(:logger) { Logger.new(StringIO.new) }
-
+  let(:path_prefix) { nil }
   let(:avro) {
     AvroTurf::Messaging.new(
       registry_url: registry_url,
@@ -72,7 +73,7 @@ describe AvroTurf::Messaging do
 
   shared_examples_for 'encoding and decoding with the schema from registry' do
     before do
-      registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger)
+      registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger, path_prefix: path_prefix)
       registry.register('person', schema)
       registry.register('people', schema)
     end
@@ -102,7 +103,7 @@ describe AvroTurf::Messaging do
 
   shared_examples_for 'encoding and decoding with the schema_id from registry' do
     before do
-      registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger)
+      registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger, path_prefix: path_prefix)
       registry.register('person', schema)
       registry.register('people', schema)
     end
@@ -399,4 +400,29 @@ describe AvroTurf::Messaging do
       end
     end
   end
-end
+
+  context 'with a registry path prefix' do
+    let(:path_prefix) { '/prefix' }
+
+    let(:avro) {
+      AvroTurf::Messaging.new(
+        registry_path_prefix: path_prefix,
+        registry_url: registry_url,
+        schemas_path: "spec/schemas",
+        logger: logger,
+        client_cert: client_cert,
+        client_key: client_key,
+        client_key_pass: client_key_pass
+      )
+    }
+
+    before do
+      stub_request(:any, /^#{registry_url}/).to_rack(FakePrefixedConfluentSchemaRegistryServer)
+      FakePrefixedConfluentSchemaRegistryServer.clear
+    end
+
+    it_behaves_like "encoding and decoding with the schema from schema store"
+    it_behaves_like 'encoding and decoding with the schema from registry'
+    it_behaves_like 'encoding and decoding with the schema_id from registry'
+  end
+end

data/spec/schema_store_spec.rb

@@ -26,6 +26,33 @@ describe AvroTurf::SchemaStore do
       expect(schema.fullname).to eq "message"
     end
 
+    it "resolves missing references when nested schema is not a named type" do
+      define_schema "root.avsc", <<-AVSC
+        {
+          "type": "record",
+          "name": "root",
+          "fields": [
+            {
+              "type": "nested",
+              "name": "nested_value"
+            }
+          ]
+        }
+      AVSC
+
+      define_schema "nested.avsc", <<-AVSC
+        {
+          "name": "nested",
+          "type": "string",
+          "logicalType": "uuid"
+        }
+      AVSC
+
+      schema = store.find("root")
+
+      expect(schema.fullname).to eq "root"
+    end
+
     it "resolves missing references" do
       define_schema "person.avsc", <<-AVSC
         {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: avro_turf
 version: !ruby/object:Gem::Version
-  version: 1.4.1
+  version: 1.7.0
 platform: ruby
 authors:
 - Daniel Schierbeck
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-07-29 00:00:00.000000000 Z
+date: 2022-07-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro
@@ -19,7 +19,7 @@ dependencies:
         version: 1.7.7
     - - "<"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '1.12'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -29,7 +29,7 @@ dependencies:
         version: 1.7.7
     - - "<"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '1.12'
 - !ruby/object:Gem::Dependency
   name: excon
   requirement: !ruby/object:Gem::Requirement
@@ -156,7 +156,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
+description:
 email:
 - dasch@zendesk.com
 executables: []
@@ -196,6 +196,7 @@ files:
 - lib/avro_turf/schema_store.rb
 - lib/avro_turf/schema_to_avro_patch.rb
 - lib/avro_turf/test/fake_confluent_schema_registry_server.rb
+- lib/avro_turf/test/fake_prefixed_confluent_schema_registry_server.rb
 - lib/avro_turf/test/fake_schema_registry_server.rb
 - lib/avro_turf/version.rb
 - perf/address.avsc
@@ -225,7 +226,8 @@ files:
 homepage: https://github.com/dasch/avro_turf
 licenses:
 - MIT
-metadata: {}
+metadata:
+  rubygems_mfa_required: 'true'
 post_install_message: |2
 
   avro_turf v0.8.0 deprecates the names AvroTurf::SchemaRegistry,
@@ -249,8 +251,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.3.3
+signing_key:
 specification_version: 4
 summary: A library that makes it easier to use the Avro serialization format from
   Ruby