avro_turf 1.7.0 → 1.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3eea429db02f7f6eba5f13f46b2bf3e82602371ef9306e374c1102561f508675
-  data.tar.gz: d798ef10b0e674f48ffdcc6bd3b87e59d9447c985d539f2ba01164e689fe4f8a
+  metadata.gz: 3cb36f9b1395a1a6210d1178ca0823636f30ad194caf57663e8fb5cc13273d68
+  data.tar.gz: 2fb489fe330b39b88ed585d6b622936e573b525c1f4c5c4fe50704b16b11fd39
 SHA512:
-  metadata.gz: e7cacb60d4e7890a700ebaba9a0cf1d46606fb01157e0746a0d6a6c900895e9ca1268f0a04dd4d03f5d1a9cec9bf6795d9c5d70acb6884965353c72bb1de4095
-  data.tar.gz: c4dbd604b66916355a6cba44ec4e7225eb439c2d9bb0682e62414b6ffa3018bb1d2a84b78d943983c99d7af3897a67658db9cb2205f7d92d6c13661ae44ba602
+  metadata.gz: be796c4b42daf30a79dd699d23842e61c61e8f8c70b1645f269ec25e4a4f4fb1e87148b0ffd69a68b24382e9abc01048cb3ba082bd3a3dd2162be714ecdcd57b
+  data.tar.gz: 332486d45cd611da3906247f4fb55386c7e1617043070dd44fff06b05c50c44237af6dad62f1fd087ca9c67580aadd53fc342f76509b69b67375532b60637bf9

data/.github/workflows/ruby.yml

@@ -7,17 +7,17 @@ jobs:
 
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        ruby: [2.3, 2.4, 2.5, 2.6, 2.7, 3.0]
+        ruby: [2.6, 2.7, "3.0", 3.1, 3.2]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Ruby ${{ matrix.ruby }}
       uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}
+        bundler-cache: true
     - name: Build and test with RSpec
       run: |
-        gem install bundler
-        bundle install --jobs 4 --retry 3
         bundle exec rspec

data/CHANGELOG.md

@@ -2,9 +2,18 @@
 
 ## Unreleased
 
+## v1.9.0
+
+- Send Accept and User-Agent headers on every request (#184)
+
+## v1.8.0
+
+- Add support for `Date` via appropriate logicalType defintion.  This is a backwards incompatible change  (#177)
+- Fixed schema file cache truncation on multiple running instances and parallel access to the cache files.
+
 ## v1.7.0
 
-- Added extra params for the validation message schem before encode (#169)
+- Added extra params for the validation message schema before encode (#169)
 - Fix infinite retry when loading schema with nested primary type in separate file (#165)
 
 ## v1.6.0
@@ -60,7 +69,7 @@
 ## v0.9.0
 
 - Compatibility with Avro v1.9.0 (#94)
-- Disable the auto registeration of schema (#95)
+- Disable the auto registration of schema (#95)
 - abstracted caching from CachedConfluentSchemaRegistry (#74)
 - Load avro-patches if installed to silence deprecation errors (#85)
 - Make schema store to be thread safe (#92)

data/avro_turf.gemspec

@@ -25,7 +25,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.2"
-  spec.add_development_dependency "fakefs", "~> 0.20.0"
+  spec.add_development_dependency "fakefs", "< 3"
   spec.add_development_dependency "webmock"
   spec.add_development_dependency "sinatra"
   spec.add_development_dependency "json_spec"

data/lib/avro_turf/cached_confluent_schema_registry.rb

@@ -13,7 +13,7 @@ class AvroTurf::CachedConfluentSchemaRegistry
   # cache     - Optional user provided Cache object that responds to all methods in the AvroTurf::InMemoryCache interface.
   def initialize(upstream, cache: nil)
     @upstream = upstream
-    @cache = cache || AvroTurf::InMemoryCache.new()
+    @cache = cache || AvroTurf::InMemoryCache.new
   end
 
   # Delegate the following methods to the upstream
@@ -34,7 +34,7 @@ class AvroTurf::CachedConfluentSchemaRegistry
 
   def subject_version(subject, version = 'latest')
     return @upstream.subject_version(subject, version) if version == 'latest'
-    
+
     @cache.lookup_by_version(subject, version) ||
       @cache.store_by_version(subject, version, @upstream.subject_version(subject, version))
   end

data/lib/avro_turf/confluent_schema_registry.rb

@@ -19,9 +19,9 @@ class AvroTurf::ConfluentSchemaRegistry
   )
     @path_prefix = path_prefix
     @logger = logger
-    headers = {
+    headers = Excon.defaults[:headers].merge(
       "Content-Type" => CONTENT_TYPE
-    }
+    )
     headers[:proxy] = proxy unless proxy.nil?
     @connection = Excon.new(
       url,

data/lib/avro_turf/core_ext/date.rb

@@ -1,5 +1,5 @@
 class Date
   def as_avro
-    iso8601
+    self
   end
 end

data/lib/avro_turf/disk_cache.rb

@@ -1,20 +1,18 @@
 # A cache for the CachedConfluentSchemaRegistry.
 # Extends the InMemoryCache to provide a write-thru to disk for persistent cache.
-class AvroTurf::DiskCache < AvroTurf::InMemoryCache
+class AvroTurf::DiskCache
 
   def initialize(disk_path, logger: Logger.new($stdout))
-    super()
-
     @logger = logger
 
     # load the write-thru cache on startup, if it exists
     @schemas_by_id_path = File.join(disk_path, 'schemas_by_id.json')
     hash = read_from_disk_cache(@schemas_by_id_path)
-    @schemas_by_id = hash if hash
+    @schemas_by_id = hash || {}
 
     @ids_by_schema_path = File.join(disk_path, 'ids_by_schema.json')
     hash = read_from_disk_cache(@ids_by_schema_path)
-    @ids_by_schema = hash if hash
+    @ids_by_schema = hash || {}
 
     @schemas_by_subject_version_path = File.join(disk_path, 'schemas_by_subject_version.json')
     @schemas_by_subject_version = {}
@@ -24,15 +22,16 @@ class AvroTurf::DiskCache < AvroTurf::InMemoryCache
   # the write-thru cache (json) does not store keys in numeric format
   # so, convert id to a string for caching purposes
   def lookup_by_id(id)
-    super(id.to_s)
+    @schemas_by_id[id.to_s]
   end
 
   # override to include write-thru cache after storing result from upstream
   def store_by_id(id, schema)
     # must return the value from storing the result (i.e. do not return result from file write)
-    value = super(id.to_s, schema)
-    File.write(@schemas_by_id_path, JSON.pretty_generate(@schemas_by_id))
-    return value
+    @schemas_by_id[id.to_s] = schema
+    write_to_disk_cache(@schemas_by_id_path, @schemas_by_id)
+
+    schema
   end
 
   # override to use a json serializable cache key
@@ -45,7 +44,8 @@ class AvroTurf::DiskCache < AvroTurf::InMemoryCache
   def store_by_schema(subject, schema, id)
     key = "#{subject}#{schema}"
     @ids_by_schema[key] = id
-    File.write(@ids_by_schema_path, JSON.pretty_generate(@ids_by_schema))
+
+    write_to_disk_cache(@ids_by_schema_path, @ids_by_schema)
     id
   end
 
@@ -78,7 +78,7 @@ class AvroTurf::DiskCache < AvroTurf::InMemoryCache
              hash[key] = schema
              hash
            else
-             {key => schema}
+             { key => schema }
            end
 
     write_to_disk_cache(@schemas_by_subject_version_path, hash)
@@ -90,17 +90,27 @@ class AvroTurf::DiskCache < AvroTurf::InMemoryCache
   # Parse the file from disk, if it exists and is not zero length
   private def read_from_disk_cache(path)
     if File.exist?(path)
-      if File.size(path)!=0
-        return JSON.parse(File.read(path))
+      if File.size(path) != 0
+        json_data = File.open(path, 'r') do |file|
+          file.flock(File::LOCK_SH)
+          file.read
+        end
+
+        return JSON.parse(json_data)
       else
         # just log a message if skipping zero length file
         @logger.warn "skipping JSON.parse of zero length file at #{path}"
       end
     end
-    return nil
+
+    nil
   end
 
   private def write_to_disk_cache(path, hash)
-    File.write(path, JSON.pretty_generate(hash))
+    # don't use "w" because it truncates the file before lock
+    File.open(path, File::RDWR | File::CREAT, 0644) do |file|
+      file.flock(File::LOCK_EX)
+      file.write(JSON.pretty_generate(hash))
+    end
   end
 end

data/lib/avro_turf/in_memory_cache.rb

@@ -17,12 +17,12 @@ class AvroTurf::InMemoryCache
   end
 
   def lookup_by_schema(subject, schema)
-    key = [subject, schema.to_s]
+    key = [subject, schema]
     @ids_by_schema[key]
   end
 
   def store_by_schema(subject, schema, id)
-    key = [subject, schema.to_s]
+    key = [subject, schema]
     @ids_by_schema[key] = id
   end
 

data/lib/avro_turf/messaging.rb

@@ -21,8 +21,20 @@ class AvroTurf
   # 1: https://github.com/confluentinc/schema-registry
   class Messaging
     MAGIC_BYTE = [0].pack("C").freeze
-    DecodedMessage = Struct.new(:schema_id, :writer_schema, :reader_schema, :message)
-    private_constant(:DecodedMessage)
+
+    class DecodedMessage
+      attr_reader :schema_id
+      attr_reader :writer_schema
+      attr_reader :reader_schema
+      attr_reader :message
+
+      def initialize(schema_id, writer_schema, reader_schema, message)
+        @schema_id = schema_id
+        @writer_schema = writer_schema
+        @reader_schema = reader_schema
+        @message = message
+      end
+    end
 
     # Instantiate a new Messaging instance with the given configuration.
     #
@@ -34,7 +46,7 @@ class AvroTurf
     # namespace            - The String default schema namespace.
     # registry_path_prefix - The String URL path prefix used to namespace schema registry requests (optional).
     # logger               - The Logger that should be used to log information (optional).
-    # proxy                - Forward the request via  proxy (optional).
+    # proxy                - Forward the request via  proxy (optional).
     # user                 - User for basic auth (optional).
     # password             - Password for basic auth (optional).
     # ssl_ca_file          - Name of file containing CA certificate (optional).
@@ -130,7 +142,7 @@ class AvroTurf
       writer.write(message, encoder)
 
       stream.string
-    rescue Excon::Errors::NotFound
+    rescue Excon::Error::NotFound
       if schema_id
         raise SchemaNotFoundError.new("Schema with id: #{schema_id} is not found on registry")
       else

data/lib/avro_turf/version.rb

@@ -1,3 +1,3 @@
 class AvroTurf
-  VERSION = "1.7.0"
+  VERSION = "1.9.0"
 end

data/lib/avro_turf.rb

@@ -121,10 +121,9 @@ class AvroTurf
   # namespace - The namespace of the Avro schema (optional).
   #
   # Returns true if the data is valid, false otherwise.
-  def valid?(data, schema_name: nil, namespace: @namespace)
+  def valid?(data, schema_name: nil, namespace: @namespace, validate_options: {})
     schema = schema_name && @schema_store.find(schema_name, namespace)
-
-    Avro::Schema.validate(schema, data.as_avro)
+    Avro::Schema.validate(schema, data.as_avro, **validate_options)
   end
 
   # Loads all schema definition files in the `schemas_dir`.

data/spec/avro_turf_spec.rb

@@ -16,6 +16,13 @@ describe AvroTurf do
               {
                 "type": "string",
                 "name": "full_name"
+              },
+              {
+                "name": "birth_date",
+                "type": {
+                  "type": "int",
+                  "logicalType": "date"
+                }
               }
             ]
           }
@@ -24,7 +31,8 @@ describe AvroTurf do
 
       it "encodes data with Avro" do
         data = {
-          "full_name" => "John Doe"
+          "full_name" => "John Doe",
+          "birth_date" => Date.new(1934, 1, 2)
         }
 
         encoded_data = avro.encode(data, schema_name: "person")
@@ -36,7 +44,8 @@ describe AvroTurf do
         compressed_avro = AvroTurf.new(schemas_path: "spec/schemas/", codec: "deflate")
 
         data = {
-          "full_name" => "John Doe" * 100
+          "full_name" => "John Doe" * 100,
+          "birth_date" => Date.new(1934, 1, 2)
         }
 
         uncompressed_data = avro.encode(data, schema_name: "person")
@@ -329,5 +338,64 @@ describe AvroTurf do
       datum = { message: "hello" }
       expect(avro.valid?(datum, schema_name: "postcard")).to eq true
     end
+
+    it "handles logicalType of date in schema" do
+      define_schema "postcard.avsc", <<-AVSC
+        {
+          "name": "postcard",
+          "type": "record",
+          "fields": [
+            {
+              "name": "message",
+              "type": "string"
+            },
+            {
+              "name": "sent_date",
+              "type": {
+                "type": "int",
+                "logicalType": "date"
+              }
+            }
+          ]
+        }
+      AVSC
+
+      datum = {
+        message: "hello",
+        sent_date: Date.new(2022, 9, 11)
+      }
+      expect(avro.valid?(datum, schema_name: "postcard")).to eq true
+    end
+
+    context "when message contains extra fields (typo in key)" do
+      let(:message) { { "fulll_name" => "John Doe" } }
+
+      before do
+        define_schema "message.avsc", <<-AVSC
+          {
+            "name": "message",
+            "type": "record",
+            "fields": [
+              { "name": "full_name", "type": "string" }
+            ]
+          }
+        AVSC
+      end
+
+      it "is valid" do
+        datum = { "full_name" => "John Doe", "extra" => "extra" }
+        expect(avro.valid?(datum, schema_name: "message")).to eq true
+      end
+
+      it "is invalid when passing fail_on_extra_fields" do
+        datum = { "full_name" => "John Doe", "extra" => "extra" }
+        validate_options = {
+          recursive: true,
+          encoded: false,
+          fail_on_extra_fields: true }
+        valid = avro.valid?(datum, schema_name: "message", validate_options: validate_options)
+        expect(valid).to eq false
+      end
+    end
   end
 end

data/spec/core_ext/date_spec.rb

@@ -1,6 +1,6 @@
 describe Date, "#as_avro" do
-  it "returns an ISO8601 string describing the time" do
+  it "returns Date object describing the time" do
     date = Date.today
-    expect(date.as_avro).to eq(date.iso8601)
+    expect(date.as_avro).to eq(date)
   end
 end

data/spec/spec_helper.rb

@@ -22,6 +22,16 @@ module Helpers
   end
 end
 
+# gem `fakefs` does not support flock for the file, and require patch
+# https://github.com/fakefs/fakefs/issues/433
+module FakeFS
+  class File < StringIO
+    def flock(*)
+      true
+    end
+  end
+end
+
 RSpec.configure do |config|
   config.include FakeFS::SpecHelpers
   config.include Helpers

data/spec/support/confluent_schema_registry_context.rb

@@ -13,9 +13,20 @@ shared_examples_for "a confluent schema registry client" do
       ]
     }.to_json
   end
+  let(:headers) do
+    {
+      'Accept'=>'*/*',
+      'Content-Type'=> AvroTurf::ConfluentSchemaRegistry::CONTENT_TYPE,
+      'Host'=> "#{URI.parse(registry_url).host}:80",
+      'User-Agent'=> "excon/#{Excon::VERSION}"
+    }
+  end
 
   before do
-    stub_request(:any, /^#{registry_url}/).to_rack(FakeConfluentSchemaRegistryServer)
+    stub_request(:any, /^#{registry_url}/)
+      .with(headers: headers)
+      .to_rack(FakeConfluentSchemaRegistryServer)
+
     FakeConfluentSchemaRegistryServer.clear
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: avro_turf
 version: !ruby/object:Gem::Version
-  version: 1.7.0
+  version: 1.9.0
 platform: ruby
 authors:
 - Daniel Schierbeck
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-07-25 00:00:00.000000000 Z
+date: 2023-06-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro
@@ -90,16 +90,16 @@ dependencies:
   name: fakefs
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.20.0
+        version: '3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.20.0
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -156,7 +156,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description:
+description: 
 email:
 - dasch@zendesk.com
 executables: []
@@ -251,8 +251,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.3
-signing_key:
+rubygems_version: 3.0.3.1
+signing_key: 
 specification_version: 4
 summary: A library that makes it easier to use the Avro serialization format from
   Ruby