avro_turf 0.7.2 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 0df3c290be95bcafd27a4473ba75df36a5275a6b
-  data.tar.gz: 17e1a0283f0479c60a691d3c52e33fb8eaf3f30c
+SHA256:
+  metadata.gz: 1e2ee4d9598bcaa2ec5738a1130fae0b19be7b5e7250f27540313b7471f97e23
+  data.tar.gz: 0b441cb30a153958c2ea283300a1a05c26218e2a97cb807fce36f8ad9d0240da
 SHA512:
-  metadata.gz: 7df0d445557d08c28a9a6e0357dea7b76159846ab3f3e456b1200e4a28c84bb1853727d4259bcf94432567f8031cfb34e80b30963b5fb1e1f43fff5ce0a52e1e
-  data.tar.gz: 144873d0859cb6c937898b94ca3649dca34284b9fa16ebaa122244faef4c20d838c871ab51ecf91c9c69d803d84fd55215867544ac57d5283a86b18fbc8ffc9f
+  metadata.gz: 12779eac5c325752cfa1be34da94ef5f332490cda4ff0aef29529a00557008cecf39592396a3f3525a2a12cb67a46744800781e0da69d4bf02511f5a2284e5e7
+  data.tar.gz: a2e4c84fb338d62296aefb8ae8c206c262d756475ec11bd9cead17bfe6015ea069ff36a891df3381385fa650cb4b5ea1584855fccc5294f5910ab34563ad973a

data/.circleci/config.yml

@@ -0,0 +1,36 @@
+version: 2
+jobs:
+  build:
+    environment:
+      CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
+      CIRCLE_TEST_REPORTS: /tmp/circleci-test-results
+    docker:
+    - image: circleci/ruby:2.6.2
+    steps:
+    - checkout
+    - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS
+    - restore_cache:
+        keys:
+        # This branch if available
+        - v1-dep-{{ .Branch }}-
+        # Default branch if not
+        - v1-dep-master-
+        # Any branch if there are none on the default branch - this should be unnecessary if you have your default branch configured correctly
+        - v1-dep-
+    - run: gem install bundler --no-document
+    - run: 'bundle check --path=vendor/bundle || bundle install --path=vendor/bundle --jobs=4 --retry=3'
+    # Save dependency cache
+    - save_cache:
+        key: v1-dep-{{ .Branch }}-{{ epoch }}
+        paths:
+        - vendor/bundle
+        - ~/.bundle
+    - run: mkdir -p $CIRCLE_TEST_REPORTS/rspec
+    - run:
+        command: bundle exec rspec --color --require spec_helper --format progress
+    - store_test_results:
+        path: /tmp/circleci-test-results
+    - store_artifacts:
+        path: /tmp/circleci-artifacts
+    - store_artifacts:
+        path: /tmp/circleci-test-results

data/.github/workflows/ruby.yml

@@ -0,0 +1,20 @@
+name: Ruby
+
+on: [push]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Ruby 2.6
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.6.x
+    - name: Build and test with RSpec
+      run: |
+        gem install bundler
+        bundle install --jobs 4 --retry 3
+        bundle exec rspec

data/CHANGELOG.md

@@ -0,0 +1,34 @@
+# avro_turf
+
+## Unreleased
+
+## v0.11.0
+
+- Add proxy support (#107)
+- Adding support for client certs (#109)
+
+## v0.10.0
+
+- Add more disk caching (#103)
+- Include schema information when decoding (#100, #101, #104)
+
+## v0.9.0
+
+- Compatibility with Avro v1.9.0 (#94)
+- Disable the auto registeration of schema (#95)
+- abstracted caching from CachedConfluentSchemaRegistry (#74)
+- Load avro-patches if installed to silence deprecation errors (#85)
+- Make schema store to be thread safe (#92)
+
+## v0.8.1
+
+- Allow accessing schema store from outside AvroTurf (#68).
+
+## v0.8.0
+
+- The names `AvroTurf::SchemaRegistry`, `AvroTurf::CachedSchemaRegistry`, and
+  `FakeSchemaRegistryServer` are deprecated and will be removed in a future release.
+  Use `AvroTurf::ConfluentSchemaRegistry`, `AvroTurf::CachedConfluentSchemaRegistry`,
+  and `FakeConfluentSchemaRegistryServer` instead.
+- Add support for the Config API (http://docs.confluent.io/3.1.2/schema-registry/docs/api.html#config)
+  to `AvroTurf::ConfluentSchemaRegistry`.

data/Gemfile

@@ -2,6 +2,3 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in avro_turf.gemspec
 gemspec
-
-# Used by CircleCI to format RSpec results.
-gem 'rspec_junit_formatter', :git => 'git@github.com:circleci/rspec_junit_formatter.git'

data/README.md

@@ -5,6 +5,17 @@ AvroTurf is a library that makes it easier to encode and decode data using the [
 * Provides an idiomatic Ruby interface.
 * Allows referencing schemas defined in another file.
 
+## Deprecation Notice
+
+The `AvroTurf::SchemaRegistry`, `AvroTurf::CachedSchemaRegistry`,
+and `FakeSchemaRegistryServer` names have been deprecated because the Avro spec recently
+introduced an incompatible [single-message encoding format](https://github.com/apache/avro/commit/30408a9c192c5f4eaaf42f01f0ffbfffd705aa57).
+
+These classes have been renamed to `AvroTurf::ConfluentSchemaRegistry`,
+`AvroTurf::CachedConfluentSchemaRegistry`, and `FakeConfluentSchemaRegistry`.
+
+The aliases for the original names will be removed in a future release.
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -80,11 +91,8 @@ In the example above, the `person` schema references the `address` schema, even
 ```json
 // person_list.avsc
 {
-  "name": "person_list",
-  "type": {
-    "type": "array",
-    "items": "person"
-  }
+  "type": "array",
+  "items": "person"
 }
 ```
 
@@ -116,16 +124,39 @@ avro = AvroTurf::Messaging.new(registry_url: "http://my-registry:8081/")
 # time a schema is used.
 data = avro.encode({ "title" => "hello, world" }, schema_name: "greeting")
 
+# If you don't want to automatically register new schemas, you can pass explicitly
+# subject and version to specify which schema should be used for encoding.
+# It will fetch that schema from the registry and cache it. Subsequent instances
+# of the same schema version will be served by the cache.
+data = avro.encode({ "title" => "hello, world" }, subject: 'greeting', version: 1)
+
+# You can also pass explicitly schema_id to specify which schema
+# should be used for encoding.
+# It will fetch that schema from the registry and cache it. Subsequent instances
+# of the same schema version will be served by the cache.
+data = avro.encode({ "title" => "hello, world" }, schema_id: 2)
+
 # When decoding, the schema will be fetched from the registry and cached. Subsequent
 # instances of the same schema id will be served by the cache.
 avro.decode(data) #=> { "title" => "hello, world" }
+
+# If you want to get decoded message as well as the schema used to encode the message,
+# you can use `#decode_message` method.
+result = avro.decode_message(data)
+result.message       #=> { "title" => "hello, world" }
+result.schema_id     #=> 3
+result.writer_schema #=> #<Avro::Schema: ...>
+result.reader_schema #=> nil
 ```
 
-In addition to encoding and decoding data, you can check whether a schema is compatible
-with a subject in the registry using the [Compatibility API](http://docs.confluent.io/2.0.0/schema-registry/docs/api.html#compatibility)
+### Confluent Schema Registry Client
+
+The ConfluentSchemaRegistry client used by the Messaging API can also be used directly.
+It can check whether a schema is compatible with a subject in the registry using the [Compatibility API](http://docs.confluent.io/3.1.2/schema-registry/docs/api.html#compatibility):
 
 ```ruby
-require 'avro_turf/messaging'
+require 'avro_turf'
+require 'avro_turf/confluent_schema_registry'
 
 schema = <<-JSON
 {
@@ -144,15 +175,22 @@ schema = <<-JSON
 }
 JSON
 
-avro = AvroTurf::Messaging.new(registry_url: "http://my-registry:8081/")
+registry = AvroTurf::ConfluentSchemaRegistry.new("http://my-registry:8081/")
 
-# Returns true if the schema is compatible, false otherwise.
-avro.compatible?("person", schema)
+# Returns true if the schema is compatible, nil if the subject or version is not registered, and false if incompatible.
+registry.compatible?("person", schema)
+```
+
+The ConfluentSchemaRegistry client can also change the global compatibility level or the compatibility level for an individual subject using the [Config API](http://docs.confluent.io/3.1.2/schema-registry/docs/api.html#config):
+
+```ruby
+registry.update_global_config(compatibility: 'FULL')
+registry.update_subject_config("person", compatibility: 'NONE')
 ```
 
 ### Testing Support
 
-AvroTurf includes a `FakeSchemaRegistryServer` that can be used in tests. The
+AvroTurf includes a `FakeConfluentSchemaRegistryServer` that can be used in tests. The
 fake schema registry server depends on Sinatra but it is _not_ listed as a runtime
 dependency for AvroTurf. Sinatra must be added to your Gemfile or gemspec in order
 to use the fake server.
@@ -160,14 +198,14 @@ to use the fake server.
 Example using RSpec:
 
 ```ruby
-require 'avro_turf/test/fake_schema_registry_server'
+require 'avro_turf/test/fake_confluent_schema_registry_server'
 require 'webmock/rspec'
 
 # within an example
 let(:registry_url) { "http://registry.example.com" }
 before do
-  stub_request(:any, /^#{registry_url}/).to_rack(FakeSchemaRegistryServer)
-  FakeSchemaRegistryServer.clear
+  stub_request(:any, /^#{registry_url}/).to_rack(FakeConfluentSchemaRegistryServer)
+  FakeConfluentSchemaRegistryServer.clear
 end
 
 # Messaging objects created with the same registry_url will now use the fake server.

data/avro_turf.gemspec

@@ -17,14 +17,25 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "avro", ">= 1.7.7", "< 1.9"
+  spec.add_dependency "avro", ">= 1.7.7", "< 1.10"
   spec.add_dependency "excon", "~> 0.45"
 
-  spec.add_development_dependency "bundler", "~> 1.7"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rspec", "~> 3.2.0"
-  spec.add_development_dependency "fakefs", "~> 0.6.7"
+  spec.add_development_dependency "bundler", "~> 2.0"
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "rspec", "~> 3.2"
+  spec.add_development_dependency "fakefs", "~> 0.20.0"
   spec.add_development_dependency "webmock"
   spec.add_development_dependency "sinatra"
   spec.add_development_dependency "json_spec"
+  spec.add_development_dependency "rack-test"
+
+  spec.post_install_message = %{
+avro_turf v0.8.0 deprecates the names AvroTurf::SchemaRegistry,
+AvroTurf::CachedSchemaRegistry, and FakeSchemaRegistryServer.
+
+Use AvroTurf::ConfluentSchemaRegistry, AvroTurf::CachedConfluentSchemaRegistry,
+and FakeConfluentSchemaRegistryServer instead.
+
+See https://github.com/dasch/avro_turf#deprecation-notice
+}
 end

data/lib/avro_turf.rb

@@ -1,9 +1,18 @@
+begin
+  require 'avro-patches'
+rescue LoadError
+  false
+end
 require 'avro_turf/version'
 require 'avro'
 require 'json'
 require 'avro_turf/schema_store'
 require 'avro_turf/core_ext'
-require 'avro_turf/schema_to_avro_patch'
+
+# check for something that indicates Avro v1.9.0 or later
+unless defined?(::Avro::LogicalTypes)
+  require 'avro_turf/schema_to_avro_patch'
+end
 
 class AvroTurf
   class Error < StandardError; end
@@ -15,13 +24,15 @@ class AvroTurf
   # Create a new AvroTurf instance with the specified configuration.
   #
   # schemas_path - The String path to the root directory containing Avro schemas (default: "./schemas").
+  # schema_store - A schema store object that responds to #find(schema_name, namespace).
   # namespace    - The String namespace that should be used to qualify schema names (optional).
   # codec        - The String name of a codec that should be used to compress messages (optional).
   #
   # Currently, the only valid codec name is `deflate`.
-  def initialize(schemas_path: nil, namespace: nil, codec: nil)
+  def initialize(schemas_path: nil, schema_store: nil, namespace: nil, codec: nil)
     @namespace = namespace
-    @schema_store = SchemaStore.new(path: schemas_path || DEFAULT_SCHEMAS_PATH)
+    @schema_store = schema_store || 
+      SchemaStore.new(path: schemas_path || DEFAULT_SCHEMAS_PATH)
     @codec = codec
   end
 

data/lib/avro_turf/cached_confluent_schema_registry.rb

@@ -0,0 +1,39 @@
+require 'avro_turf/confluent_schema_registry'
+require 'avro_turf/in_memory_cache'
+require 'avro_turf/disk_cache'
+
+# Caches registrations and lookups to the schema registry in memory.
+class AvroTurf::CachedConfluentSchemaRegistry
+
+  # Instantiate a new CachedConfluentSchemaRegistry instance with the given configuration.
+  # By default, uses a provided InMemoryCache to prevent repeated calls to the upstream registry.
+  #
+  # upstream  - The upstream schema registry object that fully responds to all methods in the
+  #             AvroTurf::ConfluentSchemaRegistry interface.
+  # cache     - Optional user provided Cache object that responds to all methods in the AvroTurf::InMemoryCache interface.
+  def initialize(upstream, cache: nil)
+    @upstream = upstream
+    @cache = cache || AvroTurf::InMemoryCache.new()
+  end
+
+  # Delegate the following methods to the upstream
+  %i(subjects subject_versions check compatible?
+     global_config update_global_config subject_config update_subject_config).each do |name|
+    define_method(name) do |*args|
+      instance_variable_get(:@upstream).send(name, *args)
+    end
+  end
+
+  def fetch(id)
+    @cache.lookup_by_id(id) || @cache.store_by_id(id, @upstream.fetch(id))
+  end
+
+  def register(subject, schema)
+    @cache.lookup_by_schema(subject, schema) || @cache.store_by_schema(subject, schema, @upstream.register(subject, schema))
+  end
+
+  def subject_version(subject, version = 'latest')
+    @cache.lookup_by_version(subject, version) ||
+      @cache.store_by_version(subject, version, @upstream.subject_version(subject, version))
+  end
+end

data/lib/avro_turf/cached_schema_registry.rb

@@ -1,26 +1,6 @@
-require 'avro_turf/schema_registry'
+require 'avro_turf/cached_confluent_schema_registry'
 
-# Caches registrations and lookups to the schema registry in memory.
-class AvroTurf::CachedSchemaRegistry
+# AvroTurf::CachedSchemaRegistry is deprecated and will be removed in a future release.
+# Use AvroTurf::CachedConfluentSchemaRegistry instead.
 
-  def initialize(upstream)
-    @upstream = upstream
-    @schemas_by_id = {}
-    @ids_by_schema = {}
-  end
-
-  # Delegate the following methods to the upstream
-  %i(subjects subject_versions subject_version check compatible?).each do |name|
-    define_method(name) do |*args|
-      instance_variable_get(:@upstream).send(name, *args)
-    end
-  end
-
-  def fetch(id)
-    @schemas_by_id[id] ||= @upstream.fetch(id)
-  end
-
-  def register(subject, schema)
-    @ids_by_schema[subject + schema.to_s] ||= @upstream.register(subject, schema)
-  end
-end
+AvroTurf::CachedSchemaRegistry = AvroTurf::CachedConfluentSchemaRegistry

data/lib/avro_turf/confluent_schema_registry.rb

@@ -0,0 +1,125 @@
+require 'excon'
+
+class AvroTurf::ConfluentSchemaRegistry
+  CONTENT_TYPE = "application/vnd.schemaregistry.v1+json".freeze
+
+  def initialize(
+    url,
+    logger: Logger.new($stdout),
+    proxy: nil,
+    client_cert: nil,
+    client_key: nil,
+    client_key_pass: nil,
+    client_cert_data: nil,
+    client_key_data: nil
+  )
+    @logger = logger
+    headers = {
+      "Content-Type" => CONTENT_TYPE
+    }
+    headers[:proxy] = proxy if proxy&.present?
+    @connection = Excon.new(
+      url,
+      headers: headers,
+      client_cert: client_cert,
+      client_key: client_key,
+      client_key_pass: client_key_pass,
+      client_cert_data: client_cert_data,
+      client_key_data: client_key_data
+    )
+  end
+
+  def fetch(id)
+    @logger.info "Fetching schema with id #{id}"
+    data = get("/schemas/ids/#{id}")
+    data.fetch("schema")
+  end
+
+  def register(subject, schema)
+    data = post("/subjects/#{subject}/versions", body: {
+      schema: schema.to_s
+    }.to_json)
+
+    id = data.fetch("id")
+
+    @logger.info "Registered schema for subject `#{subject}`; id = #{id}"
+
+    id
+  end
+
+  # List all subjects
+  def subjects
+    get('/subjects')
+  end
+
+  # List all versions for a subject
+  def subject_versions(subject)
+    get("/subjects/#{subject}/versions")
+  end
+
+  # Get a specific version for a subject
+  def subject_version(subject, version = 'latest')
+    get("/subjects/#{subject}/versions/#{version}")
+  end
+
+  # Check if a schema exists. Returns nil if not found.
+  def check(subject, schema)
+    data = post("/subjects/#{subject}",
+                expects: [200, 404],
+                body: { schema: schema.to_s }.to_json)
+    data unless data.has_key?("error_code")
+  end
+
+  # Check if a schema is compatible with the stored version.
+  # Returns:
+  # - true if compatible
+  # - nil if the subject or version does not exist
+  # - false if incompatible
+  # http://docs.confluent.io/3.1.2/schema-registry/docs/api.html#compatibility
+  def compatible?(subject, schema, version = 'latest')
+    data = post("/compatibility/subjects/#{subject}/versions/#{version}",
+                expects: [200, 404],
+                body: { schema: schema.to_s }.to_json)
+    data.fetch('is_compatible', false) unless data.has_key?('error_code')
+  end
+
+  # Get global config
+  def global_config
+    get("/config")
+  end
+
+  # Update global config
+  def update_global_config(config)
+    put("/config", { body: config.to_json })
+  end
+
+  # Get config for subject
+  def subject_config(subject)
+    get("/config/#{subject}")
+  end
+
+  # Update config for subject
+  def update_subject_config(subject, config)
+    put("/config/#{subject}", { body: config.to_json })
+  end
+
+  private
+
+  def get(path, **options)
+    request(path, method: :get, **options)
+  end
+
+  def put(path, **options)
+    request(path, method: :put, **options)
+  end
+
+  def post(path, **options)
+    request(path, method: :post, **options)
+  end
+
+  def request(path, **options)
+    options = { expects: 200 }.merge!(options)
+    response = @connection.request(path: path, **options)
+    JSON.parse(response.body)
+  end
+end