avrolution 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cbf0fc83c28860289887a88b8e789f5217f2c365
-  data.tar.gz: 54e25410dffeb032b364f5cd0568c679b56eae5a
+  metadata.gz: 44d01ea587ccbb464a081d3376aa5744b1d8c152
+  data.tar.gz: 431e8de8391b220872e68ce3927e830583fa85a5
 SHA512:
-  metadata.gz: 33fbbf8d76a0f753d6271d20ef08cf3dff2700f7b2b266fe4c3fb3f7e4748ae0bf529afda8411ad37b25c2460f6acf8a5bc2f0cf59191121854ae4512dcbbb7e
-  data.tar.gz: ad9c8d2f67de145deafb60ee1b7199bb64db193b78ccdda78c938d843b0b40441609c69c7e5a746fc8dd899aa6b0e579907ca1344b08527d881b147f46b38605
+  metadata.gz: 0a0bfe0ffc2609847a4fb606aa5065857fe7513fbbde5fd44bf1372263826122dd292b468741a0d99714f3a20bf8d8188a19c78c80eab819f6232ed1c0bb6aba
+  data.tar.gz: 8a2b82eced1d43966a21207df5d5575d3818513eb7f2f9aa348c36de903776af3875297d302d856b270f3a750a70149d70211c9ca6571bb254f408dbaf2fba6c

data/CHANGELOG.md

@@ -1,5 +1,8 @@
 # avrolution
 
+## v0.3.0
+- Add rake task to register new schema versions.
+
 ## v0.2.0
 - Add Rails generator to create `avro_compatibility_breaks.txt` file.
 - Replace the dependency on `avromatic` with `avro_schema_registry-client`.

data/README.md

@@ -39,8 +39,11 @@ The gem supports the following configuration:
 * `compatibility_breaks_file` - The path to the compability breaks file. Defaults
   to `#{Avrolution}.root/avro_compatibility_breaks.txt`.
 * `compatibility_schema_registry_url` - The URL for the schema registry to use
-  for compatibility checking. `ENV['COMPATIBILITY_SCHEMA_REGISTRY_URL]` is used
+  for compatibility checking. `ENV['COMPATIBILITY_SCHEMA_REGISTRY_URL']` is used
   as the default.
+* `deployment_schema_registry_url` - The URL for the schema registry to use
+  when registering new schema version. `ENV['DEPLOYMENT_SCHEMA_REGISTRY_URL']`
+  is used as the default.
 * `logger` - A logger used by the rake tasks in this gem. This does _NOT_ default
   to `Rails.logger` in Rails applications.
 
@@ -57,7 +60,7 @@ defined via a Railtie.
 This task does not require any arguments. It checks the
 compatibility of all Avro JSON schemas found recursively under `Avrolution.root`
 against the schema registry `Avroluion.compatibility_schema_registry_url` or
-`ENV['COMPATIBILITY_SCHEMA_REGISTRY_URL]`.
+`ENV['COMPATIBILITY_SCHEMA_REGISTRY_URL']`.
 
 ```bash
 rake avro:check_compatibility
@@ -75,6 +78,34 @@ require 'avrolution/rake/check_compatibility_task'
 Avrolution::Rake::CheckCompatibilityTask.define
 ```
 
+### Avro Register Schemas Rake Task
+
+There is a rake task to register new schemas.
+
+For Rails applications, the `avro:register_schemas` task is automatically
+defined via a Railtie.
+
+This rake task requires a comma-separated list of files for the schemas to register.
+
+```bash
+rake avro:register_schemas schemas=/app/avro/schemas/one.avsc,/app/avro/schema/two.avsc
+```
+
+Schemas are registered against the schema registry
+`Avroluion.deployment_schema_registry_url` or
+`ENV['DEPLOYMENT_SCHEMA_REGISTRY_URL']`.
+
+The `Avrolution.compatibility_breaks_file` is consulted prior to registering the
+schema, and if an entry is found then the specified compatibility settings are
+used.
+
+For non-Rails projects, tasks can be defined as:
+
+```ruby
+require 'avroluation/rake/register_schemas_task'
+Avrolution::Rake::RegisterSchemasTask.define
+```
+
 ### Avro Add Compatibility Break Rake Task
 
 There is a rake task add an entry to the `Avrolution.compatibility_breaks_file`.
@@ -86,8 +117,6 @@ This rake task accepts the following arguments:
  during registration.
 * `after_compatibility` - Optional compatibility level to set after registration.
 
-(The registration support will be available in a future version of this gem.)
-
 ```bash
 rake avro:add_compatibility_break name=com.salsify.alerts.example_value \
   fingerprint=36a2035c15c1bbbfe895494697d1f760171d00ab4fd39d0616261bf6854374f9 \

data/avrolution.gemspec

@@ -29,16 +29,18 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler', '~> 1.12'
   spec.add_development_dependency 'rake', '~> 10.0'
-  spec.add_development_dependency 'rspec', '~> 3.4'
+  spec.add_development_dependency 'rspec', '~> 3.5'
+  spec.add_development_dependency 'rspec-its'
   spec.add_development_dependency 'salsify_rubocop', '~> 0.47.2'
   spec.add_development_dependency 'overcommit'
   spec.add_development_dependency 'fakefs'
   spec.add_development_dependency 'simplecov'
 
   spec.add_runtime_dependency 'avro-salsify-fork', '1.9.0.5'
-  spec.add_runtime_dependency 'avro_schema_registry-client'
+  spec.add_runtime_dependency 'avro_schema_registry-client', '>= 0.2.0'
   spec.add_runtime_dependency 'diffy'
   spec.add_runtime_dependency 'private_attr'
   spec.add_runtime_dependency 'activesupport'
   spec.add_runtime_dependency 'activemodel'
+  spec.add_runtime_dependency 'procto'
 end

data/lib/avrolution/compatibility_break.rb

@@ -35,5 +35,11 @@ module Avrolution
     def line
       [name, fingerprint, with_compatibility, after_compatibility].compact.join(' ')
     end
+
+    def register_options
+      { with_compatibility: with_compatibility }.tap do |options|
+        options[:after_compatibility] = after_compatibility if after_compatibility.present?
+      end
+    end
   end
 end

data/lib/avrolution/configuration.rb

@@ -1,6 +1,7 @@
 module Avrolution
 
   COMPATIBILITY_SCHEMA_REGISTRY_URL = 'COMPATIBILITY_SCHEMA_REGISTRY_URL'.freeze
+  DEPLOYMENT_SCHEMA_REGISTRY_URL = 'DEPLOYMENT_SCHEMA_REGISTRY_URL'.freeze
 
   class << self
     # Root directory to search for schemas, and default location for
@@ -15,6 +16,10 @@ module Avrolution
     # compatibility checks
     attr_writer :compatibility_schema_registry_url
 
+    # The URL (including any Basic Auth) for the schema registry to use for
+    # deployment
+    attr_writer :deployment_schema_registry_url
+
     attr_accessor :logger
   end
 
@@ -29,9 +34,14 @@ module Avrolution
   end
 
   def self.compatibility_schema_registry_url
-    @compatibility_schema_registry_url ||= begin
-      raise 'compatibility_schema_registry_url must be set' unless ENV[COMPATIBILITY_SCHEMA_REGISTRY_URL]
-      ENV[COMPATIBILITY_SCHEMA_REGISTRY_URL]
+    @compatibility_schema_registry_url ||= ENV.fetch(COMPATIBILITY_SCHEMA_REGISTRY_URL) do
+      raise 'compatibility_schema_registry_url must be set'
+    end
+  end
+
+  def self.deployment_schema_registry_url
+    @deployment_schema_registry_url ||= ENV.fetch(DEPLOYMENT_SCHEMA_REGISTRY_URL) do
+      raise 'deployment_schema_registry_url must be set'
     end
   end
 

data/lib/avrolution/rake/rails_avrolution.rake

@@ -1,5 +1,7 @@
 require 'avrolution/rake/check_compatibility_task'
 require 'avrolution/rake/add_compatibility_break_task'
+require 'avrolution/rake/register_schemas_task'
 
 Avrolution::Rake::AddCompatibilityBreakTask.define(dependencies: %i(environment))
 Avrolution::Rake::CheckCompatibilityTask.define(dependencies: %i(environment))
+Avrolution::Rake::RegisterSchemasTask.define(dependencies: %i(environment))

data/lib/avrolution/rake/register_schemas_task.rb

@@ -0,0 +1,24 @@
+require 'avrolution/rake/base_task'
+
+module Avrolution
+  module Rake
+    class RegisterSchemasTask < BaseTask
+
+      def initialize(*)
+        super
+        @name ||= :register_schemas
+        @task_desc ||= 'Register the specified Avro JSON schemas'
+      end
+
+      private
+
+      def perform
+        raise 'schemas must be specified' if ENV['schemas'].blank?
+
+        schemas = ENV['schemas'].split(',')
+
+        Avrolution::RegisterSchemas.call(schemas)
+      end
+    end
+  end
+end

data/lib/avrolution/register_schemas.rb

@@ -0,0 +1,63 @@
+require 'avro_schema_registry-client'
+require 'private_attr'
+require 'procto'
+
+module Avrolution
+  class RegisterSchemas
+    extend PrivateAttr
+    include Procto.call
+
+    attr_reader :schema_files
+
+    private_attr_reader :compatibility_breaks, :schema_registry
+
+    class IncompatibleSchemaError < StandardError
+      def initialize(name)
+        super("incompatible schema #{name}")
+      end
+    end
+
+    def initialize(schema_files)
+      @schema_files = Array(schema_files)
+      @compatibility_breaks = Avrolution::CompatibilityBreaksFile.load
+      @schema_registry = build_schema_registry
+    end
+
+    def call
+      schemas.each do |(json, schema)|
+        register_schema(schema, json)
+      end
+    end
+
+    private
+
+    def register_schema(schema, json)
+      fullname = schema.fullname
+      fingerprint = schema.sha256_resolution_fingerprint.to_s(16)
+
+      compatibility_break = compatibility_breaks[[fullname, fingerprint]]
+
+      begin
+        schema_registry.register_without_lookup(
+          fullname,
+          json,
+          compatibility_break.try(:register_options) || {}
+        )
+      rescue Excon::Error::Conflict
+        raise IncompatibleSchemaError.new(fullname)
+      end
+    end
+
+    def schemas
+      @schemas ||= schema_files.map do |schema_file|
+        json = File.read(schema_file)
+        [json, Avro::Schema.parse(json)]
+      end
+    end
+
+    def build_schema_registry
+      AvroSchemaRegistry::Client.new(Avrolution.deployment_schema_registry_url,
+                                     logger: Avrolution.logger)
+    end
+  end
+end

data/lib/avrolution/version.rb

@@ -1,3 +1,3 @@
 module Avrolution
-  VERSION = '0.2.0'.freeze
+  VERSION = '0.3.0'.freeze
 end

data/lib/avrolution.rb

@@ -15,5 +15,6 @@ require 'avrolution/configuration'
 require 'avrolution/compatibility_break'
 require 'avrolution/compatibility_breaks_file'
 require 'avrolution/compatibility_check'
+require 'avrolution/register_schemas'
 
 require 'avrolution/railtie' if defined?(Rails)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: avrolution
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Salsify, Inc
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-04-11 00:00:00.000000000 Z
+date: 2017-04-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -44,14 +44,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.4'
+        version: '3.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.4'
+        version: '3.5'
+- !ruby/object:Gem::Dependency
+  name: rspec-its
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: salsify_rubocop
   requirement: !ruby/object:Gem::Requirement
@@ -128,14 +142,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.2.0
 - !ruby/object:Gem::Dependency
   name: diffy
   requirement: !ruby/object:Gem::Requirement
@@ -192,6 +206,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: procto
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Support for the evolution of Avro schemas stored in a schema registry.
 email:
 - engineering@salsify.com
@@ -224,6 +252,8 @@ files:
 - lib/avrolution/rake/base_task.rb
 - lib/avrolution/rake/check_compatibility_task.rb
 - lib/avrolution/rake/rails_avrolution.rake
+- lib/avrolution/rake/register_schemas_task.rb
+- lib/avrolution/register_schemas.rb
 - lib/avrolution/version.rb
 - lib/generators/avrolution/install_generator.rb
 - lib/generators/avrolution/templates/avro_compatibility_breaks.txt