avro_turf_enchanced 0.7.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 14a7499b6aebb21573dbee95446815f677762fc5
+  data.tar.gz: b387dede993698644fa5a1f9b2da31a40621574b
+SHA512:
+  metadata.gz: 93d80249a49cb7d4213664946e383d39fe57764c8cc069a210b835be760f7fb3c540f8a0bcda10e025ac2cf9ca77f1e95d2457b5060c4199aed616938647ac62
+  data.tar.gz: 1f6d8d33c4728f3a54fd323f134431ef8fa693dacdb7a5ebb8e50b4ccd86837a73ee03d813318bbae9436f19f3caa9092f043602a1d16d2d8b55073bdef4a871

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in avro_turf.gemspec
+gemspec
+
+# Used by CircleCI to format RSpec results.
+gem 'rspec_junit_formatter', :git => 'git@github.com:circleci/rspec_junit_formatter.git'

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Daniel Schierbeck
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,174 @@
+# AvroTurf
+
+AvroTurf is a library that makes it easier to encode and decode data using the [Apache Avro](http://avro.apache.org/) serialization format. It adds a layer on top of the official Avro gem which makes it easier to integrate Avro into your application:
+
+* Provides an idiomatic Ruby interface.
+* Allows referencing schemas defined in another file.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'avro_turf'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install avro_turf
+
+## Usage
+
+Using AvroTurf is quite simple:
+
+```ruby
+# Schemas will be looked up from the specified directory.
+avro = AvroTurf.new(schemas_path: "app/schemas/")
+
+# Decode some data using a named schema. The schema file should exist in the
+# schemas directory with the file name `<name>.avsc`.
+avro.decode(encoded_data, schema_name: "person")
+
+# Encode some data using the named schema.
+avro.encode({ "name" => "Jane", "age" => 28 }, schema_name: "person")
+```
+
+### Inter-schema references
+
+Unlike the official Avro library, AvroTurf allows schemas to reference each other. As an example:
+
+```json
+// person.avsc
+{
+  "name": "person",
+  "type": "record",
+  "fields": [
+    {
+      "name": "full_name",
+      "type": "string"
+    },
+    {
+      "name": "address",
+      "type": "address"
+    }
+  ]
+}
+
+// address.avsc
+{
+  "name": "address",
+  "type": "record",
+  "fields": [
+    {
+      "name": "street",
+      "type": "string"
+    },
+    {
+      "name": "city",
+      "type": "string"
+    }
+  ]
+}
+```
+
+In the example above, the `person` schema references the `address` schema, even though the latter is defined in another file. This makes it possible to share types across schemas, e.g.
+
+```json
+// person_list.avsc
+{
+  "name": "person_list",
+  "type": {
+    "type": "array",
+    "items": "person"
+  }
+}
+```
+
+There's no reason to copy-paste the `person` schema into the `person_list` schema, as you can reference it directly.
+
+This feature helps avoid subtle errors when the same type is represented using slightly different schemas.
+
+
+### Using a Schema Registry
+
+By default, AvroTurf will encode data in the Avro data file format. This means that the schema used to encode the data is prepended to the output. If you want to decrease the size of the output, e.g. when storing data in a log such as Apache Kafka or in a database, you can use the `AvroTurf::Messaging` API. This top-level API requires the use of [Schema Registry](https://github.com/confluentinc/schema-registry), a service which allows registering and fetching Avro schemas.
+
+The Messaging API will automatically register schemas used for encoding data, and will fetch the corresponding schema when decoding. Instead of including the full schema in the output, only a schema id generated by the registry is included. Registering the same schema twice is idempotent, so no coordination is needed.
+
+**NOTE:** [The Messaging format](https://github.com/confluentinc/schema-registry/blob/master/docs/serializer-formatter.rst#wire-format) is _not_ compatible with the Avro data file API.
+
+The Messaging API is not included by default, so you must require 'avro_turf/messaging' explicitly if you want to use it.
+
+Using the Messaging API is simple once you have set up a Schema Registry service:
+
+```ruby
+require 'avro_turf/messaging'
+
+# You need to pass the URL of your Schema Registry.
+avro = AvroTurf::Messaging.new(registry_url: "http://my-registry:8081/")
+
+# The API for encoding and decoding data is similar to the default one. Encoding
+# data has the side effect of registering the schema. This only happens the first
+# time a schema is used.
+data = avro.encode({ "title" => "hello, world" }, schema_name: "greeting")
+
+# When decoding, the schema will be fetched from the registry and cached. Subsequent
+# instances of the same schema id will be served by the cache.
+avro.decode(data) #=> { "title" => "hello, world" }
+```
+
+In addition to encoding and decoding data, you can check whether a schema is compatible
+with a subject in the registry using the [Compatibility API](http://docs.confluent.io/2.0.0/schema-registry/docs/api.html#compatibility)
+
+```ruby
+require 'avro_turf/messaging'
+
+schema = <<-JSON
+{
+  "name": "person",
+  "type": "record",
+  "fields": [
+    {
+      "name": "full_name",
+      "type": "string"
+    },
+    {
+      "name": "address",
+      "type": "address"
+    }
+  ]
+}
+JSON
+
+avro = AvroTurf::Messaging.new(registry_url: "http://my-registry:8081/")
+
+# Returns true if the schema is compatible, false otherwise.
+avro.compatible?("person", schema)
+```
+
+### Testing Support
+
+AvroTurf includes a `FakeSchemaRegistryServer` that can be used in tests. The
+fake schema registry server depends on Sinatra but it is _not_ listed as a runtime
+dependency for AvroTurf. Sinatra must be added to your Gemfile or gemspec in order
+to use the fake server.
+
+Example using RSpec:
+
+```ruby
+require 'avro_turf/test/fake_schema_registry_server'
+require 'webmock/rspec'
+
+# within an example
+let(:registry_url) { "http://registry.example.com" }
+before do
+  stub_request(:any, /^#{registry_url}/).to_rack(FakeSchemaRegistryServer)
+  FakeSchemaRegistryServer.clear
+end
+
+# Messaging objects created with the same registry_url will now use the fake server.
+```

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/avro_turf.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'avro_turf/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "avro_turf_enchanced"
+  spec.version       = AvroTurf::VERSION
+  spec.authors       = ["Daniel Schierbeck"]
+  spec.email         = ["dasch@zendesk.com"]
+  spec.summary       = "A library that makes it easier to use the Avro serialization format from Ruby"
+  spec.homepage      = "https://github.com/dasch/avro_turf"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "avro", ">= 1.7.7", "< 1.9"
+  spec.add_dependency "excon", "~> 0.45"
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.2.0"
+  spec.add_development_dependency "fakefs", "~> 0.6.7"
+  spec.add_development_dependency "webmock"
+  spec.add_development_dependency "sinatra"
+  spec.add_development_dependency "json_spec"
+end

data/circle.yml

@@ -0,0 +1,4 @@
+machine:
+  ruby:
+    version: 2.2.0
+    version: 2.0.0

data/lib/avro_turf.rb

@@ -0,0 +1,105 @@
+require 'avro_turf/version'
+require 'avro'
+require 'json'
+require 'avro_turf/schema_store'
+require 'avro_turf/core_ext'
+require 'avro_turf/schema_to_avro_patch'
+
+class AvroTurf
+  class Error < StandardError; end
+  class SchemaError < Error; end
+  class SchemaNotFoundError < Error; end
+
+  DEFAULT_SCHEMAS_PATH = "./schemas"
+
+  # Create a new AvroTurf instance with the specified configuration.
+  #
+  # schemas_path - The String path to the root directory containing Avro schemas (default: "./schemas").
+  # namespace    - The String namespace that should be used to qualify schema names (optional).
+  # codec        - The String name of a codec that should be used to compress messages (optional).
+  #
+  # Currently, the only valid codec name is `deflate`.
+  def initialize(schemas_path: nil, namespace: nil, codec: nil)
+    @namespace = namespace
+    @schema_store = SchemaStore.new(path: schemas_path || DEFAULT_SCHEMAS_PATH)
+    @codec = codec
+  end
+
+  # Encodes data to Avro using the specified schema.
+  #
+  # data        - The data that should be encoded.
+  # schema_name - The name of a schema in the `schemas_path`.
+  #
+  # Returns a String containing the encoded data.
+  def encode(data, schema_name: nil, namespace: @namespace)
+    stream = StringIO.new
+
+    encode_to_stream(data, stream: stream, schema_name: schema_name, namespace: namespace)
+
+    stream.string
+  end
+
+  # Encodes data to Avro using the specified schema and writes it to the
+  # specified stream.
+  #
+  # data        - The data that should be encoded.
+  # schema_name - The name of a schema in the `schemas_path`.
+  # stream      - An IO object that the encoded data should be written to (optional).
+  #
+  # Returns nothing.
+  def encode_to_stream(data, schema_name: nil, stream: nil, namespace: @namespace)
+    schema = @schema_store.find(schema_name, namespace)
+    writer = Avro::IO::DatumWriter.new(schema)
+
+    dw = Avro::DataFile::Writer.new(stream, writer, schema, @codec)
+    dw << data.as_avro
+    dw.close
+  end
+
+  # Decodes Avro data.
+  #
+  # encoded_data - A String containing Avro-encoded data.
+  # schema_name  - The String name of the schema that should be used to read
+  #                the data. If nil, the writer schema will be used.
+  # namespace    - The namespace of the Avro schema used to decode the data.
+  #
+  # Returns whatever is encoded in the data.
+  def decode(encoded_data, schema_name: nil, namespace: @namespace)
+    stream = StringIO.new(encoded_data)
+    decode_stream(stream, schema_name: schema_name, namespace: namespace)
+  end
+
+  # Decodes Avro data from an IO stream.
+  #
+  # stream       - An IO object containing Avro data.
+  # schema_name  - The String name of the schema that should be used to read
+  #                the data. If nil, the writer schema will be used.
+  # namespace    - The namespace of the Avro schema used to decode the data.
+  #
+  # Returns whatever is encoded in the stream.
+  def decode_stream(stream, schema_name: nil, namespace: @namespace)
+    schema = schema_name && @schema_store.find(schema_name, namespace)
+    reader = Avro::IO::DatumReader.new(nil, schema)
+    dr = Avro::DataFile::Reader.new(stream, reader)
+    dr.first
+  end
+
+  # Validates data against an Avro schema.
+  #
+  # data        - The data that should be validated.
+  # schema    - The String name of the schema that should be used to validate
+  #             the data.
+  # namespace - The namespace of the Avro schema (optional).
+  #
+  # Returns true if the data is valid, false otherwise.
+  def valid?(data, schema_name: nil, namespace: @namespace)
+    schema = schema_name && @schema_store.find(schema_name, namespace)
+
+    Avro::Schema.validate(schema, data.as_avro)
+  end
+
+  # Loads all schema definition files in the `schemas_dir`.
+  def load_schemas!
+    @schema_store.load_schemas!
+  end
+end

data/lib/avro_turf/cached_schema_registry.rb

@@ -0,0 +1,26 @@
+require 'avro_turf/schema_registry'
+
+# Caches registrations and lookups to the schema registry in memory.
+class AvroTurf::CachedSchemaRegistry
+
+  def initialize(upstream)
+    @upstream = upstream
+    @schemas_by_id = {}
+    @ids_by_schema = {}
+  end
+
+  # Delegate the following methods to the upstream
+  %i(subjects subject_versions subject_version check compatible?).each do |name|
+    define_method(name) do |*args|
+      instance_variable_get(:@upstream).send(name, *args)
+    end
+  end
+
+  def fetch(id)
+    @schemas_by_id[id] ||= @upstream.fetch(id)
+  end
+
+  def register(subject, schema)
+    @ids_by_schema[subject + schema.to_s] ||= @upstream.register(subject, schema)
+  end
+end

data/lib/avro_turf/core_ext.rb

@@ -0,0 +1,10 @@
+require 'avro_turf/core_ext/string'
+require 'avro_turf/core_ext/numeric'
+require 'avro_turf/core_ext/enumerable'
+require 'avro_turf/core_ext/hash'
+require 'avro_turf/core_ext/time'
+require 'avro_turf/core_ext/date'
+require 'avro_turf/core_ext/symbol'
+require 'avro_turf/core_ext/nil_class'
+require 'avro_turf/core_ext/true_class'
+require 'avro_turf/core_ext/false_class'

data/lib/avro_turf/core_ext/date.rb

@@ -0,0 +1,5 @@
+class Date
+  def as_avro
+    iso8601
+  end
+end

data/lib/avro_turf/core_ext/enumerable.rb

@@ -0,0 +1,5 @@
+module Enumerable
+  def as_avro
+    map(&:as_avro)
+  end
+end

data/lib/avro_turf/core_ext/false_class.rb

@@ -0,0 +1,5 @@
+class FalseClass
+  def as_avro
+    self
+  end
+end