avro_turf 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d083d0ed365ddbd16e9a5e35acf698d50a7bd7d5
-  data.tar.gz: a297b67a288046a64932ad1e388cd39e4e7a106d
+  metadata.gz: bdd8c25f6656ec55bcb0054fa42e4a85d5018fa6
+  data.tar.gz: af4d189168531a28fc31793f8f3c65146c70e976
 SHA512:
-  metadata.gz: de5d8c415fcf16b731aa9cd4b69c5e8fc4e2243b092744a637a68a36494083330b861731f7567a4677ba0d149c60a8da6cbce38e40ee32771a84cdb9c687e843
-  data.tar.gz: 0e14a355dc76805b335d2fda8871671714a3843abe60886ef9d25ec5a4bad3270127d46386c2713e968a1eba47131753389b3cbb55531e8cafd0535ac14b2f39
+  metadata.gz: 039e85f46a0d25766bc24dabe3984e4f1fc6b4c535aaacf53a87d8db8ee308693c5c210eb7c550bcf752de2ab3048cbed06a4bb003e23123b0ea8dfb37c0e648
+  data.tar.gz: d40406ac87ac25fd476b82c14a8f732daf9067d26176aa2138d234264a8bb8c543ed3d55623161c4adb9320311296485297bb93a96459ac240739ceaa659df3f

data/README.md

@@ -91,3 +91,28 @@ In the example above, the `person` schema references the `address` schema, even
 There's no reason to copy-paste the `person` schema into the `person_list` schema, as you can reference it directly.
 
 This feature helps avoid subtle errors when the same type is represented using slightly different schemas.
+
+
+### Using a Schema Registry
+
+By default, AvroTurf will encode data in the Avro data file format. This means that the schema used to encode the data is prepended to the output. If you want to decrease the size of the output, e.g. when storing data in a log such as Apache Kafka or in a database, you can use the `AvroTurf::Messaging` API. This top-level API requires the use of [Schema Registry](https://github.com/confluentinc/schema-registry), a service which allows registering and fetching Avro schemas.
+
+The Messaging API will automatically register schemas used for encoding data, and will fetch the corresponding schema when decoding. Instead of including the full schema in the output, only a schema id generated by the registry is included. Registering the same schema twice is idempotent, so no coordination is needed.
+
+**NOTE:** The Messaging format is _not_ compatible with the Avro data file API.
+
+Using the Messaging API is simple once you have set up a Schema Registry service:
+
+```ruby
+# You need to pass the URL of your Schema Registry.
+avro = AvroTurf::Messaging.new(registry_url: "http://my-registry:8081/")
+
+# The API for encoding and decoding data is similar to the default one. Encoding
+# data has the side effect of registering the schema. This only happens the first
+# time a schema is used.
+data = avro.encode({ "title" => "hello, world" }, schema_name: "greeting")
+
+# When decoding, the schema will be fetched from the registry and cached. Subsequent
+# instances of the same schema id will be served by the cache.
+avro.decode(data) #=> { "title" => "hello, world" }
+```

data/avro_turf.gemspec

@@ -18,9 +18,12 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "avro", "~> 1.7.7"
+  spec.add_dependency "excon", "~> 0.45.4"
 
   spec.add_development_dependency "bundler", "~> 1.7"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.2.0"
-  spec.add_development_dependency "fakefs"
+  spec.add_development_dependency "fakefs", "~> 0.6.7"
+  spec.add_development_dependency "webmock"
+  spec.add_development_dependency "sinatra"
 end

data/lib/avro_turf/cached_schema_registry.rb

@@ -0,0 +1,16 @@
+# Caches registrations and lookups to the schema registry in memory.
+class AvroTurf::CachedSchemaRegistry
+  def initialize(upstream)
+    @upstream = upstream
+    @schemas_by_id = {}
+    @ids_by_schema = {}
+  end
+
+  def fetch(id)
+    @schemas_by_id[id] ||= @upstream.fetch(id)
+  end
+
+  def register(subject, schema)
+    @ids_by_schema[subject + schema.to_s] ||= @upstream.register(subject, schema)
+  end
+end

data/lib/avro_turf/messaging.rb

@@ -0,0 +1,95 @@
+require 'logger'
+require 'avro_turf'
+require 'avro_turf/schema_store'
+require 'avro_turf/schema_registry'
+require 'avro_turf/cached_schema_registry'
+
+class AvroTurf
+
+  # Provides a way to encode and decode messages without having to embed schemas
+  # in the encoded data. Confluent's Schema Registry[1] is used to register
+  # a schema when encoding a message -- the registry will issue a schema id that
+  # will be included in the encoded data alongside the actual message. When
+  # decoding the data, the schema id will be used to look up the writer's schema
+  # from the registry.
+  #
+  # 1: https://github.com/confluentinc/schema-registry
+  class Messaging
+    MAGIC_BYTE = [0].pack("C").freeze
+
+    # Instantiate a new Messaging instance with the given configuration.
+    #
+    # registry_url - The String URL of the schema registry that should be used.
+    # schemas_path - The String file system path where local schemas are stored.
+    # namespace    - The String default schema namespace.
+    # logger       - The Logger that should be used to log information (optional).
+    def initialize(registry_url: nil, schemas_path: nil, namespace: nil, logger: nil)
+      @logger = logger || Logger.new($stderr)
+      @namespace = namespace
+      @schema_store = SchemaStore.new(path: schemas_path || DEFAULT_SCHEMAS_PATH)
+      @registry = CachedSchemaRegistry.new(SchemaRegistry.new(registry_url, logger: @logger))
+    end
+
+    # Encodes a message using the specified schema.
+    #
+    # message     - The message that should be encoded. Must be compatible with
+    #               the schema.
+    # schema_name - The String name of the schema that should be used to encode
+    #               the data.
+    # namespace   - The namespace of the schema (optional).
+    #
+    # Returns the encoded data as a String.
+    def encode(message, schema_name: nil, namespace: @namespace)
+      schema = @schema_store.find(schema_name, namespace)
+
+      # Schemas are registered under the full name of the top level Avro record
+      # type.
+      schema_id = @registry.register(schema.fullname, schema)
+
+      stream = StringIO.new
+      writer = Avro::IO::DatumWriter.new(schema)
+      encoder = Avro::IO::BinaryEncoder.new(stream)
+
+      # Always start with the magic byte.
+      encoder.write(MAGIC_BYTE)
+
+      # The schema id is encoded as a 4-byte big-endian integer.
+      encoder.write([schema_id].pack("N"))
+
+      # The actual message comes last.
+      writer.write(message, encoder)
+
+      stream.string
+    end
+
+    # Decodes data into the original message.
+    #
+    # data        - A String containing encoded data.
+    # schema_name - The String name of the schema that should be used to decode
+    #               the data. Must match the schema used when encoding (optional).
+    # namespace   - The namespace of the schema (optional).
+    #
+    # Returns the decoded message.
+    def decode(data, schema_name: nil, namespace: @namespace)
+      readers_schema = schema_name && @schema_store.find(schema_name, namespace)
+      stream = StringIO.new(data)
+      decoder = Avro::IO::BinaryDecoder.new(stream)
+
+      # The first byte is MAGIC!!!
+      magic_byte = decoder.read(1)
+
+      if magic_byte != MAGIC_BYTE
+        raise "Expected data to begin with a magic byte, got `#{magic_byte.inspect}`"
+      end
+
+      # The schema id is a 4-byte big-endian integer.
+      schema_id = decoder.read(4).unpack("N").first
+
+      writers_schema_json = @registry.fetch(schema_id)
+      writers_schema = Avro::Schema.parse(writers_schema_json)
+
+      reader = Avro::IO::DatumReader.new(writers_schema, readers_schema)
+      reader.read(decoder)
+    end
+  end
+end

data/lib/avro_turf/schema_registry.rb

@@ -0,0 +1,45 @@
+require 'excon'
+
+class AvroTurf::SchemaRegistry
+  CONTENT_TYPE = "application/vnd.schemaregistry.v1+json".freeze
+
+  def initialize(url, logger: Logger.new($stdout))
+    @logger = logger
+    @connection = Excon.new(url, headers: {
+      "Content-Type" => CONTENT_TYPE,
+    })
+  end
+
+  def fetch(id)
+    @logger.info "Fetching schema with id #{id}"
+    data = get("/schemas/ids/#{id}")
+    data.fetch("schema")
+  end
+
+  def register(subject, schema)
+    data = post("/subjects/#{subject}/versions", body: {
+      schema: schema
+    }.to_json)
+
+    id = data.fetch("id")
+
+    @logger.info "Registered schema for subject `#{subject}`; id = #{id}"
+
+    id
+  end
+
+  private
+
+  def get(path, **options)
+    request(path, method: :get, **options)
+  end
+
+  def post(path, **options)
+    request(path, method: :post, **options)
+  end
+
+  def request(path, **options)
+    response = @connection.request(path: path, expects: 200, **options)
+    JSON.parse(response.body)
+  end
+end

data/lib/avro_turf/version.rb

@@ -1,3 +1,3 @@
 class AvroTurf
-  VERSION = "0.5.0"
+  VERSION = "0.6.0"
 end

data/spec/fake_schema_registry_server.rb

@@ -0,0 +1,25 @@
+require 'sinatra/base'
+
+class FakeSchemaRegistryServer < Sinatra::Base
+  SCHEMAS = []
+
+  post "/subjects/:subject/versions" do
+    request.body.rewind
+    schema = JSON.parse(request.body.read).fetch("schema")
+
+    SCHEMAS << schema
+    schema_id = SCHEMAS.size - 1
+
+    { id: schema_id }.to_json
+  end
+
+  get "/schemas/ids/:schema_id" do
+    schema = SCHEMAS.at(params[:schema_id].to_i)
+
+    { schema: schema }.to_json
+  end
+
+  def self.clear
+    SCHEMAS.clear
+  end
+end

data/spec/messaging_spec.rb

@@ -0,0 +1,52 @@
+require 'webmock/rspec'
+require 'avro_turf/messaging'
+require_relative 'fake_schema_registry_server'
+
+describe AvroTurf::Messaging do
+  let(:registry_url) { "http://registry.example.com" }
+  let(:logger) { Logger.new(StringIO.new) }
+
+  let(:avro) {
+    AvroTurf::Messaging.new(
+      registry_url: registry_url,
+      schemas_path: "spec/schemas",
+      logger: logger
+    )
+  }
+
+  before do
+    FileUtils.mkdir_p("spec/schemas")
+  end
+
+  before do
+    stub_request(:any, /^#{registry_url}/).to_rack(FakeSchemaRegistryServer)
+    FakeSchemaRegistryServer.clear
+  end
+
+  before do
+    define_schema "person.avsc", <<-AVSC
+      {
+        "name": "person",
+        "type": "record",
+        "fields": [
+          {
+            "type": "string",
+            "name": "full_name"
+          }
+        ]
+      }
+    AVSC
+  end
+
+  it "encodes and decodes messages" do
+    message = { "full_name" => "John Doe" }
+    data = avro.encode(message, schema_name: "person")
+    expect(avro.decode(data)).to eq message
+  end
+
+  it "allows specifying a reader's schema" do
+    message = { "full_name" => "John Doe" }
+    data = avro.encode(message, schema_name: "person")
+    expect(avro.decode(data, schema_name: "person")).to eq message
+  end
+end

data/spec/schema_registry_spec.rb

@@ -0,0 +1,32 @@
+require 'webmock/rspec'
+require 'avro_turf/schema_registry'
+require_relative 'fake_schema_registry_server'
+
+describe AvroTurf::SchemaRegistry do
+  let(:registry_url) { "http://registry.example.com" }
+
+  before do
+    stub_request(:any, /^#{registry_url}/).to_rack(FakeSchemaRegistryServer)
+    FakeSchemaRegistryServer.clear
+  end
+
+  it "allows registering a schema" do
+    logger = Logger.new(StringIO.new)
+    registry = described_class.new(registry_url, logger: logger)
+
+    schema = <<-JSON
+      {
+        "type": "record",
+        "name": "person",
+        "fields": [
+          { "name": "name", "type": "string" }
+        ]
+      }
+    JSON
+
+    id = registry.register("some-subject", schema)
+    fetched_schema = registry.fetch(id)
+
+    expect(JSON.parse(fetched_schema)).to eq JSON.parse(schema)
+  end
+end

data/spec/spec_helper.rb

@@ -1,4 +1,5 @@
 require 'bundler/setup'
+require 'logger'
 require 'fakefs/spec_helpers'
 require 'avro_turf'
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: avro_turf
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Daniel Schierbeck
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.7.7
+- !ruby/object:Gem::Dependency
+  name: excon
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.45.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.45.4
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -68,6 +82,34 @@ dependencies:
         version: 3.2.0
 - !ruby/object:Gem::Dependency
   name: fakefs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.7
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.7
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sinatra
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -96,6 +138,7 @@ files:
 - avro_turf.gemspec
 - circle.yml
 - lib/avro_turf.rb
+- lib/avro_turf/cached_schema_registry.rb
 - lib/avro_turf/core_ext.rb
 - lib/avro_turf/core_ext/date.rb
 - lib/avro_turf/core_ext/enumerable.rb
@@ -107,6 +150,8 @@ files:
 - lib/avro_turf/core_ext/symbol.rb
 - lib/avro_turf/core_ext/time.rb
 - lib/avro_turf/core_ext/true_class.rb
+- lib/avro_turf/messaging.rb
+- lib/avro_turf/schema_registry.rb
 - lib/avro_turf/schema_store.rb
 - lib/avro_turf/version.rb
 - perf/address.avsc
@@ -124,6 +169,9 @@ files:
 - spec/core_ext/symbol_spec.rb
 - spec/core_ext/time_spec.rb
 - spec/core_ext/true_class_spec.rb
+- spec/fake_schema_registry_server.rb
+- spec/messaging_spec.rb
+- spec/schema_registry_spec.rb
 - spec/schema_store_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/dasch/avro_turf
@@ -163,6 +211,9 @@ test_files:
 - spec/core_ext/symbol_spec.rb
 - spec/core_ext/time_spec.rb
 - spec/core_ext/true_class_spec.rb
+- spec/fake_schema_registry_server.rb
+- spec/messaging_spec.rb
+- spec/schema_registry_spec.rb
 - spec/schema_store_spec.rb
 - spec/spec_helper.rb
 has_rdoc: