avro_turf_enchanced 0.7.3

data/spec/schema_registry_spec.rb

@@ -0,0 +1,9 @@
+require 'webmock/rspec'
+require 'avro_turf/schema_registry'
+require 'avro_turf/test/fake_schema_registry_server'
+
+describe AvroTurf::SchemaRegistry do
+  it_behaves_like "a schema registry client" do
+    let(:registry) { described_class.new(registry_url, logger: logger) }
+  end
+end

data/spec/schema_store_spec.rb

@@ -0,0 +1,253 @@
+require 'avro_turf/schema_store'
+
+describe AvroTurf::SchemaStore do
+  let(:store) { AvroTurf::SchemaStore.new(path: "spec/schemas") }
+
+  before do
+    FileUtils.mkdir_p("spec/schemas")
+  end
+
+  describe "#find" do
+    it "finds schemas on the file system" do
+      define_schema "message.avsc", <<-AVSC
+        {
+          "name": "message",
+          "type": "record",
+          "fields": [
+            {
+              "type": "string",
+              "name": "message"
+            }
+          ]
+        }
+      AVSC
+
+      schema = store.find("message")
+      expect(schema.fullname).to eq "message"
+    end
+
+    it "resolves missing references" do
+      define_schema "person.avsc", <<-AVSC
+        {
+          "name": "person",
+          "type": "record",
+          "fields": [
+            {
+              "name": "address",
+              "type": "address"
+            }
+          ]
+        }
+      AVSC
+
+      define_schema "address.avsc", <<-AVSC
+        {
+          "type": "record",
+          "name": "address",
+          "fields": []
+        }
+      AVSC
+
+      schema = store.find("person")
+
+      expect(schema.fullname).to eq "person"
+    end
+
+    it "finds namespaced schemas" do
+      FileUtils.mkdir_p("spec/schemas/test/people")
+
+      define_schema "test/people/person.avsc", <<-AVSC
+        {
+          "name": "person",
+          "namespace": "test.people",
+          "type": "record",
+          "fields": [
+            {
+              "name": "address",
+              "type": "test.people.address"
+            }
+          ]
+        }
+      AVSC
+
+      define_schema "test/people/address.avsc", <<-AVSC
+        {
+          "name": "address",
+          "namespace": "test.people",
+          "type": "record",
+          "fields": []
+        }
+      AVSC
+
+      schema = store.find("person", "test.people")
+
+      expect(schema.fullname).to eq "test.people.person"
+    end
+
+    it "ignores the namespace when the name contains a dot" do
+      FileUtils.mkdir_p("spec/schemas/test/acme")
+
+      define_schema "test/acme/message.avsc", <<-AVSC
+        {
+          "name": "message",
+          "namespace": "test.acme",
+          "type": "record",
+          "fields": []
+        }
+      AVSC
+
+      schema = store.find("test.acme.message", "test.yolo")
+
+      expect(schema.fullname).to eq "test.acme.message"
+    end
+
+    it "raises AvroTurf::SchemaNotFoundError if there's no schema file matching the name" do
+      expect {
+        store.find("not_there")
+      }.to raise_error(AvroTurf::SchemaNotFoundError, "could not find Avro schema at `spec/schemas/not_there.avsc'")
+    end
+
+    it "raises AvroTurf::SchemaNotFoundError if a type reference cannot be resolved" do
+      define_schema "person.avsc", <<-AVSC
+        {
+          "name": "person",
+          "type": "record",
+          "fields": [
+            {
+              "name": "address",
+              "type": "address"
+            }
+          ]
+        }
+      AVSC
+
+      expect {
+        store.find("person")
+      }.to raise_exception(AvroTurf::SchemaNotFoundError)
+    end
+
+    it "raises AvroTurf::SchemaError if the schema's namespace doesn't match the file location" do
+      FileUtils.mkdir_p("spec/schemas/test/people")
+
+      define_schema "test/people/person.avsc", <<-AVSC
+        {
+          "name": "person",
+          "namespace": "yoyoyo.nanana",
+          "type": "record",
+          "fields": []
+        }
+      AVSC
+
+      expect {
+        store.find("test.people.person")
+      }.to raise_error(AvroTurf::SchemaError, "expected schema `spec/schemas/test/people/person.avsc' to define type `test.people.person'")
+    end
+
+    it "handles circular dependencies" do
+      define_schema "a.avsc", <<-AVSC
+        {
+          "name": "a",
+          "type": "record",
+          "fields": [
+            {
+              "type": "b",
+              "name": "b"
+            }
+          ]
+        }
+      AVSC
+
+      define_schema "b.avsc", <<-AVSC
+        {
+          "name": "b",
+          "type": "record",
+          "fields": [
+            {
+              "type": "a",
+              "name": "a"
+            }
+          ]
+        }
+      AVSC
+
+      schema = store.find("a")
+      expect(schema.fullname).to eq "a"
+    end
+
+    it "caches schemas in memory" do
+      define_schema "person.avsc", <<-AVSC
+        {
+          "name": "person",
+          "type": "record",
+          "fields": [
+            {
+              "type": "string",
+              "name": "full_name"
+            }
+          ]
+        }
+      AVSC
+
+      # Warm the schema cache.
+      store.find("person")
+
+      # Force a failure if the schema file is read again.
+      FileUtils.rm("spec/schemas/person.avsc")
+
+      schema = store.find("person")
+      expect(schema.fullname).to eq "person"
+    end
+  end
+
+  describe "#load_schemas!" do
+    it "loads schemas defined in the `schemas_path` directory" do
+      define_schema "person.avsc", <<-AVSC
+        {
+          "name": "person",
+          "type": "record",
+          "fields": [
+            {
+              "type": "string",
+              "name": "full_name"
+            }
+          ]
+        }
+      AVSC
+
+      # Warm the schema cache.
+      store.load_schemas!
+
+      # Force a failure if the schema file is read again.
+      FileUtils.rm("spec/schemas/person.avsc")
+
+      schema = store.find("person")
+      expect(schema.fullname).to eq "person"
+    end
+
+    it "recursively finds schema definitions in subdirectories" do
+      FileUtils.mkdir_p("spec/schemas/foo/bar")
+
+      define_schema "foo/bar/person.avsc", <<-AVSC
+        {
+          "name": "foo.bar.person",
+          "type": "record",
+          "fields": [
+            {
+              "type": "string",
+              "name": "full_name"
+            }
+          ]
+        }
+      AVSC
+
+      # Warm the schema cache.
+      store.load_schemas!
+
+      # Force a failure if the schema file is read again.
+      FileUtils.rm("spec/schemas/foo/bar/person.avsc")
+
+      schema = store.find("foo.bar.person")
+      expect(schema.fullname).to eq "foo.bar.person"
+    end
+  end
+end

data/spec/schema_to_avro_patch_spec.rb

@@ -0,0 +1,66 @@
+require 'webmock/rspec'
+
+# This spec verifies the monkey-patch that we have to apply until the avro
+# gem releases a fix for bug AVRO-1848:
+# https://issues.apache.org/jira/browse/AVRO-1848
+
+describe Avro::Schema do
+  it "correctly handles falsey field defaults" do
+    schema = Avro::Schema.parse <<-SCHEMA
+      {"type": "record", "name": "Record", "namespace": "my.name.space",
+        "fields": [
+          {"name": "is_usable", "type": "boolean", "default": false}
+        ]
+      }
+    SCHEMA
+    
+    expect(schema.to_avro).to eq({
+      'type' => 'record', 'name' => 'Record', 'namespace' => 'my.name.space',
+      'fields' => [
+        {'name' => 'is_usable', 'type' => 'boolean', 'default' => false}
+      ]
+    })
+  end
+end
+
+
+describe Avro::IO::DatumReader do
+  let(:writer_schema) do
+    Avro::Schema.parse <<-AVSC
+      {
+        "name": "no_default",
+        "type": "record",
+        "fields": [
+          { "type": "string", "name": "one" }
+        ]
+      }
+    AVSC
+  end
+  let(:reader_schema) do
+    Avro::Schema.parse <<-AVSC
+      {
+        "name": "no_default",
+        "type": "record",
+        "fields": [
+          { "type": "string", "name": "one" },
+          { "type": "string", "name": "two" }
+        ]
+      }
+    AVSC
+  end
+
+  it "raises an error for missing fields without a default" do
+    stream = StringIO.new
+    writer = Avro::IO::DatumWriter.new(writer_schema)
+    encoder = Avro::IO::BinaryEncoder.new(stream)
+    writer.write({ 'one' => 'first' }, encoder)
+    encoded = stream.string
+
+    stream = StringIO.new(encoded)
+    decoder = Avro::IO::BinaryDecoder.new(stream)
+    reader = Avro::IO::DatumReader.new(writer_schema, reader_schema)
+    expect do
+      reader.read(decoder)
+    end.to raise_error(Avro::AvroError, 'Missing data for "string" with no default')
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,20 @@
+require 'bundler/setup'
+require 'logger'
+require 'json_spec'
+require 'fakefs/spec_helpers'
+require 'avro_turf'
+
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each { |f| require f }
+
+module Helpers
+  def define_schema(path, content)
+    File.open(File.join("spec/schemas", path), "w") do |f|
+      f.write(content)
+    end
+  end
+end
+
+RSpec.configure do |config|
+  config.include FakeFS::SpecHelpers
+  config.include Helpers
+end

data/spec/support/schema_registry_context.rb

@@ -0,0 +1,190 @@
+# This shared example expects a registry variable to be defined
+# with an instance of the registry class being tested.
+shared_examples_for "a schema registry client" do
+  let(:logger) { Logger.new(StringIO.new) }
+  let(:registry_url) { "http://registry.example.com" }
+  let(:subject_name) { "some-subject" }
+  let(:schema) do
+    {
+      type: "record",
+      name: "person",
+      fields: [
+        { name: "name", type: "string" }
+      ]
+    }.to_json
+  end
+
+  before do
+    stub_request(:any, /^#{registry_url}/).to_rack(FakeSchemaRegistryServer)
+    FakeSchemaRegistryServer.clear
+  end
+
+  describe "#register and #fetch" do
+    it "allows registering a schema" do
+      id = registry.register(subject_name, schema)
+      fetched_schema = registry.fetch(id)
+
+      expect(fetched_schema).to eq(schema)
+    end
+
+    context "with an Avro::Schema" do
+      let(:avro_schema) { Avro::Schema.parse(schema) }
+
+      it "allows registration using an Avro::Schema" do
+        id = registry.register(subject_name, avro_schema)
+        expect(registry.fetch(id)).to eq(avro_schema.to_s)
+      end
+
+      context "with ActiveSupport present" do
+        before do
+          break_to_json(avro_schema)
+        end
+
+        it "allows registering an Avro schema" do
+          id = registry.register(subject_name, avro_schema)
+          expect(registry.fetch(id)).to eq(avro_schema.to_s)
+        end
+      end
+    end
+  end
+
+  describe "#fetch" do
+    context "when the schema does not exist" do
+      it "raises an error" do
+        expect do
+          registry.fetch(-1)
+        end.to raise_error(Excon::Errors::NotFound)
+      end
+    end
+  end
+
+  describe "#subjects" do
+    it "lists the subjects in the registry" do
+      subjects = Array.new(2) { |n| "subject#{n}" }
+      subjects.each { |subject| registry.register(subject, schema) }
+      expect(registry.subjects).to be_json_eql(subjects.to_json)
+    end
+  end
+
+  describe "#subject_versions" do
+    it "lists all the versions for the subject" do
+      2.times do |n|
+        registry.register(subject_name,
+                          { type: :record, name: "r#{n}", fields: [] }.to_json)
+      end
+      expect(registry.subject_versions(subject_name))
+        .to be_json_eql((1..2).to_a.to_json)
+    end
+
+    context "when the subject does not exist" do
+      let(:subject_name) { 'missing' }
+
+      it "raises an error" do
+        expect do
+          registry.subject_versions(subject_name).inspect
+        end.to raise_error(Excon::Errors::NotFound)
+      end
+    end
+  end
+
+  describe "#subject_version" do
+    before do
+      2.times do |n|
+        registry.register(subject_name,
+                          { type: :record, name: "r#{n}", fields: [] }.to_json)
+      end
+    end
+    let(:expected) do
+      {
+        name: subject_name,
+        version: 1,
+        schema: { type: :record, name: "r0", fields: [] }.to_json
+      }.to_json
+    end
+
+    it "returns a specific version of a schema" do
+      expect(registry.subject_version(subject_name, 1))
+        .to eq(JSON.parse(expected))
+    end
+
+    context "when the version is not specified" do
+      let(:expected) do
+        {
+          name: subject_name,
+          version: 2,
+          schema: { type: :record, name: "r1", fields: [] }.to_json
+        }.to_json
+      end
+
+      it "returns the latest version" do
+        expect(registry.subject_version(subject_name))
+          .to eq(JSON.parse(expected))
+      end
+    end
+
+    context "when the subject does not exist" do
+      it "raises an error" do
+        expect do
+          registry.subject_version('missing')
+        end.to raise_error(Excon::Errors::NotFound)
+      end
+    end
+
+    context "when the version does not exist" do
+      it "raises an error" do
+        expect do
+          registry.subject_version(subject_name, 3)
+        end.to raise_error(Excon::Errors::NotFound)
+      end
+    end
+  end
+
+  describe "#check" do
+    context "when the schema exists" do
+      let!(:schema_id) { registry.register(subject_name, schema) }
+      let(:expected) do
+        {
+          subject: subject_name,
+          id: schema_id,
+          version: 1,
+          schema: schema
+        }.to_json
+      end
+      it "returns the schema details" do
+        expect(registry.check(subject_name, schema)).to eq(JSON.parse(expected))
+      end
+
+      context "with an Avro::Schema" do
+        let(:avro_schema) { Avro::Schema.parse(schema) }
+
+        it "supports a check using an Avro schema" do
+          expect(registry.check(subject_name, avro_schema)).to eq(JSON.parse(expected))
+        end
+
+        context "with ActiveSupport present" do
+          before { break_to_json(avro_schema) }
+
+          it "supports a check using an Avro schema" do
+            expect(registry.check(subject_name, avro_schema)).to eq(JSON.parse(expected))
+          end
+        end
+      end
+    end
+
+    context "when the schema is not registered" do
+      it "returns nil" do
+        expect(registry.check("missing", schema)).to be_nil
+      end
+    end
+  end
+
+  # Monkey patch an Avro::Schema to simulate the presence of
+  # active_support/core_ext.
+  def break_to_json(avro_schema)
+    def avro_schema.to_json(*args)
+      instance_variables.each_with_object(Hash.new) do |ivar, result|
+        result[ivar.to_s.sub('@', '')] = instance_variable_get(ivar)
+      end.to_json(*args)
+    end
+  end
+end