avro_turf_enchanced 0.7.3

data/perf/encoding_size.rb

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+#
+# Measures the encoded size of messages of increasing size.
+
+$LOAD_PATH.unshift(File.expand_path("../lib", File.dirname(__FILE__)))
+
+require 'benchmark'
+require 'avro_turf'
+
+sizes = [1, 10, 100, 1_000, 10_000]
+avro = AvroTurf.new(schemas_path: File.dirname(__FILE__))
+
+sizes.each do |size|
+  data = {
+    "name" => "John" * size,
+    "address" => {
+      "street" => "1st st." * size,
+      "city" => "Citytown" * size
+    }
+  }
+
+  result = avro.encode(data, schema_name: "person")
+  encoded_size = result.bytesize
+  encode_factor = result.bytesize / size.to_f
+  puts "size #{size}: #{encoded_size} bytes (encoding factor #{encode_factor})"
+end

data/perf/encoding_speed.rb

@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+#
+# Measures the time it takes to encode messages of increasing size.
+
+$LOAD_PATH.unshift(File.expand_path("../lib", File.dirname(__FILE__)))
+
+require 'benchmark'
+require 'avro_turf'
+
+# Number of iterations per run.
+N = 10_000
+
+Benchmark.bm(15) do |x|
+  sizes = [1, 10, 100, 1_000, 10_000]
+  avro = AvroTurf.new(schemas_path: File.dirname(__FILE__))
+
+  sizes.each do |size|
+    data = {
+      "name" => "John" * size,
+      "address" => {
+        "street" => "1st st." * size,
+        "city" => "Citytown" * size
+      }
+    }
+
+    x.report("size #{size}:") do
+      N.times { avro.encode(data, schema_name: "person") }
+    end
+  end
+end

data/perf/person.avsc

@@ -0,0 +1,14 @@
+{
+  "name": "person",
+  "type": "record",
+  "fields": [
+    {
+      "name": "name",
+      "type": "string"
+    },
+    {
+      "name": "address",
+      "type": "address"
+    }
+  ]
+}

data/spec/avro_turf_spec.rb

@@ -0,0 +1,161 @@
+describe AvroTurf do
+  let(:avro) { AvroTurf.new(schemas_path: "spec/schemas/") }
+
+  before do
+    FileUtils.mkdir_p("spec/schemas")
+  end
+
+  describe "#encode" do
+    before do
+      define_schema "person.avsc", <<-AVSC
+        {
+          "name": "person",
+          "type": "record",
+          "fields": [
+            {
+              "type": "string",
+              "name": "full_name"
+            }
+          ]
+        }
+      AVSC
+    end
+
+    it "encodes data with Avro" do
+      data = {
+        "full_name" => "John Doe"
+      }
+
+      encoded_data = avro.encode(data, schema_name: "person")
+
+      expect(avro.decode(encoded_data)).to eq(data)
+    end
+
+    it "allows specifying a codec that should be used to compress messages" do
+      compressed_avro = AvroTurf.new(schemas_path: "spec/schemas/", codec: "deflate")
+
+      data = {
+        "full_name" => "John Doe" * 100
+      }
+
+      uncompressed_data = avro.encode(data, schema_name: "person")
+      compressed_data = compressed_avro.encode(data, schema_name: "person")
+
+      expect(compressed_data.bytesize).to be < uncompressed_data.bytesize
+      expect(compressed_avro.decode(compressed_data)).to eq(data)
+    end
+  end
+
+  describe "#decode" do
+    it "decodes Avro data using the inlined writer's schema" do
+      define_schema "message.avsc", <<-AVSC
+        {
+          "name": "message",
+          "type": "string"
+        }
+      AVSC
+
+      encoded_data = avro.encode("hello, world", schema_name: "message")
+
+      expect(avro.decode(encoded_data)).to eq "hello, world"
+    end
+
+    it "decodes Avro data using a specified reader's schema" do
+      FileUtils.mkdir_p("spec/schemas/reader")
+
+      define_schema "point.avsc", <<-AVSC
+        {
+          "name": "point",
+          "type": "record",
+          "fields": [
+            { "name": "x", "type": "long" },
+            { "name": "y", "type": "long" }
+          ]
+        }
+      AVSC
+
+      define_schema "reader/point.avsc", <<-AVSC
+        {
+          "name": "point",
+          "type": "record",
+          "fields": [
+            { "name": "x", "type": "long" }
+          ]
+        }
+      AVSC
+
+      encoded_data = avro.encode({ "x" => 42, "y" => 13 }, schema_name: "point")
+      reader_avro = AvroTurf.new(schemas_path: "spec/schemas/reader")
+
+      expect(reader_avro.decode(encoded_data, schema_name: "point")).to eq({ "x" => 42 })
+    end
+  end
+
+  describe "#encode_to_stream" do
+    it "writes encoded data to an existing stream" do
+      define_schema "message.avsc", <<-AVSC
+        {
+          "name": "message",
+          "type": "string"
+        }
+      AVSC
+
+      stream = StringIO.new
+      avro.encode_to_stream("hello", stream: stream, schema_name: "message")
+
+      expect(avro.decode(stream.string)).to eq "hello"
+    end
+  end
+
+  describe "#decode_stream" do
+    it "decodes Avro data from a stream" do
+      define_schema "message.avsc", <<-AVSC
+        {
+          "name": "message",
+          "type": "string"
+        }
+      AVSC
+
+      encoded_data = avro.encode("hello", schema_name: "message")
+      stream = StringIO.new(encoded_data)
+
+      expect(avro.decode_stream(stream)).to eq "hello"
+    end
+  end
+
+  describe "#valid?" do
+    before do
+      define_schema "message.avsc", <<-AVSC
+        {
+          "name": "message",
+          "type": "string"
+        }
+      AVSC
+    end
+
+    it "returns true if the datum matches the schema" do
+      datum = "hello"
+      expect(avro.valid?(datum, schema_name: "message")).to eq true
+    end
+
+    it "returns false if the datum does not match the schema" do
+      datum = 42
+      expect(avro.valid?(datum, schema_name: "message")).to eq false
+    end
+
+    it "handles symbol keys in hashes" do
+      define_schema "postcard.avsc", <<-AVSC
+        {
+          "name": "postcard",
+          "type": "record",
+          "fields": [
+            { "name": "message", "type": "string" }
+          ]
+        }
+      AVSC
+
+      datum = { message: "hello" }
+      expect(avro.valid?(datum, schema_name: "postcard")).to eq true
+    end
+  end
+end

data/spec/cached_schema_registry_spec.rb

@@ -0,0 +1,41 @@
+require 'webmock/rspec'
+require 'avro_turf/cached_schema_registry'
+require 'avro_turf/test/fake_schema_registry_server'
+
+describe AvroTurf::CachedSchemaRegistry do
+  let(:upstream) { instance_double(AvroTurf::SchemaRegistry) }
+  let(:registry) { described_class.new(upstream) }
+  let(:id) { rand(999) }
+  let(:schema) do
+    {
+      type: "record",
+      name: "person",
+      fields: [{ name: "name", type: "string" }]
+    }.to_json
+  end
+
+  describe "#fetch" do
+    it "caches the result of fetch" do
+      allow(upstream).to receive(:fetch).with(id).and_return(schema)
+      registry.fetch(id)
+      expect(registry.fetch(id)).to eq(schema)
+      expect(upstream).to have_received(:fetch).exactly(1).times
+    end
+  end
+
+  describe "#register" do
+    let(:subject_name) { "a_subject" }
+
+    it "caches the result of register" do
+      allow(upstream).to receive(:register).with(subject_name, schema).and_return(id)
+      registry.register(subject_name, schema)
+      expect(registry.register(subject_name, schema)).to eq(id)
+      expect(upstream).to have_received(:register).exactly(1).times
+    end
+  end
+
+  it_behaves_like "a schema registry client" do
+    let(:upstream) { AvroTurf::SchemaRegistry.new(registry_url, logger: logger) }
+    let(:registry) { described_class.new(upstream) }
+  end
+end

data/spec/core_ext/date_spec.rb

@@ -0,0 +1,6 @@
+describe Date, "#as_avro" do
+  it "returns an ISO8601 string describing the time" do
+    date = Date.today
+    expect(date.as_avro).to eq(date.iso8601)
+  end
+end

data/spec/core_ext/enumerable_spec.rb

@@ -0,0 +1,12 @@
+describe Enumerable, "#as_avro" do
+  it "returns an array" do
+    expect(Set.new.as_avro).to eq []
+  end
+
+  it "coerces the items to Avro" do
+    x = double(as_avro: "x")
+    y = double(as_avro: "y")
+
+    expect([x, y].as_avro).to eq ["x", "y"]
+  end
+end

data/spec/core_ext/false_class_spec.rb

@@ -0,0 +1,5 @@
+describe FalseClass, "#as_avro" do
+  it "returns itself" do
+    expect(false.as_avro).to eq false
+  end
+end

data/spec/core_ext/hash_spec.rb

@@ -0,0 +1,8 @@
+describe Hash, "#as_avro" do
+  it "coerces the keys and values to Avro" do
+    x = double(as_avro: "x")
+    y = double(as_avro: "y")
+
+    expect({ x => y }.as_avro).to eq({ "x" => "y" })
+  end
+end

data/spec/core_ext/nil_class_spec.rb

@@ -0,0 +1,5 @@
+describe NilClass, "#as_avro" do
+  it "returns itself" do
+    expect(nil.as_avro).to eq nil
+  end
+end

data/spec/core_ext/numeric_spec.rb

@@ -0,0 +1,6 @@
+describe Numeric, "#as_avro" do
+  it "returns the number itself" do
+    expect(42.as_avro).to eq 42
+    expect(4.2.as_avro).to eq 4.2
+  end
+end

data/spec/core_ext/string_spec.rb

@@ -0,0 +1,5 @@
+describe String, "#as_avro" do
+  it "returns itself" do
+    expect("hello".as_avro).to eq "hello"
+  end
+end

data/spec/core_ext/symbol_spec.rb

@@ -0,0 +1,5 @@
+describe Symbol, "#as_avro" do
+  it "returns the String representation of the Symbol" do
+    expect(:hello.as_avro).to eq("hello")
+  end
+end

data/spec/core_ext/time_spec.rb

@@ -0,0 +1,6 @@
+describe Time, "#as_avro" do
+  it "returns an ISO8601 string describing the time" do
+    time = Time.now
+    expect(time.as_avro).to eq(time.iso8601)
+  end
+end

data/spec/core_ext/true_class_spec.rb

@@ -0,0 +1,5 @@
+describe TrueClass, "#as_avro" do
+  it "returns itself" do
+    expect(true.as_avro).to eq true
+  end
+end

data/spec/messaging_spec.rb

@@ -0,0 +1,112 @@
+require 'webmock/rspec'
+require 'avro_turf/messaging'
+require 'avro_turf/test/fake_schema_registry_server'
+
+describe AvroTurf::Messaging do
+  let(:registry_url) { "http://registry.example.com" }
+  let(:logger) { Logger.new(StringIO.new) }
+
+  let(:avro) {
+    AvroTurf::Messaging.new(
+      registry_url: registry_url,
+      schemas_path: "spec/schemas",
+      logger: logger
+    )
+  }
+
+  let(:message) { { "full_name" => "John Doe" } }
+
+  before do
+    FileUtils.mkdir_p("spec/schemas")
+  end
+
+  before do
+    stub_request(:any, /^#{registry_url}/).to_rack(FakeSchemaRegistryServer)
+    FakeSchemaRegistryServer.clear
+  end
+
+  before do
+    define_schema "person.avsc", <<-AVSC
+      {
+        "name": "person",
+        "type": "record",
+        "fields": [
+          {
+            "type": "string",
+            "name": "full_name"
+          }
+        ]
+      }
+    AVSC
+  end
+
+  shared_examples_for "encoding and decoding" do
+    it "encodes and decodes messages" do
+      data = avro.encode(message, schema_name: "person")
+      expect(avro.decode(data)).to eq message
+    end
+
+    it "allows specifying a reader's schema" do
+      data = avro.encode(message, schema_name: "person")
+      expect(avro.decode(data, schema_name: "person")).to eq message
+    end
+
+    it "caches parsed schemas for decoding" do
+      data = avro.encode(message, schema_name: "person")
+      avro.decode(data)
+      allow(Avro::Schema).to receive(:parse).and_call_original
+      expect(avro.decode(data)).to eq message
+      expect(Avro::Schema).not_to have_received(:parse)
+    end
+  end
+
+  it_behaves_like "encoding and decoding"
+
+  context "with a provided registry" do
+    let(:registry) { AvroTurf::SchemaRegistry.new(registry_url, logger: logger) }
+
+    let(:avro) do
+      AvroTurf::Messaging.new(
+        registry: registry,
+        schemas_path: "spec/schemas",
+        logger: logger
+      )
+    end
+
+    it_behaves_like "encoding and decoding"
+
+    it "uses the provided registry" do
+      allow(registry).to receive(:register).and_call_original
+      message = { "full_name" => "John Doe" }
+      avro.encode(message, schema_name: "person")
+      expect(registry).to have_received(:register).with("person", anything)
+    end
+
+    it "allows specifying a schema registry subject" do
+      allow(registry).to receive(:register).and_call_original
+      message = { "full_name" => "John Doe" }
+      avro.encode(message, schema_name: "person", subject: "people")
+      expect(registry).to have_received(:register).with("people", anything)
+    end
+  end
+
+  context "with a provided schema store" do
+    let(:schema_store) { AvroTurf::SchemaStore.new(path: "spec/schemas") }
+
+    let(:avro) do
+      AvroTurf::Messaging.new(
+        registry_url: registry_url,
+        schema_store: schema_store,
+        logger: logger
+      )
+    end
+
+    it_behaves_like "encoding and decoding"
+
+    it "uses the provided schema store" do
+      allow(schema_store).to receive(:find).and_call_original
+      avro.encode(message, schema_name: "person")
+      expect(schema_store).to have_received(:find).with("person", nil)
+    end
+  end
+end