avro_turf 0.7.1 → 0.10.0

data/lib/avro_turf/schema_to_avro_patch.rb

@@ -34,8 +34,19 @@ class AvroTurf
         end
       end
     end
+
+    module DatumReader
+      def read_default_value(field_schema, default_value)
+        if default_value == :no_default
+          raise Avro::AvroError, "Missing data for #{field_schema} with no default"
+        end
+
+        super
+      end
+    end
   end
 end
 
 Avro::Schema::RecordSchema.send(:prepend, AvroTurf::AvroGemPatch::RecordSchema)
 Avro::Schema::Field.send(:prepend, AvroTurf::AvroGemPatch::Field)
+Avro::IO::DatumReader.send(:prepend, AvroTurf::AvroGemPatch::DatumReader)

data/lib/avro_turf/test/fake_confluent_schema_registry_server.rb

@@ -0,0 +1,141 @@
+require 'sinatra/base'
+
+class FakeConfluentSchemaRegistryServer < Sinatra::Base
+  SUBJECTS = Hash.new { Array.new }
+  SCHEMAS = []
+  CONFIGS = Hash.new
+  SUBJECT_NOT_FOUND = { error_code: 40401, message: 'Subject not found' }.to_json.freeze
+  VERSION_NOT_FOUND = { error_code: 40402, message: 'Version not found' }.to_json.freeze
+  SCHEMA_NOT_FOUND = { error_code: 40403, message: 'Schema not found' }.to_json.freeze
+  DEFAULT_GLOBAL_CONFIG = { 'compatibility' => 'BACKWARD'.freeze }.freeze
+
+  @global_config = DEFAULT_GLOBAL_CONFIG.dup
+
+  class << self
+    attr_reader :global_config
+  end
+
+  helpers do
+    def parse_schema
+      request.body.rewind
+      JSON.parse(request.body.read).fetch("schema").tap do |schema|
+        Avro::Schema.parse(schema)
+      end
+    end
+
+    def parse_config
+      request.body.rewind
+      JSON.parse(request.body.read)
+    end
+
+    def global_config
+      self.class.global_config
+    end
+  end
+
+  post "/subjects/:subject/versions" do
+    schema = parse_schema
+    ids_for_subject = SUBJECTS[params[:subject]]
+
+    schemas_for_subject =
+      SCHEMAS.select
+             .with_index { |_, i| ids_for_subject.include?(i) }
+
+    if schemas_for_subject.include?(schema)
+      schema_id = SCHEMAS.index(schema)
+    else
+      SCHEMAS << schema
+      schema_id = SCHEMAS.size - 1
+      SUBJECTS[params[:subject]] = SUBJECTS[params[:subject]] << schema_id
+    end
+
+    { id: schema_id }.to_json
+  end
+
+  get "/schemas/ids/:schema_id" do
+    schema = SCHEMAS.at(params[:schema_id].to_i)
+    halt(404, SCHEMA_NOT_FOUND) unless schema
+    { schema: schema }.to_json
+  end
+
+  get "/subjects" do
+    SUBJECTS.keys.to_json
+  end
+
+  get "/subjects/:subject/versions" do
+    schema_ids = SUBJECTS[params[:subject]]
+    halt(404, SUBJECT_NOT_FOUND) if schema_ids.empty?
+    (1..schema_ids.size).to_a.to_json
+  end
+
+  get "/subjects/:subject/versions/:version" do
+    schema_ids = SUBJECTS[params[:subject]]
+    halt(404, SUBJECT_NOT_FOUND) if schema_ids.empty?
+
+    schema_id = if params[:version] == 'latest'
+                  schema_ids.last
+                else
+                  schema_ids.at(Integer(params[:version]) - 1)
+                end
+    halt(404, VERSION_NOT_FOUND) unless schema_id
+
+    schema = SCHEMAS.at(schema_id)
+
+    {
+      name: params[:subject],
+      version: schema_ids.index(schema_id) + 1,
+      id: schema_id,
+      schema: schema
+    }.to_json
+  end
+
+  post "/subjects/:subject" do
+    schema = parse_schema
+
+    # Note: this does not actually handle the same schema registered under
+    # multiple subjects
+    schema_id = SCHEMAS.index(schema)
+
+    halt(404, SCHEMA_NOT_FOUND) unless schema_id
+
+    {
+      subject: params[:subject],
+      id: schema_id,
+      version: SUBJECTS[params[:subject]].index(schema_id) + 1,
+      schema: schema
+    }.to_json
+  end
+
+  post "/compatibility/subjects/:subject/versions/:version" do
+    # The ruby avro gem does not yet include a compatibility check between schemas.
+    # See https://github.com/apache/avro/pull/170
+    raise NotImplementedError
+  end
+
+  get "/config" do
+    global_config.to_json
+  end
+
+  put "/config" do
+    global_config.merge!(parse_config).to_json
+  end
+
+  get "/config/:subject" do
+    CONFIGS.fetch(params[:subject], global_config).to_json
+  end
+
+  put "/config/:subject" do
+    config = parse_config
+    subject = params[:subject]
+    CONFIGS.fetch(subject) do
+      CONFIGS[subject] = {}
+    end.merge!(config).to_json
+  end
+
+  def self.clear
+    SUBJECTS.clear
+    SCHEMAS.clear
+    CONFIGS.clear
+    @global_config = DEFAULT_GLOBAL_CONFIG.dup
+  end
+end

data/lib/avro_turf/test/fake_schema_registry_server.rb

@@ -1,84 +1,6 @@
-require 'sinatra/base'
+require 'avro_turf/test/fake_confluent_schema_registry_server'
 
-class FakeSchemaRegistryServer < Sinatra::Base
-  SUBJECTS = Hash.new { Array.new }
-  SCHEMAS = []
-  SUBJECT_NOT_FOUND = { error_code: 40401, message: 'Subject not found' }.to_json.freeze
-  VERSION_NOT_FOUND = { error_code: 40402, message: 'Version not found' }.to_json.freeze
-  SCHEMA_NOT_FOUND = { error_code: 40403, message: 'Schema not found' }.to_json.freeze
+# FakeSchemaRegistryServer is deprecated and will be removed in a future release.
+# Use FakeConfluentSchemaRegistryServer instead.
 
-  helpers do
-    def parse_schema
-      request.body.rewind
-      JSON.parse(request.body.read).fetch("schema").tap do |schema|
-        Avro::Schema.parse(schema)
-      end
-    end
-  end
-
-  post "/subjects/:subject/versions" do
-    SCHEMAS << parse_schema
-
-    schema_id = SCHEMAS.size - 1
-    SUBJECTS[params[:subject]] = SUBJECTS[params[:subject]] << schema_id
-    { id: schema_id }.to_json
-  end
-
-  get "/schemas/ids/:schema_id" do
-    schema = SCHEMAS.at(params[:schema_id].to_i)
-    halt(404, SCHEMA_NOT_FOUND) unless schema
-    { schema: schema }.to_json
-  end
-
-  get "/subjects" do
-    SUBJECTS.keys.to_json
-  end
-
-  get "/subjects/:subject/versions" do
-    schema_ids = SUBJECTS[params[:subject]]
-    halt(404, SUBJECT_NOT_FOUND) if schema_ids.empty?
-    (1..schema_ids.size).to_a.to_json
-  end
-
-  get "/subjects/:subject/versions/:version" do
-    schema_ids = SUBJECTS[params[:subject]]
-    halt(404, SUBJECT_NOT_FOUND) if schema_ids.empty?
-
-    schema_id = if params[:version] == 'latest'
-                  schema_ids.last
-                else
-                  schema_ids.at(Integer(params[:version]) - 1)
-                end
-    halt(404, VERSION_NOT_FOUND) unless schema_id
-
-    schema = SCHEMAS.at(schema_id)
-
-    {
-      name: params[:subject],
-      version: schema_ids.index(schema_id) + 1,
-      schema: schema
-    }.to_json
-  end
-
-  post "/subjects/:subject" do
-    schema = parse_schema
-
-    # Note: this does not actually handle the same schema registered under
-    # multiple subjects
-    schema_id = SCHEMAS.index(schema)
-
-    halt(404, SCHEMA_NOT_FOUND) unless schema_id
-
-    {
-      subject: params[:subject],
-      id: schema_id,
-      version: SUBJECTS[params[:subject]].index(schema_id) + 1,
-      schema: schema
-    }.to_json
-  end
-
-  def self.clear
-    SUBJECTS.clear
-    SCHEMAS.clear
-  end
-end
+FakeSchemaRegistryServer = FakeConfluentSchemaRegistryServer

data/lib/avro_turf/version.rb

@@ -1,3 +1,3 @@
 class AvroTurf
-  VERSION = "0.7.1"
+  VERSION = "0.10.0"
 end

data/spec/cached_confluent_schema_registry_spec.rb

@@ -0,0 +1,63 @@
+require 'webmock/rspec'
+require 'avro_turf/cached_confluent_schema_registry'
+require 'avro_turf/test/fake_confluent_schema_registry_server'
+
+describe AvroTurf::CachedConfluentSchemaRegistry do
+  let(:upstream) { instance_double(AvroTurf::ConfluentSchemaRegistry) }
+  let(:registry) { described_class.new(upstream) }
+  let(:id) { rand(999) }
+  let(:schema) do
+    {
+      type: "record",
+      name: "person",
+      fields: [{ name: "name", type: "string" }]
+    }.to_json
+  end
+
+  describe "#fetch" do
+    it "caches the result of fetch" do
+      # multiple calls return same result, with only one upstream call
+      allow(upstream).to receive(:fetch).with(id).and_return(schema)
+      expect(registry.fetch(id)).to eq(schema)
+      expect(registry.fetch(id)).to eq(schema)
+      expect(upstream).to have_received(:fetch).exactly(1).times
+    end
+  end
+
+  describe "#register" do
+    let(:subject_name) { "a_subject" }
+
+    it "caches the result of register" do
+      # multiple calls return same result, with only one upstream call
+      allow(upstream).to receive(:register).with(subject_name, schema).and_return(id)
+      expect(registry.register(subject_name, schema)).to eq(id)
+      expect(registry.register(subject_name, schema)).to eq(id)
+      expect(upstream).to have_received(:register).exactly(1).times
+    end
+  end
+
+  describe '#subject_version' do
+    let(:subject_name) { 'a_subject' }
+    let(:version) { 1 }
+    let(:schema_with_meta) do
+      {
+        subject: subject_name,
+        id: 1,
+        version: 1,
+        schema: schema
+      }
+    end
+
+    it 'caches the result of subject_version' do
+      allow(upstream).to receive(:subject_version).with(subject_name, version).and_return(schema_with_meta)
+      registry.subject_version(subject_name, version)
+      registry.subject_version(subject_name, version)
+      expect(upstream).to have_received(:subject_version).exactly(1).times
+    end
+  end
+
+  it_behaves_like "a confluent schema registry client" do
+    let(:upstream) { AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger) }
+    let(:registry) { described_class.new(upstream) }
+  end
+end

data/spec/confluent_schema_registry_spec.rb

@@ -0,0 +1,9 @@
+require 'webmock/rspec'
+require 'avro_turf/confluent_schema_registry'
+require 'avro_turf/test/fake_confluent_schema_registry_server'
+
+describe AvroTurf::ConfluentSchemaRegistry do
+  it_behaves_like "a confluent schema registry client" do
+    let(:registry) { described_class.new(registry_url, logger: logger) }
+  end
+end

data/spec/disk_cached_confluent_schema_registry_spec.rb

@@ -0,0 +1,159 @@
+require 'webmock/rspec'
+require 'avro_turf/cached_confluent_schema_registry'
+require 'avro_turf/test/fake_confluent_schema_registry_server'
+
+describe AvroTurf::CachedConfluentSchemaRegistry do
+  let(:upstream) { instance_double(AvroTurf::ConfluentSchemaRegistry) }
+  let(:cache)    { AvroTurf::DiskCache.new("spec/cache")}
+  let(:registry) { described_class.new(upstream, cache: cache) }
+  let(:id) { rand(999) }
+  let(:schema) do
+    {
+      type: "record",
+      name: "person",
+      fields: [{ name: "name", type: "string" }]
+    }.to_json
+  end
+
+  let(:city_id) { rand(999) }
+  let(:city_schema) do
+    {
+      type: "record",
+      name: "city",
+      fields: [{ name: "name", type: "string" }]
+    }.to_json
+  end
+
+  let(:subject) { 'subject' }
+  let(:version) { rand(999) }
+  let(:subject_version_schema) do
+    {
+      subject: subject,
+      version: version,
+      id: id,
+      schema:  {
+        type: "record",
+        name: "city",
+        fields: { name: "name", type: "string" }
+      }
+    }.to_json
+  end
+
+  before do
+    FileUtils.mkdir_p("spec/cache")
+  end
+
+  describe "#fetch" do
+    let(:cache_before) do
+      {
+        "#{id}" => "#{schema}"
+      }
+    end
+    let(:cache_after) do
+      {
+        "#{id}" => "#{schema}",
+        "#{city_id}" => "#{city_schema}"
+      }
+    end
+
+    # setup the disk cache to avoid performing the upstream fetch
+    before do
+      store_cache("schemas_by_id.json", cache_before)
+    end
+
+    it "uses preloaded disk cache" do
+      # multiple calls return same result, with zero upstream calls
+      allow(upstream).to receive(:fetch).with(id).and_return(schema)
+      expect(registry.fetch(id)).to eq(schema)
+      expect(registry.fetch(id)).to eq(schema)
+      expect(upstream).to have_received(:fetch).exactly(0).times
+      expect(load_cache("schemas_by_id.json")).to eq cache_before
+    end
+
+    it "writes thru to disk cache" do
+      # multiple calls return same result, with only one upstream call
+      allow(upstream).to receive(:fetch).with(city_id).and_return(city_schema)
+      expect(registry.fetch(city_id)).to eq(city_schema)
+      expect(registry.fetch(city_id)).to eq(city_schema)
+      expect(upstream).to have_received(:fetch).exactly(1).times
+      expect(load_cache("schemas_by_id.json")).to eq cache_after
+    end
+  end
+
+  describe "#register" do
+    let(:subject_name) { "a_subject" }
+    let(:cache_before) do
+      {
+        "#{subject_name}#{schema}" => id
+      }
+    end
+
+    let(:city_name) { "a_city" }
+    let(:cache_after) do 
+      {
+        "#{subject_name}#{schema}" => id,
+        "#{city_name}#{city_schema}" => city_id
+      }
+    end
+
+    # setup the disk cache to avoid performing the upstream register
+    before do
+      store_cache("ids_by_schema.json", cache_before)
+    end
+
+    it "uses preloaded disk cache" do
+      # multiple calls return same result, with zero upstream calls
+      allow(upstream).to receive(:register).with(subject_name, schema).and_return(id)
+      expect(registry.register(subject_name, schema)).to eq(id) 
+      expect(registry.register(subject_name, schema)).to eq(id)
+      expect(upstream).to have_received(:register).exactly(0).times
+      expect(load_cache("ids_by_schema.json")).to eq cache_before
+    end
+
+    it "writes thru to disk cache" do
+      # multiple calls return same result, with only one upstream call
+      allow(upstream).to receive(:register).with(city_name, city_schema).and_return(city_id)
+      expect(registry.register(city_name, city_schema)).to eq(city_id)
+      expect(registry.register(city_name, city_schema)).to eq(city_id)
+      expect(upstream).to have_received(:register).exactly(1).times
+      expect(load_cache("ids_by_schema.json")).to eq cache_after
+    end
+  end
+
+  describe "#subject_version" do
+    it "writes thru to disk cache" do
+      # multiple calls return same result, with zero upstream calls
+      allow(upstream).to receive(:subject_version).with(subject, version).and_return(subject_version_schema)
+      expect(File).not_to exist("./spec/cache/schemas_by_subject_version.json")
+
+      expect(registry.subject_version(subject, version)).to eq(subject_version_schema)
+
+      json = JSON.parse(File.read("./spec/cache/schemas_by_subject_version.json"))["#{subject}#{version}"]
+      expect(json).to eq(subject_version_schema)
+
+      expect(registry.subject_version(subject, version)).to eq(subject_version_schema)
+      expect(upstream).to have_received(:subject_version).exactly(1).times
+    end
+
+    it "reads from disk cache and populates mem cache" do
+      allow(upstream).to receive(:subject_version).with(subject, version).and_return(subject_version_schema)
+      key = "#{subject}#{version}"
+      hash = {key => subject_version_schema}
+      cache.send(:write_to_disk_cache, "./spec/cache/schemas_by_subject_version.json", hash)
+
+      cached_schema = cache.instance_variable_get(:@schemas_by_subject_version)
+      expect(cached_schema).to eq({})
+
+      expect(registry.subject_version(subject, version)).to eq(subject_version_schema)
+      expect(upstream).to have_received(:subject_version).exactly(0).times
+
+      cached_schema = cache.instance_variable_get(:@schemas_by_subject_version)
+      expect(cached_schema).to eq({key => subject_version_schema})
+    end
+  end
+
+  it_behaves_like "a confluent schema registry client" do
+    let(:upstream) { AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger) }
+    let(:registry) { described_class.new(upstream) }
+  end
+end