avro_turf 0.8.0 → 1.0.0

data/lib/avro_turf/messaging.rb

@@ -21,21 +21,53 @@ class AvroTurf
   # 1: https://github.com/confluentinc/schema-registry
   class Messaging
     MAGIC_BYTE = [0].pack("C").freeze
+    DecodedMessage = Struct.new(:schema_id, :writer_schema, :reader_schema, :message)
+    private_constant(:DecodedMessage)
 
     # Instantiate a new Messaging instance with the given configuration.
     #
-    # registry     - A schema registry object that responds to all methods in the
-    #                AvroTurf::ConfluentSchemaRegistry interface.
-    # registry_url - The String URL of the schema registry that should be used.
-    # schema_store - A schema store object that responds to #find(schema_name, namespace).
-    # schemas_path - The String file system path where local schemas are stored.
-    # namespace    - The String default schema namespace.
-    # logger       - The Logger that should be used to log information (optional).
-    def initialize(registry: nil, registry_url: nil, schema_store: nil, schemas_path: nil, namespace: nil, logger: nil)
+    # registry          - A schema registry object that responds to all methods in the
+    #                     AvroTurf::ConfluentSchemaRegistry interface.
+    # registry_url      - The String URL of the schema registry that should be used.
+    # schema_store      - A schema store object that responds to #find(schema_name, namespace).
+    # schemas_path      - The String file system path where local schemas are stored.
+    # namespace         - The String default schema namespace.
+    # logger            - The Logger that should be used to log information (optional).
+    # proxy             - Forward the request via  proxy (optional).
+    # client_cert       - Name of file containing client certificate (optional).
+    # client_key        - Name of file containing client private key to go with client_cert (optional).
+    # client_key_pass   - Password to go with client_key (optional).
+    # client_cert_data  - In-memory client certificate (optional).
+    # client_key_data   - In-memory client private key to go with client_cert_data (optional).
+    def initialize(
+      registry: nil,
+      registry_url: nil,
+      schema_store: nil,
+      schemas_path: nil,
+      namespace: nil,
+      logger: nil,
+      proxy: nil,
+      client_cert: nil,
+      client_key: nil,
+      client_key_pass: nil,
+      client_cert_data: nil,
+      client_key_data: nil
+    )
       @logger = logger || Logger.new($stderr)
       @namespace = namespace
       @schema_store = schema_store || SchemaStore.new(path: schemas_path || DEFAULT_SCHEMAS_PATH)
-      @registry = registry || CachedConfluentSchemaRegistry.new(ConfluentSchemaRegistry.new(registry_url, logger: @logger))
+      @registry = registry || CachedConfluentSchemaRegistry.new(
+        ConfluentSchemaRegistry.new(
+          registry_url,
+          logger: @logger,
+          proxy: proxy,
+          client_cert: client_cert,
+          client_key: client_key,
+          client_key_pass: client_key_pass,
+          client_cert_data: client_cert_data,
+          client_key_data: client_key_data
+        )
+      )
       @schemas_by_id = {}
     end
 
@@ -46,14 +78,24 @@ class AvroTurf
     # schema_name - The String name of the schema that should be used to encode
     #               the data.
     # namespace   - The namespace of the schema (optional).
+    # subject     - The subject name the schema should be registered under in
+    #               the schema registry (optional).
+    # version     - The integer version of the schema that should be used to decode
+    #               the data. Must match the schema used when encoding (optional).
+    # schema_id   - The integer id of the schema that should be used to encode
+    #               the data.
     #
     # Returns the encoded data as a String.
-    def encode(message, schema_name: nil, namespace: @namespace, subject: nil)
-      schema = @schema_store.find(schema_name, namespace)
-
-      # Schemas are registered under the full name of the top level Avro record
-      # type, or `subject` if it's provided.
-      schema_id = @registry.register(subject || schema.fullname, schema)
+    def encode(message, schema_name: nil, namespace: @namespace, subject: nil, version: nil, schema_id: nil)
+      schema_id, schema = if schema_id
+        fetch_schema_by_id(schema_id)
+      elsif subject && version
+        fetch_schema(subject, version)
+      elsif schema_name
+        register_schema(subject, schema_name, namespace)
+      else
+        raise ArgumentError.new('Neither schema_name nor schema_id nor subject + version provided to determine the schema.')
+      end
 
       stream = StringIO.new
       writer = Avro::IO::DatumWriter.new(schema)
@@ -69,6 +111,12 @@ class AvroTurf
       writer.write(message, encoder)
 
       stream.string
+    rescue Excon::Error::NotFound
+      if schema_id
+        raise SchemaNotFoundError.new("Schema with id: #{schema_id} is not found on registry")
+      else
+        raise SchemaNotFoundError.new("Schema with subject: `#{subject}` version: `#{version}` is not found on registry")
+      end
     end
 
     # Decodes data into the original message.
@@ -80,6 +128,20 @@ class AvroTurf
     #
     # Returns the decoded message.
     def decode(data, schema_name: nil, namespace: @namespace)
+      decode_message(data, schema_name: schema_name, namespace: namespace).message
+    end
+
+    # Decodes data into the original message.
+    #
+    # data        - A String containing encoded data.
+    # schema_name - The String name of the schema that should be used to decode
+    #               the data. Must match the schema used when encoding (optional).
+    # namespace   - The namespace of the schema (optional).
+    #
+    # Returns Struct with the next attributes:
+    #   schema_id  - The integer id of schema used to encode the message
+    #   message    - The decoded message
+    def decode_message(data, schema_name: nil, namespace: @namespace)
       readers_schema = schema_name && @schema_store.find(schema_name, namespace)
       stream = StringIO.new(data)
       decoder = Avro::IO::BinaryDecoder.new(stream)
@@ -100,7 +162,38 @@ class AvroTurf
       end
 
       reader = Avro::IO::DatumReader.new(writers_schema, readers_schema)
-      reader.read(decoder)
+      message = reader.read(decoder)
+
+      DecodedMessage.new(schema_id, writers_schema, readers_schema, message)
+    rescue Excon::Error::NotFound
+      raise SchemaNotFoundError.new("Schema with id: #{schema_id} is not found on registry")
+    end
+
+    private
+
+    # Providing subject and version to determine the schema,
+    # which skips the auto registeration of schema on the schema registry.
+    # Fetch the schema from registry with the provided subject name and version.
+    def fetch_schema(subject, version)
+      schema_data = @registry.subject_version(subject, version)
+      schema_id = schema_data.fetch('id')
+      schema = Avro::Schema.parse(schema_data.fetch('schema'))
+      [schema_id, schema]
+    end
+
+    # Fetch the schema from registry with the provided schema_id.
+    def fetch_schema_by_id(schema_id)
+      schema_json = @registry.fetch(schema_id)
+      schema = Avro::Schema.parse(schema_json)
+      [schema_id, schema]
+    end
+
+    # Schemas are registered under the full name of the top level Avro record
+    # type, or `subject` if it's provided.
+    def register_schema(subject, schema_name, namespace)
+      schema = @schema_store.find(schema_name, namespace)
+      schema_id = @registry.register(subject || schema.fullname, schema)
+      [schema_id, schema]
     end
   end
 end

data/lib/avro_turf/mutable_schema_store.rb

@@ -0,0 +1,18 @@
+require 'avro_turf/schema_store'
+
+class AvroTurf
+  # A schema store that allows you to add or remove schemas, and to access
+  # them externally.
+  class MutableSchemaStore < SchemaStore
+    attr_accessor :schemas
+
+    # @param schema_hash [Hash]
+    def add_schema(schema_hash)
+      name = schema_hash['name']
+      namespace = schema_hash['namespace']
+      full_name = Avro::Name.make_fullname(name, namespace)
+      return if @schemas.key?(full_name)
+      Avro::Schema.real_parse(schema_hash, @schemas)
+    end
+  end
+end

data/lib/avro_turf/schema_store.rb

@@ -1,7 +1,9 @@
 class AvroTurf::SchemaStore
+
   def initialize(path: nil)
     @path = path or raise "Please specify a schema path"
     @schemas = Hash.new
+    @mutex = Mutex.new
   end
 
   # Resolves and returns a schema.
@@ -11,48 +13,82 @@ class AvroTurf::SchemaStore
   # Returns an Avro::Schema.
   def find(name, namespace = nil)
     fullname = Avro::Name.make_fullname(name, namespace)
-
+    # Optimistic non-blocking read from @schemas
+    # No sense to lock the resource when all the schemas already loaded
     return @schemas[fullname] if @schemas.key?(fullname)
 
+    # Pessimistic blocking write to @schemas
+    @mutex.synchronize do
+      # Still need to check is the schema already loaded
+      return @schemas[fullname] if @schemas.key?(fullname)
+
+      load_schema!(fullname, namespace)
+    end
+  end
+
+  # Loads all schema definition files in the `schemas_dir`.
+  def load_schemas!
+    pattern = [@path, "**", "*.avsc"].join("/")
+
+    Dir.glob(pattern) do |schema_path|
+      # Remove the path prefix.
+      schema_path.sub!(/^\/?#{@path}\//, "")
+
+      # Replace `/` with `.` and chop off the file extension.
+      schema_name = File.basename(schema_path.tr("/", "."), ".avsc")
+
+      # Load and cache the schema.
+      find(schema_name)
+    end
+  end
+
+  private
+
+  # Loads single schema
+  # Such method is not thread-safe, do not call it of from mutex synchronization routine
+  def load_schema!(fullname, namespace = nil, local_schemas_cache = {})
     *namespace, schema_name = fullname.split(".")
     schema_path = File.join(@path, *namespace, schema_name + ".avsc")
     schema_json = JSON.parse(File.read(schema_path))
-    schema = Avro::Schema.real_parse(schema_json, @schemas)
 
+    schema = Avro::Schema.real_parse(schema_json, local_schemas_cache)
+
+    # Don't cache the parsed schema until after its fullname is validated
     if schema.respond_to?(:fullname) && schema.fullname != fullname
       raise AvroTurf::SchemaError, "expected schema `#{schema_path}' to define type `#{fullname}'"
     end
 
+    # Cache only this new top-level schema by its fullname. It's critical
+    # not to make every sub-schema resolvable at the top level here because
+    # multiple different avsc files may define the same sub-schema, and
+    # if we share the @schemas cache across all parsing contexts, the Avro
+    # gem will raise an Avro::SchemaParseError when parsing another avsc
+    # file that contains a subschema with the same fullname as one
+    # encountered previously in a different file:
+    # <Avro::SchemaParseError: The name "foo.bar" is already in use.>
+    # Essentially, the only schemas that should be resolvable in @schemas
+    # are those that have their own .avsc files on disk.
+    @schemas[fullname] = schema
+
     schema
   rescue ::Avro::SchemaParseError => e
     # This is a hack in order to figure out exactly which type was missing. The
     # Avro gem ought to provide this data directly.
     if e.to_s =~ /"([\w\.]+)" is not a schema we know about/
-      find($1)
+      # Try to first resolve a referenced schema from disk.
+      # If this is successful, the Avro gem will have mutated the
+      # local_schemas_cache, adding all the new schemas it found.
+      load_schema!($1, nil, local_schemas_cache)
 
-      # Re-resolve the original schema now that the dependency has been resolved.
-      @schemas.delete(fullname)
-      find(fullname)
+      # Attempt to re-parse the original schema now that the dependency
+      # has been resolved and use the now-updated local_schemas_cache to
+      # pick up where we left off.
+      local_schemas_cache.delete(fullname)
+      load_schema!(fullname, nil, local_schemas_cache)
     else
       raise
     end
   rescue Errno::ENOENT, Errno::ENAMETOOLONG
     raise AvroTurf::SchemaNotFoundError, "could not find Avro schema at `#{schema_path}'"
   end
-
-  # Loads all schema definition files in the `schemas_dir`.
-  def load_schemas!
-    pattern = [@path, "**", "*.avsc"].join("/")
-
-    Dir.glob(pattern) do |schema_path|
-      # Remove the path prefix.
-      schema_path.sub!(/^\/?#{@path}\//, "")
-
-      # Replace `/` with `.` and chop off the file extension.
-      schema_name = File.basename(schema_path.tr("/", "."), ".avsc")
-
-      # Load and cache the schema.
-      find(schema_name)
-    end
-  end
 end

data/lib/avro_turf/test/fake_confluent_schema_registry_server.rb

@@ -34,10 +34,21 @@ class FakeConfluentSchemaRegistryServer < Sinatra::Base
   end
 
   post "/subjects/:subject/versions" do
-    SCHEMAS << parse_schema
+    schema = parse_schema
+    ids_for_subject = SUBJECTS[params[:subject]]
+
+    schemas_for_subject =
+      SCHEMAS.select
+             .with_index { |_, i| ids_for_subject.include?(i) }
+
+    if schemas_for_subject.include?(schema)
+      schema_id = SCHEMAS.index(schema)
+    else
+      SCHEMAS << schema
+      schema_id = SCHEMAS.size - 1
+      SUBJECTS[params[:subject]] = SUBJECTS[params[:subject]] << schema_id
+    end
 
-    schema_id = SCHEMAS.size - 1
-    SUBJECTS[params[:subject]] = SUBJECTS[params[:subject]] << schema_id
     { id: schema_id }.to_json
   end
 
@@ -73,6 +84,7 @@ class FakeConfluentSchemaRegistryServer < Sinatra::Base
     {
       name: params[:subject],
       version: schema_ids.index(schema_id) + 1,
+      id: schema_id,
       schema: schema
     }.to_json
   end

data/lib/avro_turf/version.rb

@@ -1,3 +1,3 @@
 class AvroTurf
-  VERSION = "0.8.0"
+  VERSION = "1.0.0"
 end

data/spec/cached_confluent_schema_registry_spec.rb

@@ -16,8 +16,9 @@ describe AvroTurf::CachedConfluentSchemaRegistry do
 
   describe "#fetch" do
     it "caches the result of fetch" do
+      # multiple calls return same result, with only one upstream call
       allow(upstream).to receive(:fetch).with(id).and_return(schema)
-      registry.fetch(id)
+      expect(registry.fetch(id)).to eq(schema)
       expect(registry.fetch(id)).to eq(schema)
       expect(upstream).to have_received(:fetch).exactly(1).times
     end
@@ -27,13 +28,34 @@ describe AvroTurf::CachedConfluentSchemaRegistry do
     let(:subject_name) { "a_subject" }
 
     it "caches the result of register" do
+      # multiple calls return same result, with only one upstream call
       allow(upstream).to receive(:register).with(subject_name, schema).and_return(id)
-      registry.register(subject_name, schema)
+      expect(registry.register(subject_name, schema)).to eq(id)
       expect(registry.register(subject_name, schema)).to eq(id)
       expect(upstream).to have_received(:register).exactly(1).times
     end
   end
 
+  describe '#subject_version' do
+    let(:subject_name) { 'a_subject' }
+    let(:version) { 1 }
+    let(:schema_with_meta) do
+      {
+        subject: subject_name,
+        id: 1,
+        version: 1,
+        schema: schema
+      }
+    end
+
+    it 'caches the result of subject_version' do
+      allow(upstream).to receive(:subject_version).with(subject_name, version).and_return(schema_with_meta)
+      registry.subject_version(subject_name, version)
+      registry.subject_version(subject_name, version)
+      expect(upstream).to have_received(:subject_version).exactly(1).times
+    end
+  end
+
   it_behaves_like "a confluent schema registry client" do
     let(:upstream) { AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger) }
     let(:registry) { described_class.new(upstream) }

data/spec/confluent_schema_registry_spec.rb

@@ -3,7 +3,19 @@ require 'avro_turf/confluent_schema_registry'
 require 'avro_turf/test/fake_confluent_schema_registry_server'
 
 describe AvroTurf::ConfluentSchemaRegistry do
+  let(:client_cert) { "test client cert" }
+  let(:client_key) { "test client key" }
+  let(:client_key_pass) { "test client key password" }
+
   it_behaves_like "a confluent schema registry client" do
-    let(:registry) { described_class.new(registry_url, logger: logger) }
+    let(:registry) {
+      described_class.new(
+        registry_url,
+        logger: logger,
+        client_cert: client_cert,
+        client_key: client_key,
+        client_key_pass: client_key_pass
+      )
+    }
   end
 end

data/spec/disk_cached_confluent_schema_registry_spec.rb

@@ -0,0 +1,159 @@
+require 'webmock/rspec'
+require 'avro_turf/cached_confluent_schema_registry'
+require 'avro_turf/test/fake_confluent_schema_registry_server'
+
+describe AvroTurf::CachedConfluentSchemaRegistry do
+  let(:upstream) { instance_double(AvroTurf::ConfluentSchemaRegistry) }
+  let(:cache)    { AvroTurf::DiskCache.new("spec/cache")}
+  let(:registry) { described_class.new(upstream, cache: cache) }
+  let(:id) { rand(999) }
+  let(:schema) do
+    {
+      type: "record",
+      name: "person",
+      fields: [{ name: "name", type: "string" }]
+    }.to_json
+  end
+
+  let(:city_id) { rand(999) }
+  let(:city_schema) do
+    {
+      type: "record",
+      name: "city",
+      fields: [{ name: "name", type: "string" }]
+    }.to_json
+  end
+
+  let(:subject) { 'subject' }
+  let(:version) { rand(999) }
+  let(:subject_version_schema) do
+    {
+      subject: subject,
+      version: version,
+      id: id,
+      schema:  {
+        type: "record",
+        name: "city",
+        fields: { name: "name", type: "string" }
+      }
+    }.to_json
+  end
+
+  before do
+    FileUtils.mkdir_p("spec/cache")
+  end
+
+  describe "#fetch" do
+    let(:cache_before) do
+      {
+        "#{id}" => "#{schema}"
+      }
+    end
+    let(:cache_after) do
+      {
+        "#{id}" => "#{schema}",
+        "#{city_id}" => "#{city_schema}"
+      }
+    end
+
+    # setup the disk cache to avoid performing the upstream fetch
+    before do
+      store_cache("schemas_by_id.json", cache_before)
+    end
+
+    it "uses preloaded disk cache" do
+      # multiple calls return same result, with zero upstream calls
+      allow(upstream).to receive(:fetch).with(id).and_return(schema)
+      expect(registry.fetch(id)).to eq(schema)
+      expect(registry.fetch(id)).to eq(schema)
+      expect(upstream).to have_received(:fetch).exactly(0).times
+      expect(load_cache("schemas_by_id.json")).to eq cache_before
+    end
+
+    it "writes thru to disk cache" do
+      # multiple calls return same result, with only one upstream call
+      allow(upstream).to receive(:fetch).with(city_id).and_return(city_schema)
+      expect(registry.fetch(city_id)).to eq(city_schema)
+      expect(registry.fetch(city_id)).to eq(city_schema)
+      expect(upstream).to have_received(:fetch).exactly(1).times
+      expect(load_cache("schemas_by_id.json")).to eq cache_after
+    end
+  end
+
+  describe "#register" do
+    let(:subject_name) { "a_subject" }
+    let(:cache_before) do
+      {
+        "#{subject_name}#{schema}" => id
+      }
+    end
+
+    let(:city_name) { "a_city" }
+    let(:cache_after) do 
+      {
+        "#{subject_name}#{schema}" => id,
+        "#{city_name}#{city_schema}" => city_id
+      }
+    end
+
+    # setup the disk cache to avoid performing the upstream register
+    before do
+      store_cache("ids_by_schema.json", cache_before)
+    end
+
+    it "uses preloaded disk cache" do
+      # multiple calls return same result, with zero upstream calls
+      allow(upstream).to receive(:register).with(subject_name, schema).and_return(id)
+      expect(registry.register(subject_name, schema)).to eq(id) 
+      expect(registry.register(subject_name, schema)).to eq(id)
+      expect(upstream).to have_received(:register).exactly(0).times
+      expect(load_cache("ids_by_schema.json")).to eq cache_before
+    end
+
+    it "writes thru to disk cache" do
+      # multiple calls return same result, with only one upstream call
+      allow(upstream).to receive(:register).with(city_name, city_schema).and_return(city_id)
+      expect(registry.register(city_name, city_schema)).to eq(city_id)
+      expect(registry.register(city_name, city_schema)).to eq(city_id)
+      expect(upstream).to have_received(:register).exactly(1).times
+      expect(load_cache("ids_by_schema.json")).to eq cache_after
+    end
+  end
+
+  describe "#subject_version" do
+    it "writes thru to disk cache" do
+      # multiple calls return same result, with zero upstream calls
+      allow(upstream).to receive(:subject_version).with(subject, version).and_return(subject_version_schema)
+      expect(File).not_to exist("./spec/cache/schemas_by_subject_version.json")
+
+      expect(registry.subject_version(subject, version)).to eq(subject_version_schema)
+
+      json = JSON.parse(File.read("./spec/cache/schemas_by_subject_version.json"))["#{subject}#{version}"]
+      expect(json).to eq(subject_version_schema)
+
+      expect(registry.subject_version(subject, version)).to eq(subject_version_schema)
+      expect(upstream).to have_received(:subject_version).exactly(1).times
+    end
+
+    it "reads from disk cache and populates mem cache" do
+      allow(upstream).to receive(:subject_version).with(subject, version).and_return(subject_version_schema)
+      key = "#{subject}#{version}"
+      hash = {key => subject_version_schema}
+      cache.send(:write_to_disk_cache, "./spec/cache/schemas_by_subject_version.json", hash)
+
+      cached_schema = cache.instance_variable_get(:@schemas_by_subject_version)
+      expect(cached_schema).to eq({})
+
+      expect(registry.subject_version(subject, version)).to eq(subject_version_schema)
+      expect(upstream).to have_received(:subject_version).exactly(0).times
+
+      cached_schema = cache.instance_variable_get(:@schemas_by_subject_version)
+      expect(cached_schema).to eq({key => subject_version_schema})
+    end
+  end
+
+  it_behaves_like "a confluent schema registry client" do
+    let(:upstream) { AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger) }
+    let(:registry) { described_class.new(upstream) }
+  end
+end