avro_turf_enchanced 0.7.3

data/lib/avro_turf/core_ext/hash.rb

@@ -0,0 +1,7 @@
+class Hash
+  def as_avro
+    hsh = Hash.new
+    each {|k, v| hsh[k.as_avro] = v.as_avro }
+    hsh
+  end
+end

data/lib/avro_turf/core_ext/nil_class.rb

@@ -0,0 +1,5 @@
+class NilClass
+  def as_avro
+    self
+  end
+end

data/lib/avro_turf/core_ext/numeric.rb

@@ -0,0 +1,5 @@
+class Numeric
+  def as_avro
+    self
+  end
+end

data/lib/avro_turf/core_ext/string.rb

@@ -0,0 +1,5 @@
+class String
+  def as_avro
+    self
+  end
+end

data/lib/avro_turf/core_ext/symbol.rb

@@ -0,0 +1,5 @@
+class Symbol
+  def as_avro
+    to_s
+  end
+end

data/lib/avro_turf/core_ext/time.rb

@@ -0,0 +1,5 @@
+class Time
+  def as_avro
+    iso8601
+  end
+end

data/lib/avro_turf/core_ext/true_class.rb

@@ -0,0 +1,5 @@
+class TrueClass
+  def as_avro
+    self
+  end
+end

data/lib/avro_turf/messaging.rb

@@ -0,0 +1,102 @@
+require 'logger'
+require 'avro_turf'
+require 'avro_turf/schema_store'
+require 'avro_turf/schema_registry'
+require 'avro_turf/cached_schema_registry'
+
+class AvroTurf
+
+  # Provides a way to encode and decode messages without having to embed schemas
+  # in the encoded data. Confluent's Schema Registry[1] is used to register
+  # a schema when encoding a message -- the registry will issue a schema id that
+  # will be included in the encoded data alongside the actual message. When
+  # decoding the data, the schema id will be used to look up the writer's schema
+  # from the registry.
+  #
+  # 1: https://github.com/confluentinc/schema-registry
+  class Messaging
+    MAGIC_BYTE = [0].pack("C").freeze
+
+    # Instantiate a new Messaging instance with the given configuration.
+    #
+    # registry     - A schema registry object that responds to all methods in the
+    #                AvroTurf::SchemaRegistry interface.
+    # registry_url - The String URL of the schema registry that should be used.
+    # schema_store - A schema store object that responds to #find(schema_name, namespace).
+    # schemas_path - The String file system path where local schemas are stored.
+    # namespace    - The String default schema namespace.
+    # logger       - The Logger that should be used to log information (optional).
+    def initialize(registry: nil, registry_url: nil, schema_store: nil, schemas_path: nil, namespace: nil, logger: nil)
+      @logger = logger || Logger.new($stderr)
+      @namespace = namespace
+      @schema_store = schema_store || SchemaStore.new(path: schemas_path || DEFAULT_SCHEMAS_PATH)
+      @registry = registry || CachedSchemaRegistry.new(SchemaRegistry.new(registry_url, logger: @logger))
+      @schemas_by_id = {}
+    end
+
+    # Encodes a message using the specified schema.
+    #
+    # message     - The message that should be encoded. Must be compatible with
+    #               the schema.
+    # schema_name - The String name of the schema that should be used to encode
+    #               the data.
+    # namespace   - The namespace of the schema (optional).
+    #
+    # Returns the encoded data as a String.
+    def encode(message, schema: nil, namespace: @namespace, subject: nil)
+      # schema = @schema_store.find(schema_name, namespace)
+
+      # Schemas are registered under the full name of the top level Avro record
+      # type, or `subject` if it's provided.
+      schema_id = @registry.register(subject || schema.fullname, schema)
+
+      stream = StringIO.new
+      writer = Avro::IO::DatumWriter.new(schema)
+      encoder = Avro::IO::BinaryEncoder.new(stream)
+
+      # Always start with the magic byte.
+      encoder.write(MAGIC_BYTE)
+
+      # The schema id is encoded as a 4-byte big-endian integer.
+      encoder.write([schema_id].pack("N"))
+
+      # The actual message comes last.
+      writer.write(message, encoder)
+
+      stream.string
+    end
+
+    # Decodes data into the original message.
+    #
+    # data        - A String containing encoded data.
+    # schema_name - The String name of the schema that should be used to decode
+    #               the data. Must match the schema used when encoding (optional).
+    # namespace   - The namespace of the schema (optional).
+    #
+    # Returns the decoded message.
+    def decode(data, schema_name: nil, namespace: @namespace)
+      readers_schema = schema_name && @schema_store.find(schema_name, namespace)
+      stream = StringIO.new(data)
+      decoder = Avro::IO::BinaryDecoder.new(stream)
+
+      # The first byte is MAGIC!!!
+      magic_byte = decoder.read(1)
+
+      if magic_byte != MAGIC_BYTE
+        raise "Expected data to begin with a magic byte, got `#{magic_byte.inspect}`"
+      end
+
+      # The schema id is a 4-byte big-endian integer.
+      schema_id = decoder.read(4).unpack("N").first
+
+      writers_schema = @schemas_by_id.fetch(schema_id) do
+        schema_json = @registry.fetch(schema_id)
+        @schemas_by_id[schema_id] = Avro::Schema.parse(schema_json)
+      end
+
+      reader = Avro::IO::DatumReader.new(writers_schema, readers_schema)
+      reader.read(decoder)
+    end
+  end
+end
+

data/lib/avro_turf/schema_registry.rb

@@ -0,0 +1,79 @@
+require 'excon'
+
+class AvroTurf::SchemaRegistry
+  CONTENT_TYPE = "application/vnd.schemaregistry.v1+json".freeze
+
+  def initialize(url, logger: Logger.new($stdout))
+    @logger = logger
+    @connection = Excon.new(url, headers: {
+      "Content-Type" => CONTENT_TYPE,
+    })
+  end
+
+  def fetch(id)
+    @logger.info "Fetching schema with id #{id}"
+    data = get("/schemas/ids/#{id}")
+    data.fetch("schema")
+  end
+
+  def register(subject, schema)
+    data = post("/subjects/#{subject}/versions", body: {
+      schema: schema.to_s
+    }.to_json)
+
+    id = data.fetch("id")
+
+    @logger.info "Registered schema for subject `#{subject}`; id = #{id}"
+
+    id
+  end
+
+  # List all subjects
+  def subjects
+    get('/subjects')
+  end
+
+  # List all versions for a subject
+  def subject_versions(subject)
+    get("/subjects/#{subject}/versions")
+  end
+
+  # Get a specific version for a subject
+  def subject_version(subject, version = 'latest')
+    get("/subjects/#{subject}/versions/#{version}")
+  end
+
+  # Check if a schema exists. Returns nil if not found.
+  def check(subject, schema)
+    data = post("/subjects/#{subject}",
+                expects: [200, 404],
+                body: { schema: schema.to_s }.to_json)
+    data unless data.has_key?("error_code")
+  end
+
+  # Check if a schema is compatible with the stored version.
+  # Returns true if compatible, false otherwise
+  # http://docs.confluent.io/2.0.0/schema-registry/docs/api.html#compatibility
+  def compatible?(subject, schema, version = 'latest')
+    data = post("/compatibility/subjects/#{subject}/versions/#{version}",
+                expects: [200, 404],
+                body: { schema: schema.to_s }.to_json)
+    data.fetch('is_compatible', false) unless data.has_key?('error_code')
+  end
+
+  private
+
+  def get(path, **options)
+    request(path, method: :get, **options)
+  end
+
+  def post(path, **options)
+    request(path, method: :post, **options)
+  end
+
+  def request(path, **options)
+    options = { expects: 200 }.merge!(options)
+    response = @connection.request(path: path, **options)
+    JSON.parse(response.body)
+  end
+end

data/lib/avro_turf/schema_store.rb

@@ -0,0 +1,58 @@
+class AvroTurf::SchemaStore
+  def initialize(path: nil)
+    @path = path or raise "Please specify a schema path"
+    @schemas = Hash.new
+  end
+
+  # Resolves and returns a schema.
+  #
+  # schema_name - The String name of the schema to resolve.
+  #
+  # Returns an Avro::Schema.
+  def find(name, namespace = nil)
+    fullname = Avro::Name.make_fullname(name, namespace)
+
+    return @schemas[fullname] if @schemas.key?(fullname)
+
+    *namespace, schema_name = fullname.split(".")
+    schema_path = File.join(@path, *namespace, schema_name + ".avsc")
+    schema_json = JSON.parse(File.read(schema_path))
+    schema = Avro::Schema.real_parse(schema_json, @schemas)
+
+    if schema.respond_to?(:fullname) && schema.fullname != fullname
+      raise AvroTurf::SchemaError, "expected schema `#{schema_path}' to define type `#{fullname}'"
+    end
+
+    schema
+  rescue ::Avro::SchemaParseError => e
+    # This is a hack in order to figure out exactly which type was missing. The
+    # Avro gem ought to provide this data directly.
+    if e.to_s =~ /"([\w\.]+)" is not a schema we know about/
+      find($1)
+
+      # Re-resolve the original schema now that the dependency has been resolved.
+      @schemas.delete(fullname)
+      find(fullname)
+    else
+      raise
+    end
+  rescue Errno::ENOENT, Errno::ENAMETOOLONG
+    raise AvroTurf::SchemaNotFoundError, "could not find Avro schema at `#{schema_path}'"
+  end
+
+  # Loads all schema definition files in the `schemas_dir`.
+  def load_schemas!
+    pattern = [@path, "**", "*.avsc"].join("/")
+
+    Dir.glob(pattern) do |schema_path|
+      # Remove the path prefix.
+      schema_path.sub!(/^\/?#{@path}\//, "")
+
+      # Replace `/` with `.` and chop off the file extension.
+      schema_name = File.basename(schema_path.tr("/", "."), ".avsc")
+
+      # Load and cache the schema.
+      find(schema_name)
+    end
+  end
+end

data/lib/avro_turf/schema_to_avro_patch.rb

@@ -0,0 +1,52 @@
+class AvroTurf
+  module AvroGemPatch
+    module RecordSchema
+      module ClassMethods
+        def make_field_objects(field_data, names, namespace=nil)
+          new_field_data = []
+          field_data.each do |field|
+            if field.respond_to?(:[]) && !field.key?('default')
+              field = field.clone
+              field['default'] = :no_default
+            end
+            new_field_data << field
+          end
+          super(new_field_data, names, namespace)
+        end
+      end
+      
+      def self.prepended(base)
+        class << base
+          prepend ClassMethods
+        end
+      end
+    end
+    
+    module Field
+      def initialize(type, name, default=:no_default, order=nil, names=nil, namespace=nil)
+        super(type, name, default, order, names, namespace)
+      end
+      
+      def to_avro(names=Set.new)
+        {'name' => name, 'type' => type.to_avro(names)}.tap do |avro|
+          avro['default'] = default unless default == :no_default
+          avro['order'] = order if order
+        end
+      end
+    end
+
+    module DatumReader
+      def read_default_value(field_schema, default_value)
+        if default_value == :no_default
+          raise Avro::AvroError, "Missing data for #{field_schema} with no default"
+        end
+
+        super
+      end
+    end
+  end
+end
+
+Avro::Schema::RecordSchema.send(:prepend, AvroTurf::AvroGemPatch::RecordSchema)
+Avro::Schema::Field.send(:prepend, AvroTurf::AvroGemPatch::Field)
+Avro::IO::DatumReader.send(:prepend, AvroTurf::AvroGemPatch::DatumReader)

data/lib/avro_turf/test/fake_schema_registry_server.rb

@@ -0,0 +1,84 @@
+require 'sinatra/base'
+
+class FakeSchemaRegistryServer < Sinatra::Base
+  SUBJECTS = Hash.new { Array.new }
+  SCHEMAS = []
+  SUBJECT_NOT_FOUND = { error_code: 40401, message: 'Subject not found' }.to_json.freeze
+  VERSION_NOT_FOUND = { error_code: 40402, message: 'Version not found' }.to_json.freeze
+  SCHEMA_NOT_FOUND = { error_code: 40403, message: 'Schema not found' }.to_json.freeze
+
+  helpers do
+    def parse_schema
+      request.body.rewind
+      JSON.parse(request.body.read).fetch("schema").tap do |schema|
+        Avro::Schema.parse(schema)
+      end
+    end
+  end
+
+  post "/subjects/:subject/versions" do
+    SCHEMAS << parse_schema
+
+    schema_id = SCHEMAS.size - 1
+    SUBJECTS[params[:subject]] = SUBJECTS[params[:subject]] << schema_id
+    { id: schema_id }.to_json
+  end
+
+  get "/schemas/ids/:schema_id" do
+    schema = SCHEMAS.at(params[:schema_id].to_i)
+    halt(404, SCHEMA_NOT_FOUND) unless schema
+    { schema: schema }.to_json
+  end
+
+  get "/subjects" do
+    SUBJECTS.keys.to_json
+  end
+
+  get "/subjects/:subject/versions" do
+    schema_ids = SUBJECTS[params[:subject]]
+    halt(404, SUBJECT_NOT_FOUND) if schema_ids.empty?
+    (1..schema_ids.size).to_a.to_json
+  end
+
+  get "/subjects/:subject/versions/:version" do
+    schema_ids = SUBJECTS[params[:subject]]
+    halt(404, SUBJECT_NOT_FOUND) if schema_ids.empty?
+
+    schema_id = if params[:version] == 'latest'
+                  schema_ids.last
+                else
+                  schema_ids.at(Integer(params[:version]) - 1)
+                end
+    halt(404, VERSION_NOT_FOUND) unless schema_id
+
+    schema = SCHEMAS.at(schema_id)
+
+    {
+      name: params[:subject],
+      version: schema_ids.index(schema_id) + 1,
+      schema: schema
+    }.to_json
+  end
+
+  post "/subjects/:subject" do
+    schema = parse_schema
+
+    # Note: this does not actually handle the same schema registered under
+    # multiple subjects
+    schema_id = SCHEMAS.index(schema)
+
+    halt(404, SCHEMA_NOT_FOUND) unless schema_id
+
+    {
+      subject: params[:subject],
+      id: schema_id,
+      version: SUBJECTS[params[:subject]].index(schema_id) + 1,
+      schema: schema
+    }.to_json
+  end
+
+  def self.clear
+    SUBJECTS.clear
+    SCHEMAS.clear
+  end
+end

data/lib/avro_turf/version.rb

@@ -0,0 +1,3 @@
+class AvroTurf
+  VERSION = "0.7.3"
+end

data/perf/address.avsc

@@ -0,0 +1,14 @@
+{
+  "name": "address",
+  "type": "record",
+  "fields": [
+    {
+      "name": "street",
+      "type": "string"
+    },
+    {
+      "name": "city",
+      "type": "string"
+    }
+  ]
+}