avro-registered-schema-decoder 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d490a609d6c3b2d0d1ecceba3019d1392021b7c6
+  data.tar.gz: 6e10f76246cabc413f8a1e602bf0f10ae0e9d1e2
+SHA512:
+  metadata.gz: c87c293bd4b20094cf66105e0cf09db28bae70e8c0609f66ebc3025d7058ab25d186887f05928e36a2add66c38bbf322c5e467bf153edaeb04d9abf78930489c
+  data.tar.gz: 3918b0ed6311532def01d253db08c71c4a5e3921a735bf5cd9fc704d3210adb56769bb640df618d1274133cc81e98de8438b6764b8e3049ca3a4aff1306efa2b

data/lib/avro-registered-schema-decoder.rb

@@ -0,0 +1 @@
+require File.join(File.dirname(__FILE__), 'avro', 'registered_schema_decoder')

data/lib/avro/registered_schema_decoder.rb

@@ -0,0 +1,97 @@
+require 'avro'
+require 'schema_registry'
+
+module Avro
+  class RegisteredSchemaDecoder
+    class Error < StandardError
+      attr_reader :cause
+      def initialize(message, cause = $!)
+        super(message)
+        @cause = cause
+      end
+
+      def to_s
+        "#{super}#{" (caused by: #{cause})" if cause}"
+      end
+    end
+
+    def initialize(registry_url, logger: nil)
+      @registry = SchemaRegistry::Client.new(registry_url)
+      @logger = logger
+      @readers = {}
+    end
+
+    def decode_key(key_bytes)
+      _, key = decode_key_with_schema(key_bytes)
+      key
+    end
+
+    def decode_key_with_schema(key_bytes)
+      # expected for unkeyed messages
+      return if key_bytes.nil?
+
+      decode_message(key_bytes)
+    end
+
+    def decode_value(value_bytes)
+      _, value = decode_value_with_schema(value_bytes)
+      value
+    end
+
+    def decode_value_with_schema(value_bytes)
+      # expected, used to signify record deletion
+      return if value_bytes.nil?
+      # handle producers using older versions of librdkafka, which sent zero-
+      # length messages instead of null messages:
+      # see https://github.com/confluentinc/bottledwater-pg/pull/33
+      return if value_bytes.empty?
+
+      decode_message(value_bytes)
+    end
+
+    private
+    # message format:
+    #
+    # Every message sent to Kafka is Avro-encoded, and prefixed with five
+    # bytes:
+    #  - The first byte is always 0, and reserved for future use.
+    #  - The next four bytes are the schema ID in big-endian byte order.
+    #
+    # per https://github.com/confluentinc/bottledwater-pg/blob/8a11825/kafka/registry.c#L6-L8
+    def decode_message(bytes)
+      raise "Empty message" if bytes.empty?
+      reserved, schema_id, avro = bytes.unpack('cNa*')
+      raise "Reserved byte #{reserved.inspect} in message header (expected 0)!\nmessage: #{bytes.inspect}" unless 0 == reserved
+
+      reader = reader_for(schema_id)
+      decoder = Avro::IO::BinaryDecoder.new(StringIO.new(avro))
+
+      wrap_error "parsing message with schema #{schema_id}" do
+        [reader.writers_schema, reader.read(decoder)]
+      end
+    end
+
+    def reader_for(schema_id)
+      if @readers.key?(schema_id)
+        return @readers[schema_id]
+      end
+
+      @logger.debug("Fetching schema #{schema_id}") if @logger
+      schema = nil
+      wrap_error "fetching schema #{schema_id}" do
+        schema_json = @registry.schema(schema_id)
+        schema = Avro::Schema.parse(schema_json)
+      end
+
+      Avro::IO::DatumReader.new(schema).tap do |reader|
+        @readers[schema_id] = reader
+      end
+    end
+
+    def wrap_error(doing_what)
+      yield
+    rescue
+      raise Error, "error #{doing_what}"
+    end
+  end
+end

metadata

@@ -0,0 +1,121 @@
+--- !ruby/object:Gem::Specification
+name: avro-registered-schema-decoder
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Sam Stokes
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-03-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: avro
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: schema_registry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: |
+  Provides a decoder for messages encoded as "prefixed Avro", where the binary
+  Avro encoding is prefixed with a schema id referring to a schema stored in a
+  Confluent Schema Registry
+  (http://docs.confluent.io/1.0/schema-registry/docs/intro.html).
+email:
+- me@samstokes.co.uk
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/avro-registered-schema-decoder.rb
+- lib/avro/registered_schema_decoder.rb
+homepage: https://github.com/samstokes/avro-registered-schema-decoder
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5.1
+signing_key: 
+specification_version: 4
+summary: Decodes messages encoded as prefixed Avro, looking up schemas in a schema
+  registry
+test_files: []