logstash-codec-avro 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: bb0bcc66f0b7e43cae0ac6936ddda5f7cf65dc44
+  data.tar.gz: 095ac47a666d8b0db08757e07132021db22f83e4
+SHA512:
+  metadata.gz: 69e7e6ca8fcbe43db61ff6ed8246507e0df04e253cb1aaebb0549256bc017c634a12032337a6b9cf1dc539b353b05c6afd8e1dc3f71ad17c001888d5aa8f466b
+  data.tar.gz: 6f738e2a34e9de2acdf2a67f9d7b982bd8859c2bf5bf1f77d431d227d7901c4fff0960b5b1d1e7a13d8300a4d006c49c3ea70f44cd962192c4702329db21a096

data/.gitignore

@@ -0,0 +1,3 @@
+*.gem
+Gemfile.lock
+.bundle

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+gemspec
+gem "logstash", :github => "elasticsearch/logstash", :branch => "1.5"

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2012-2014 Elasticsearch <http://www.elasticsearch.org>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,66 @@
+Logstash Avro Codec
+===================
+
+How to Install
+--------------
+
+```
+bin/plugin install logstash-avro-codec
+```
+
+How to Use
+----------
+You can use this codec to decode avro messages
+in a Kafka topic input.
+
+Here is an example schema for tweets.
+
+### tweet.avsc
+```
+{
+  "type" : "record",
+  "name" : "twitter_schema",
+  "namespace" : "com.miguno.avro",
+  "fields" : [ {
+    "name" : "username",
+    "type" : "string",
+    "doc" : "Name of the user account on Twitter.com"
+  }, {
+    "name" : "tweet",
+    "type" : "string",
+    "doc" : "The content of the user's Twitter message"
+  }, {
+    "name" : "timestamp",
+    "type" : "long",
+    "doc" : "Unix epoch time in seconds"
+  } ],
+  "doc:" : "A basic schema for storing Twitter messages"
+}
+```
+
+Along with the logstash config for reading in messages of this 
+type using the avro codec with the logstash-input-kafka plugin.
+
+### logstash.conf
+
+```
+input {
+  kafka {
+    topic_id => 'test_topic'
+      codec => avro {
+        schema_file => 'tweet.avsc'
+      }
+  }
+}
+
+output {
+  stdout {
+    codec => rubydebug
+  }
+}
+```
+
+### Running the setup
+```
+bin/logstash -f logstash.conf
+```

data/Rakefile

@@ -0,0 +1,7 @@
+@files=[]
+
+task :default do
+    system("rake -T")
+end
+
+require "logstash/devutils/rake"

data/lib/logstash/codecs/avro.rb

@@ -0,0 +1,46 @@
+# encoding: utf-8
+require "open-uri"
+require "avro"
+require "logstash/codecs/base"
+require "logstash/event"
+require "logstash/timestamp"
+require "logstash/util"
+
+class LogStash::Codecs::Avro < LogStash::Codecs::Base
+  config_name "avro"
+
+  milestone 1
+
+  # schema path to fetch the schema from
+  # This can be a 'http' or 'file' scheme URI
+  # example:
+  #     http - "http://example.com/schema.avsc"
+  #     file - "/path/to/schema.avsc"
+  config :schema_uri, :validate => :string, :required => true
+
+  def open_and_read(uri_string)
+    open(uri_string).read
+  end
+
+  public
+  def register
+    @schema = Avro::Schema.parse(open_and_read(schema_uri))
+  end
+
+  public
+  def decode(data)
+    datum = StringIO.new(data)
+    decoder = Avro::IO::BinaryDecoder.new(datum)
+    datum_reader = Avro::IO::DatumReader.new(@schema)
+    yield LogStash::Event.new(datum_reader.read(decoder))
+  end
+
+  public
+  def encode(event)
+    dw = Avro::IO::DatumWriter.new(@schema)
+    buffer = StringIO.new
+    encoder = Avro::IO::BinaryEncoder.new(buffer)
+    dw.write(event.to_hash, encoder)
+    @on_event.call(buffer.string)
+  end
+end

data/logstash-codec-avro.gemspec

@@ -0,0 +1,29 @@
+Gem::Specification.new do |s|
+
+  s.name            = 'logstash-codec-avro'
+  s.version         = '0.1.0'
+  s.licenses        = ['Apache License (2.0)']
+  s.summary         = "Encode and decode avro formatted data"
+  s.description     = "Encode and decode avro formatted data"
+  s.authors         = ["Elasticsearch"]
+  s.email           = 'info@elasticsearch.com'
+  s.homepage        = "http://www.elasticsearch.org/guide/en/logstash/current/index.html"
+  s.require_paths   = ["lib"]
+
+  # Files
+  s.files = `git ls-files`.split($\)
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "codec" }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
+
+  s.add_runtime_dependency "avro"  #(Apache 2.0 license)
+
+  s.add_development_dependency 'logstash-devutils'
+end
+

data/spec/codecs/avro_spec.rb

@@ -0,0 +1,54 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require 'avro'
+require 'logstash/codecs/avro'
+require 'logstash/event'
+
+describe LogStash::Codecs::Avro do
+  let (:avro_config) {{'schema_uri' => '
+                        {"type": "record", "name": "Test",
+                        "fields": [{"name": "foo", "type": ["null", "string"]},
+                                   {"name": "bar", "type": "int"}]}'}}
+  let (:test_event) { LogStash::Event.new({"foo" => "hello", "bar" => 10}) }
+
+  subject do
+    allow_any_instance_of(LogStash::Codecs::Avro).to \
+      receive(:open_and_read).and_return(avro_config['schema_uri'])
+    next LogStash::Codecs::Avro.new(avro_config)
+  end
+
+  context "#decode" do
+    it "should return an LogStash::Event from avro data" do
+      schema = Avro::Schema.parse(avro_config['schema_uri'])
+      dw = Avro::IO::DatumWriter.new(schema)
+      buffer = StringIO.new
+      encoder = Avro::IO::BinaryEncoder.new(buffer)
+      dw.write(test_event.to_hash, encoder)
+
+      subject.decode(buffer.string) do |event|
+        insist { event.is_a? LogStash::Event }
+        insist { event["foo"] } == test_event["foo"]
+        insist { event["bar"] } == test_event["bar"]
+      end
+    end
+  end
+
+  context "#encode" do
+    it "should return avro data from a LogStash::Event" do
+      got_event = false
+      subject.on_event do |data|
+        schema = Avro::Schema.parse(avro_config['schema_uri'])
+        datum = StringIO.new(data)
+        decoder = Avro::IO::BinaryDecoder.new(datum)
+        datum_reader = Avro::IO::DatumReader.new(schema)
+        record = datum_reader.read(decoder)
+
+        insist { record["foo"] } == test_event["foo"]
+        insist { record["bar"] } == test_event["bar"]
+        got_event = true
+      end
+      subject.encode(test_event)
+      insist { got_event }
+    end
+  end
+end

metadata

@@ -0,0 +1,102 @@
+--- !ruby/object:Gem::Specification
+name: logstash-codec-avro
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Elasticsearch
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2014-12-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: avro
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+description: Encode and decode avro formatted data
+email: info@elasticsearch.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- lib/logstash/codecs/avro.rb
+- logstash-codec-avro.gemspec
+- spec/codecs/avro_spec.rb
+homepage: http://www.elasticsearch.org/guide/en/logstash/current/index.html
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: codec
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: Encode and decode avro formatted data
+test_files:
+- spec/codecs/avro_spec.rb