logstash-codec-avro 3.1.0-java → 3.2.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 800d769a22f692329606bfe7b30fef7b22894038
-  data.tar.gz: 62f39efb3bcaf5074dd81d1b657f08a6ec857664
+  metadata.gz: 118c841f2c7a535679e7259b93799339c12997ea
+  data.tar.gz: 1f293ccd56c2b30fd09e776d728c077edbeb69f2
 SHA512:
-  metadata.gz: 2a17a7e2514b3f8b97f019ff6bf0f5a8b6b68b19c41e6c5f95b324a79eb3ff72374276d37fde76f2a3f5c58384cf5466de73bd4c0d570f228a00c3d6d96ebbaa
-  data.tar.gz: 07cef39e0789d1638f9f55f5edf1e290358f445ce878642cd00bea65d54638baadf54c743f9e7721028d9dfbb0b03baf8e7180b1ac07f4c452255e6edca38607
+  metadata.gz: ce35c4b91b1b97d55d9a5142ca4d0796b9663cf50dd41f927f077989cbc676cee37891b3f352731bf78e040940471d6e6c3cae62cc3360d531f9ad7d1df4b30b
+  data.tar.gz: 1afb54a0aac2920568ca0f048edb4c451b5151333d36eb754b79954f85b38829004da91280a6f711ad05271413fca780e41a82b58332f0915183ddad9412d340

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.2.0
+ - Fixed an issue with the encoding that prevented certain fields from being serialized in a way compatible with the Kafka plugins
+
 ## 3.1.0
  - Introduce `tag_on_failure` option to tag events with `_avroparsefailure` instead of throwing an exception when decoding
 

data/docs/index.asciidoc

@@ -0,0 +1,103 @@
+:plugin: avro
+:type: codec
+
+///////////////////////////////////////////
+START - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+:version: %VERSION%
+:release_date: %RELEASE_DATE%
+:changelog_url: %CHANGELOG_URL%
+:include_path: ../../../logstash/docs/include
+///////////////////////////////////////////
+END - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+
+[id="plugins-{type}-{plugin}"]
+
+=== Avro
+
+include::{include_path}/plugin_header.asciidoc[]
+
+==== Description
+
+Read serialized Avro records as Logstash events
+
+This plugin is used to serialize Logstash events as 
+Avro datums, as well as deserializing Avro datums into 
+Logstash events.
+
+==== Encoding
+
+This codec is for serializing individual Logstash events 
+as Avro datums that are Avro binary blobs. It does not encode 
+Logstash events into an Avro file.
+
+
+==== Decoding
+
+This codec is for deserializing individual Avro records. It is not for reading
+Avro files. Avro files have a unique format that must be handled upon input.
+
+
+==== Usage
+Example usage with Kafka input.
+
+[source,ruby]
+----------------------------------
+input {
+  kafka {
+    codec => avro {
+        schema_uri => "/tmp/schema.avsc"
+    }
+  }
+}
+filter {
+  ...
+}
+output {
+  ...
+}
+----------------------------------
+
+[id="plugins-{type}s-{plugin}-options"]
+==== Avro Codec Configuration Options
+
+This plugin supports the following configuration options plus the <<plugins-{type}s-common-options>> described later.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-schema_uri>> |<<string,string>>|Yes
+| <<plugins-{type}s-{plugin}-tag_on_failure>> |<<boolean,boolean>>|No
+|=======================================================================
+
+Also see <<plugins-{type}s-common-options>> for a list of options supported by all
+codec plugins.
+
+&nbsp;
+
+[id="plugins-{type}s-{plugin}-schema_uri"]
+===== `schema_uri` 
+
+  * This is a required setting.
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+schema path to fetch the schema from.
+This can be a 'http' or 'file' scheme URI
+example:
+
+* http - `http://example.com/schema.avsc`
+* file - `/path/to/schema.avsc`
+
+[id="plugins-{type}s-{plugin}-tag_on_failure"]
+===== `tag_on_failure` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+tag events with `_avroparsefailure` when decode fails
+
+
+
+include::{include_path}/{type}.asciidoc[]

data/lib/logstash/codecs/avro.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 require "open-uri"
 require "avro"
+require "base64"
 require "logstash/codecs/base"
 require "logstash/event"
 require "logstash/timestamp"
@@ -70,7 +71,7 @@ class LogStash::Codecs::Avro < LogStash::Codecs::Base
 
   public
   def decode(data)
-    datum = StringIO.new(data)
+    datum = StringIO.new(Base64.strict_decode64(data)) rescue StringIO.new(data)
     decoder = Avro::IO::BinaryDecoder.new(datum)
     datum_reader = Avro::IO::DatumReader.new(@schema)
     yield LogStash::Event.new(datum_reader.read(decoder))
@@ -89,6 +90,6 @@ class LogStash::Codecs::Avro < LogStash::Codecs::Base
     buffer = StringIO.new
     encoder = Avro::IO::BinaryEncoder.new(buffer)
     dw.write(event.to_hash, encoder)
-    @on_event.call(event, buffer.string)
+    @on_event.call(event, Base64.strict_encode64(buffer.string))
   end
 end

data/logstash-codec-avro.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-avro'
-  s.version         = '3.1.0'
+  s.version         = '3.2.0'
   s.platform        = 'java'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Encode and decode avro formatted data"
@@ -12,7 +12,7 @@ Gem::Specification.new do |s|
   s.require_paths   = ["lib"]
 
   # Files
-  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})

data/spec/codecs/avro_spec.rb

@@ -1,70 +1,114 @@
 # encoding: utf-8
-require "logstash/devutils/rspec/spec_helper"
+require 'logstash/devutils/rspec/spec_helper'
 require 'avro'
+require 'base64'
 require 'logstash/codecs/avro'
 require 'logstash/event'
 
 describe LogStash::Codecs::Avro do
-  let (:avro_config) {{'schema_uri' => '
+
+  context "non binary data" do
+    let (:avro_config) {{ 'schema_uri' => '
                         {"type": "record", "name": "Test",
                         "fields": [{"name": "foo", "type": ["null", "string"]},
-                                   {"name": "bar", "type": "int"}]}'}}
-  let (:test_event) { LogStash::Event.new({"foo" => "hello", "bar" => 10}) }
+                                   {"name": "bar", "type": "int"}]}' }}
+    let (:test_event) {LogStash::Event.new({ "foo" => "hello", "bar" => 10 })}
 
-  subject do
-    allow_any_instance_of(LogStash::Codecs::Avro).to \
+    subject do
+      allow_any_instance_of(LogStash::Codecs::Avro).to \
       receive(:open_and_read).and_return(avro_config['schema_uri'])
-    next LogStash::Codecs::Avro.new(avro_config)
-  end
+      next LogStash::Codecs::Avro.new(avro_config)
+    end
 
-  context "#decode" do
-    it "should return an LogStash::Event from avro data" do
-      schema = Avro::Schema.parse(avro_config['schema_uri'])
-      dw = Avro::IO::DatumWriter.new(schema)
-      buffer = StringIO.new
-      encoder = Avro::IO::BinaryEncoder.new(buffer)
-      dw.write(test_event.to_hash, encoder)
+    context "#decode" do
+      it "should return an LogStash::Event from raw and base64 encoded avro data" do
+        schema = Avro::Schema.parse(avro_config['schema_uri'])
+        dw = Avro::IO::DatumWriter.new(schema)
+        buffer = StringIO.new
+        encoder = Avro::IO::BinaryEncoder.new(buffer)
+        dw.write(test_event.to_hash, encoder)
 
-      subject.decode(buffer.string) do |event|
-        insist { event.is_a? LogStash::Event }
-        insist { event.get("foo") } == test_event.get("foo")
-        insist { event.get("bar") } == test_event.get("bar")
+        subject.decode(Base64.strict_encode64(buffer.string)) do |event|
+          insist {event.is_a? LogStash::Event}
+          insist {event.get("foo")} == test_event.get("foo")
+          insist {event.get("bar")} == test_event.get("bar")
+        end
+        subject.decode(buffer.string) do |event|
+          insist {event.is_a? LogStash::Event}
+          insist {event.get("foo")} == test_event.get("foo")
+          insist {event.get("bar")} == test_event.get("bar")
+        end
       end
-    end
 
-    it "should throw exception if decoding fails" do
-      expect { subject.decode("not avro") { |_| } }.to raise_error NoMethodError
+      it "should throw exception if decoding fails" do
+        expect {subject.decode("not avro") {|_| }}.to raise_error NoMethodError
+      end
     end
-  end
 
-  context "#decode with tag_on_failure" do
-    let (:avro_config) { super.merge("tag_on_failure" => true) }
+    context "#decode with tag_on_failure" do
+      let (:avro_config) {super.merge("tag_on_failure" => true)}
 
-    it "should tag event on failure" do
-      subject.decode("not avro") do |event|
-        insist { event.is_a? LogStash::Event }
-        insist { event.get("tags") } == ["_avroparsefailure"]
+      it "should tag event on failure" do
+        subject.decode("not avro") do |event|
+          insist {event.is_a? LogStash::Event}
+          insist {event.get("tags")} == ["_avroparsefailure"]
+        end
       end
     end
-  end
 
-  context "#encode" do
-    it "should return avro data from a LogStash::Event" do
-      got_event = false
-      subject.on_event do |event, data|
-        schema = Avro::Schema.parse(avro_config['schema_uri'])
-        datum = StringIO.new(data)
-        decoder = Avro::IO::BinaryDecoder.new(datum)
-        datum_reader = Avro::IO::DatumReader.new(schema)
-        record = datum_reader.read(decoder)
+    context "#encode" do
+      it "should return avro data from a LogStash::Event" do
+        got_event = false
+        subject.on_event do |event, data|
+          schema = Avro::Schema.parse(avro_config['schema_uri'])
+          datum = StringIO.new(Base64.strict_decode64(data))
+          decoder = Avro::IO::BinaryDecoder.new(datum)
+          datum_reader = Avro::IO::DatumReader.new(schema)
+          record = datum_reader.read(decoder)
+
+          insist {record["foo"]} == test_event.get("foo")
+          insist {record["bar"]} == test_event.get("bar")
+          insist {event.is_a? LogStash::Event}
+          got_event = true
+        end
+        subject.encode(test_event)
+        insist {got_event}
+      end
+
+      context "binary data" do
+
+        let (:avro_config) {{ 'schema_uri' => '{"namespace": "com.systems.test.data",
+                      "type": "record",
+                      "name": "TestRecord",
+                      "fields": [
+                        {"name": "name", "type": ["string", "null"]},
+                        {"name": "longitude", "type": ["double", "null"]},
+                        {"name": "latitude", "type": ["double", "null"]}
+                      ]
+                    }' }}
+        let (:test_event) {LogStash::Event.new({ "name" => "foo", "longitude" => 21.01234.to_f, "latitude" => 111.0123.to_f })}
+
+        subject do
+          allow_any_instance_of(LogStash::Codecs::Avro).to \
+      receive(:open_and_read).and_return(avro_config['schema_uri'])
+          next LogStash::Codecs::Avro.new(avro_config)
+        end
+
+        it "should correctly encode binary data" do
+          schema = Avro::Schema.parse(avro_config['schema_uri'])
+          dw = Avro::IO::DatumWriter.new(schema)
+          buffer = StringIO.new
+          encoder = Avro::IO::BinaryEncoder.new(buffer)
+          dw.write(test_event.to_hash, encoder)
 
-        insist { record["foo"] } == test_event.get("foo")
-        insist { record["bar"] } == test_event.get("bar")
-        insist { event.is_a? LogStash::Event }
-        got_event = true
+          subject.decode(Base64.strict_encode64(buffer.string)) do |event|
+            insist {event.is_a? LogStash::Event}
+            insist {event.get("name")} == test_event.get("name")
+            insist {event.get("longitude")} == test_event.get("longitude")
+            insist {event.get("latitude")} == test_event.get("latitude")
+          end
+        end
       end
-      subject.encode(test_event)
-      insist { got_event }
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-avro
 version: !ruby/object:Gem::Version
-  version: 3.1.0
+  version: 3.2.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-02-24 00:00:00.000000000 Z
+date: 2017-05-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -71,6 +71,7 @@ files:
 - LICENSE
 - NOTICE.TXT
 - README.md
+- docs/index.asciidoc
 - lib/logstash/codecs/avro.rb
 - logstash-codec-avro.gemspec
 - spec/codecs/avro_spec.rb