logstash-codec-avro 3.3.1-java → 3.4.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 772c2d05da7c85d818be97b57d65587730bc7204624199b670ba1f40de35b8a6
-  data.tar.gz: 124b77e35ebc93783576228831ee427081bc9b052535abe8f20108cf962293f4
+  metadata.gz: 58f5ee3b7dd49fc9d5ea8a85c8b91cd042e67027a6b80440b48f3c943f149397
+  data.tar.gz: f0321dffe64cc44165dd5ccfc95fb4d8b7cbcff561aa8f1a048f7c79f53881ba
 SHA512:
-  metadata.gz: 42b7e3f92cca071b122851cd18c79735c109570ba42cd4ce82ab87bd19bd887297006d9822d7fecbc5b54e1034ba33b33ab4f98b1c0f1cbb76df4e5f6e3ce36f
-  data.tar.gz: 79771d37cc0b6dde478dff2a1f09f026ce644967aaeaf7a8e6ddaf77318d995f17301da3c681391dacd1f8bce9a5ae5e43d6ac5e14b006a93845cc87ae386300
+  metadata.gz: 9af8121f74ead4621f3979b638125c209e6790248d81681fdeb060f886d0f84e79cbb560c6c009d1cc21c5e94120e484830dd26186d93a5726c08e2187ab744f
+  data.tar.gz: 13e6a4db21017088f70f062d054625547b728a633788fdd5478f13a81746e9f9f23840cc73140fa8b82617f827c9a746b586b5fca669db300a08ef0209b4b06d

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 3.4.1
+  - Fixes `(Errno::ENOENT) No such file or directory` error [#43](https://github.com/logstash-plugins/logstash-codec-avro/pull/43)
+
+## 3.4.0
+  - Add `encoding` option to select the encoding of Avro payload, could be `binary` or `base64` [#39](https://github.com/logstash-plugins/logstash-codec-avro/pull/39)
+
 ## 3.3.1
   - Pin avro gem to 1.10.x, as 1.11+ requires ruby 2.6+ [#37](https://github.com/logstash-plugins/logstash-codec-avro/pull/37)
 

data/docs/index.asciidoc

@@ -82,6 +82,7 @@ output {
 |=======================================================================
 |Setting |Input type|Required
 | <<plugins-{type}s-{plugin}-ecs_compatibility>> | <<string,string>>|No
+| <<plugins-{type}s-{plugin}-encoding>> | <<string,string>>, one of `["binary", "base64"]`|No
 | <<plugins-{type}s-{plugin}-schema_uri>> |<<string,string>>|Yes
 | <<plugins-{type}s-{plugin}-tag_on_failure>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-target>> |<<string,string>>|No
@@ -99,6 +100,17 @@ output {
 
 Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)].
 
+[id="plugins-{type}s-{plugin}-encoding"]
+===== `encoding`
+
+* Value can be any of: `binary`, `base64`
+* Default value is `base64`
+
+Set encoding for Avro's payload. 
+Use `base64` (default) to indicate that this codec sends or expects to receive base64-encoded bytes.
+
+Set this option to `binary` to indicate that this codec sends or expects to receive binary Avro data.
+
 
 [id="plugins-{type}s-{plugin}-schema_uri"]
 ===== `schema_uri` 

data/lib/logstash/codecs/avro.rb

@@ -59,6 +59,14 @@ class LogStash::Codecs::Avro < LogStash::Codecs::Base
 
   include LogStash::PluginMixins::EventSupport::EventFactoryAdapter
 
+  BINARY_ENCODING = "binary".freeze
+  BASE64_ENCODING = "base64".freeze
+
+  # Set encoding for Avro's payload.
+  #  Use `base64` (default) encoding to convert the raw binary bytes to a `base64` encoded string.
+  #  Set this option to `binary` to use the plain binary bytes.
+  config :encoding, :validate => [BINARY_ENCODING, BASE64_ENCODING], :default => BASE64_ENCODING
+
   # schema path to fetch the schema from.
   # This can be a 'http' or 'file' scheme URI
   # example:
@@ -77,7 +85,7 @@ class LogStash::Codecs::Avro < LogStash::Codecs::Base
   config :target, :validate => :field_reference
 
   def open_and_read(uri_string)
-    open(uri_string).read
+    URI.open(uri_string, &:read)
   end
 
   public
@@ -92,7 +100,11 @@ class LogStash::Codecs::Avro < LogStash::Codecs::Base
 
   public
   def decode(data)
-    datum = StringIO.new(Base64.strict_decode64(data)) rescue StringIO.new(data)
+    if encoding == BASE64_ENCODING
+      datum = StringIO.new(Base64.strict_decode64(data)) rescue StringIO.new(data)
+    else
+      datum = StringIO.new(data)
+    end
     decoder = Avro::IO::BinaryDecoder.new(datum)
     datum_reader = Avro::IO::DatumReader.new(@schema)
     event = targeted_event_factory.new_event(datum_reader.read(decoder))
@@ -113,6 +125,10 @@ class LogStash::Codecs::Avro < LogStash::Codecs::Base
     buffer = StringIO.new
     encoder = Avro::IO::BinaryEncoder.new(buffer)
     dw.write(event.to_hash, encoder)
-    @on_event.call(event, Base64.strict_encode64(buffer.string))
+    if encoding == BASE64_ENCODING
+      @on_event.call(event, Base64.strict_encode64(buffer.string))
+    else
+      @on_event.call(event, buffer.string)
+    end  
   end
 end

data/logstash-codec-avro.gemspec

@@ -1,14 +1,14 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-avro'
-  s.version         = '3.3.1'
+  s.version         = '3.4.1'
   s.platform        = 'java'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Reads serialized Avro records as Logstash events"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors         = ["Elastic"]
   s.email           = 'info@elastic.co'
-  s.homepage        = "http://www.elastic.co/guide/en/logstash/current/index.html"
+  s.homepage        = "https://www.elastic.co/logstash"
   s.require_paths   = ["lib"]
 
   # Files

data/spec/codecs/avro_spec.rb

@@ -55,6 +55,35 @@ describe LogStash::Codecs::Avro, :ecs_compatibility_support, :aggregate_failures
         end
       end
 
+      context "with binary encoding" do
+        let (:avro_config) { super().merge('encoding' => 'binary') }
+
+        it "should return an LogStash::Event from raw and base64 encoded avro data" do
+          schema = Avro::Schema.parse(avro_config['schema_uri'])
+          dw = Avro::IO::DatumWriter.new(schema)
+          buffer = StringIO.new
+          encoder = Avro::IO::BinaryEncoder.new(buffer)
+          dw.write(test_event.to_hash, encoder)
+
+          subject.decode(buffer.string) do |event|
+            expect(event).to be_a_kind_of(LogStash::Event)
+            expect(event.get("foo")).to eq(test_event.get("foo"))
+            expect(event.get("bar")).to eq(test_event.get("bar"))
+            expect(event.get('[event][original]')).to eq(buffer.string) if ecs_compatibility != :disabled
+          end
+        end
+
+        it "should raise an error if base64 encoded data is provided" do
+          schema = Avro::Schema.parse(avro_config['schema_uri'])
+          dw = Avro::IO::DatumWriter.new(schema)
+          buffer = StringIO.new
+          encoder = Avro::IO::BinaryEncoder.new(buffer)
+          dw.write(test_event.to_hash, encoder)
+
+          expect {subject.decode(Base64.strict_encode64(buffer.string))}.to raise_error
+        end
+      end
+
       context "#decode with tag_on_failure" do
         let (:avro_config) {{ 'schema_uri' => '
                         {"type": "record", "name": "Test",
@@ -111,6 +140,28 @@ describe LogStash::Codecs::Avro, :ecs_compatibility_support, :aggregate_failures
           insist {got_event}
         end
 
+        context "with binary encoding" do
+          let (:avro_config) { super().merge('encoding' => 'binary') }
+
+          it "should return avro data from a LogStash::Event not base64 encoded" do
+            got_event = false
+            subject.on_event do |event, data|
+              schema = Avro::Schema.parse(avro_config['schema_uri'])
+              datum = StringIO.new(data)
+              decoder = Avro::IO::BinaryDecoder.new(datum)
+              datum_reader = Avro::IO::DatumReader.new(schema)
+              record = datum_reader.read(decoder)
+
+              expect(event).to be_a_kind_of(LogStash::Event)
+              expect(event.get("foo")).to eq(test_event.get("foo"))
+              expect(event.get("bar")).to eq(test_event.get("bar"))
+              got_event = true
+            end
+            subject.encode(test_event)
+            expect(got_event).to be true
+          end
+        end
+
         context "binary data" do
 
           let (:avro_config) {{ 'schema_uri' => '{"namespace": "com.systems.test.data",

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-avro
 version: !ruby/object:Gem::Version
-  version: 3.3.1
+  version: 3.4.1
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-07 00:00:00.000000000 Z
+date: 2023-10-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -133,7 +133,7 @@ files:
 - lib/logstash/codecs/avro.rb
 - logstash-codec-avro.gemspec
 - spec/codecs/avro_spec.rb
-homepage: http://www.elastic.co/guide/en/logstash/current/index.html
+homepage: https://www.elastic.co/logstash
 licenses:
 - Apache-2.0
 metadata:
@@ -154,7 +154,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.2.33
 signing_key:
 specification_version: 4
 summary: Reads serialized Avro records as Logstash events