logstash-codec-fluent 3.0.2-java → 3.1.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: eb123960167e676111c8e007bec4f2afa4f8e3e8
-  data.tar.gz: 6d38eda36c07509e76c5301e9e11c45400498027
+  metadata.gz: 2c9c31b9f410df8d1944039aa31ea73374aa0a82
+  data.tar.gz: 429c22a14929f5aa6d370fe7b4047435af915e01
 SHA512:
-  metadata.gz: fa6495a51fb97482d79ae95375f507bc2e9430b45655ec205492d3a0ec3a1b9f7bde17d6c384b64e63d3e431ba941e2c8becbb4aa435b395f6ff9bbf0222ad1e
-  data.tar.gz: 901c19d737da2a96e8eaa0ad5515b0f92e21520b6d75871a2777b55dff46b90886fa18cf670983afe1d91fc84051eb875065560b10476760c590526a6fcc0403
+  metadata.gz: 9486c210461b70b5aea183e803f8574b3e4f42279dda27a02577a6f6de65eb4b4973a765a936f8af7ee14c88514aa7b06cbe2371986b22152151de2857186190
+  data.tar.gz: df6c52150440881313b96a28bccd70fc87a9f618770dd61122dccf1509d617f0432acd0c55e0db63aae8977fade0aeb87f5ec6142ba521b682348cf654ffbf0c

data/lib/logstash/codecs/fluent.rb

@@ -28,25 +28,17 @@ require "logstash/util"
 class LogStash::Codecs::Fluent < LogStash::Codecs::Base
   config_name "fluent"
 
-  public
   def register
     require "msgpack"
     @decoder = MessagePack::Unpacker.new
   end
 
-  public
-  def decode(data)
-    @decoder.feed(data)
-    @decoder.each do |tag, epochtime, map|
-      event = LogStash::Event.new(map.merge(
-        LogStash::Event::TIMESTAMP => LogStash::Timestamp.at(epochtime),
-        "tags" => tag
-      ))
-      yield event
+  def decode(data, &block)
+    @decoder.feed_each(data) do |item|
+      decode_event(item, &block)
     end
   end # def decode
 
-  public
   def encode(event)
     tag = event.get("tags") || "log"
     epochtime = event.timestamp.to_i
@@ -59,4 +51,57 @@ class LogStash::Codecs::Fluent < LogStash::Codecs::Base
     @on_event.call(event, MessagePack.pack([tag, epochtime, data.merge(LogStash::Event::TIMESTAMP => event.timestamp.to_iso8601)]))
   end # def encode
 
+  private
+
+  def decode_event(data, &block)
+    tag = data[0]
+    entries = data[1]
+
+    case entries
+    when String
+      # PackedForward
+      option = data[2]
+      compressed = (option && option['compressed'] == 'gzip')
+      if compressed
+        raise(LogStash::Error, "PackedForward with compression is not supported")
+      end
+
+      entries_decoder = MessagePack::Unpacker.new
+      entries_decoder.feed_each(entries) do |entry|
+        epochtime = entry[0]
+        map = entry[1]
+        event = LogStash::Event.new(map.merge(
+                                      LogStash::Event::TIMESTAMP => LogStash::Timestamp.at(epochtime),
+                                      "tags" => [ tag ]
+                                    ))
+        yield event
+      end
+    when Array
+      # Forward
+      entries.each do |entry|
+        epochtime = entry[0]
+        map = entry[1]
+        event = LogStash::Event.new(map.merge(
+                                      LogStash::Event::TIMESTAMP => LogStash::Timestamp.at(epochtime),
+                                      "tags" => [ tag ]
+                                    ))
+        yield event
+      end
+    when Fixnum
+      # Message
+      epochtime = entries
+      map = data[2]
+      event = LogStash::Event.new(map.merge(
+                                    LogStash::Event::TIMESTAMP => LogStash::Timestamp.at(epochtime),
+                                    "tags" => [ tag ]
+                                  ))
+      yield event
+    else
+      raise(LogStash::Error, "Unknown event type")
+    end
+  rescue StandardError => e
+    @logger.error("Fluent parse error, original data now in message field", :error => e, :data => data)
+    yield LogStash::Event.new("message" => data, "tags" => [ "_fluentparsefailure" ])
+  end
+
 end # class LogStash::Codecs::Fluent

data/logstash-codec-fluent.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-fluent'
-  s.version         = '3.0.2'
+  s.version         = '3.1.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This codec handles fluentd's msgpack schema."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -29,6 +29,6 @@ Gem::Specification.new do |s|
     s.add_runtime_dependency 'msgpack'
   end
 
-  s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'logstash-devutils', ">= 1.0.0"
 end
 

data/spec/codecs/fluent_spec.rb

@@ -37,7 +37,60 @@ describe LogStash::Codecs::Fluent do
 
     it "should decode without errors" do
       subject.decode(message) do |event|
-       expect(event.get("name")).to eq("foo")
+        expect(event.get("name")).to eq("foo")
+      end
+    end
+
+  end
+
+  describe "event decoding (buckets of events)" do
+
+    let(:tag)       { "mytag" }
+    let(:epochtime) { event.timestamp.to_i }
+    let(:data)      { LogStash::Util.normalize(event.to_hash) }
+    let(:message) do
+      MessagePack.pack([tag,
+                        [
+                          [epochtime, data.merge(LogStash::Event::TIMESTAMP => event.timestamp.to_iso8601)],
+                          [epochtime, data.merge(LogStash::Event::TIMESTAMP => event.timestamp.to_iso8601)],
+                          [epochtime, data.merge(LogStash::Event::TIMESTAMP => event.timestamp.to_iso8601)]
+                        ]
+                       ])
+    end
+
+    it "should decode without errors" do
+      count = 0
+
+      subject.decode(message) do |event|
+        expect(event.get("name")).to eq("foo")
+        count += 1
+      end
+
+      expect(count).to eq(3)
+    end
+
+  end
+
+  describe "event decoding (broken package)" do
+
+    let(:tag)       { "mytag" }
+    let(:epochtime) { event.timestamp.to_s }
+    let(:data)      { LogStash::Util.normalize(event.to_hash) }
+    let(:message) do
+      MessagePack.pack([tag,
+                        epochtime, data.merge(LogStash::Event::TIMESTAMP => event.timestamp.to_iso8601)
+                       ])
+    end
+
+    it "should decode with errors" do
+      subject.decode(message) do |event|
+        expect(event.get("name")).not_to eq("foo")
+      end
+    end
+
+    it "should inject a failure event" do
+      subject.decode(message) do |event|
+        expect(event.get("tags")).to include("_fluentparsefailure")
       end
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-fluent
 version: !ruby/object:Gem::Version
-  version: 3.0.2
+  version: 3.1.1
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-07-14 00:00:00.000000000 Z
+date: 2017-03-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -49,7 +49,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.0
   name: logstash-devutils
   prerelease: false
   type: :development
@@ -57,7 +57,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.0
 description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
@@ -96,7 +96,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.3
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: This codec handles fluentd's msgpack schema.