logstash-codec-fluent 3.2.0-java → 3.3.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ee03e06bd2f0fc38268db7f16e6de3405328331f63e08007feb24a91bd4cef34
-  data.tar.gz: 2413c637ab79150cc9dc6731f6cefd36969909487ac217d145d7f1298ba28250
+  metadata.gz: 03c1ce6a081dedf60755a7a37842e814dd3a1ffc3af4d433a00cdaba91f9bc69
+  data.tar.gz: bb1c16c6a60754f41fb7285a5b9fe422dbf4c1d85bfd2e5040793cffd810ec5c
 SHA512:
-  metadata.gz: 9932e23fdfe7efd1896fa45be2b86c630a8150ad2d823f7b6ab980ce458045db9e019580dcf33b14565d3411efea0b5b1c88d0c5e8a7fbffc7bb622f8a85b70b
-  data.tar.gz: 3ee22d094f5be07cb97310a94d9de06127377d757770d0899ae0ff11ad0f45969ac4fde29b1866e964691a24bd228c61a62b945114b0ae59819ee1ae9b17f47d
+  metadata.gz: a2bf6928d517ebeca49942829d246965f14d1efaf08628e5b1767e5059f33df17e64f56338db35a5033b163fd87820388fd7d958a98179f80f3676af67baad29
+  data.tar.gz: 820ae450a849d4aaa24eb5cd0814ed066bfc1f7abb98c24428f676c87beb10e745c7d5c9cb96d8549bb7c9a8749e02955e4f00bf88340127cd0680df90967401

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.3.0
+  - Handle EventTime msgpack extension to handle nanosecond precision time and add its parameter [#18](https://github.com/logstash-plugins/logstash-codec-fluent/pull/18)
+
 ## 3.2.0
   - Encode tags as fluent forward protocol tags. Ref: https://github.com/logstash-plugins/logstash-codec-fluent/pull/21
 

data/lib/logstash/codecs/fluent.rb

@@ -10,7 +10,9 @@ require "logstash/util"
 # [source,ruby]
 #     input {
 #       tcp {
-#         codec => fluent
+#         codec => fluent {
+#           nanosecond_precision => true
+#         }
 #         port => 4000
 #       }
 #     }
@@ -22,15 +24,23 @@ require "logstash/util"
 #
 # Notes:
 #
-# * the fluent uses a second-precision time for events, so you will never see
-#   subsecond precision on events processed by this codec.
+# * to handle EventTime msgpack extension, you must specify nanosecond_precision parameter as true.
 #
 class LogStash::Codecs::Fluent < LogStash::Codecs::Base
+  require "logstash/codecs/fluent/event_time"
+
   config_name "fluent"
 
+  config :nanosecond_precision, :validate => :boolean, :default => false
+
   def register
     require "msgpack"
-    @decoder = MessagePack::Unpacker.new
+    @factory = MessagePack::Factory.new
+    if @nanosecond_precision
+      @factory.register_type(EventTime::TYPE, EventTime)
+    end
+    @packer = @factory.packer
+    @decoder = @factory.unpacker
   end
 
   def decode(data, &block)
@@ -43,14 +53,19 @@ class LogStash::Codecs::Fluent < LogStash::Codecs::Base
     # Ensure tag to "tag1.tag2.tag3" style string.
     # Fluentd cannot handle Array class value in forward protocol's tag.
     tag = forwardable_tag(event)
-    epochtime = event.timestamp.to_i
+    epochtime = if @nanosecond_precision
+                  EventTime.new(event.timestamp.to_i, event.timestamp.usec * 1000)
+                else
+                  event.timestamp.to_i
+                end
 
     # use normalize to make sure returned Hash is pure Ruby for
     # MessagePack#pack which relies on pure Ruby object recognition
     data = LogStash::Util.normalize(event.to_hash)
     # timestamp is serialized as a iso8601 string
     # merge to avoid modifying data which could have side effects if multiple outputs
-    @on_event.call(event, MessagePack.pack([tag, epochtime, data.merge(LogStash::Event::TIMESTAMP => event.timestamp.to_iso8601)]))
+    @packer.clear
+    @on_event.call(event, @packer.pack([tag, epochtime, data.merge(LogStash::Event::TIMESTAMP => event.timestamp.to_iso8601)]))
   end # def encode
 
   def forwardable_tag(event)
@@ -67,6 +82,15 @@ class LogStash::Codecs::Fluent < LogStash::Codecs::Base
 
   private
 
+  def decode_fluent_time(fluent_time)
+    case fluent_time
+    when Fixnum
+      fluent_time
+    when EventTime
+      Time.at(fluent_time.sec, fluent_time.nsec)
+    end
+  end
+
   def decode_event(data, &block)
     tag = data[0]
     entries = data[1]
@@ -80,9 +104,9 @@ class LogStash::Codecs::Fluent < LogStash::Codecs::Base
         raise(LogStash::Error, "PackedForward with compression is not supported")
       end
 
-      entries_decoder = MessagePack::Unpacker.new
+      entries_decoder = @decoder
       entries_decoder.feed_each(entries) do |entry|
-        epochtime = entry[0]
+        epochtime = decode_fluent_time(entry[0])
         map = entry[1]
         event = LogStash::Event.new(map.merge(
                                       LogStash::Event::TIMESTAMP => LogStash::Timestamp.at(epochtime),
@@ -93,7 +117,7 @@ class LogStash::Codecs::Fluent < LogStash::Codecs::Base
     when Array
       # Forward
       entries.each do |entry|
-        epochtime = entry[0]
+        epochtime = decode_fluent_time(entry[0])
         map = entry[1]
         event = LogStash::Event.new(map.merge(
                                       LogStash::Event::TIMESTAMP => LogStash::Timestamp.at(epochtime),
@@ -101,9 +125,9 @@ class LogStash::Codecs::Fluent < LogStash::Codecs::Base
                                     ))
         yield event
       end
-    when Fixnum
+    when Fixnum, EventTime
       # Message
-      epochtime = entries
+      epochtime = decode_fluent_time(entries)
       map = data[2]
       event = LogStash::Event.new(map.merge(
                                     LogStash::Event::TIMESTAMP => LogStash::Timestamp.at(epochtime),

data/lib/logstash/codecs/fluent/event_time.rb

@@ -0,0 +1,28 @@
+module LogStash; module Codecs; class Fluent;
+  class EventTime
+    attr_reader :sec, :nsec
+
+    TYPE = 0
+
+    def initialize(sec, nsec = 0)
+      @sec = sec
+      @nsec = nsec
+    end
+
+    def to_msgpack(io = nil)
+      @sec.to_msgpack(io)
+    end
+
+    def to_msgpack_ext
+      [@sec, @nsec].pack('NN')
+    end
+
+    def self.from_msgpack_ext(data)
+      new(*data.unpack('NN'))
+    end
+
+    def to_json(*args)
+      @sec
+    end
+  end
+end; end; end

data/logstash-codec-fluent.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-fluent'
-  s.version         = '3.2.0'
+  s.version         = '3.3.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads the `fluentd` `msgpack` schema"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/codecs/fluent_spec.rb

@@ -2,8 +2,16 @@
 require_relative "../spec_helper"
 require "logstash/plugin"
 require "logstash/event"
+require "msgpack"
 
 describe LogStash::Codecs::Fluent do
+  before do
+    @factory = MessagePack::Factory.new
+    @factory.register_type(LogStash::Codecs::Fluent::EventTime::TYPE,
+                           LogStash::Codecs::Fluent::EventTime)
+    @packer = @factory.packer
+    @unpacker = @factory.unpacker
+  end
 
   let(:properties) { {:name => "foo" } }
   let(:event)      { LogStash::Event.new(properties) }
@@ -17,9 +25,25 @@ describe LogStash::Codecs::Fluent do
 
     it "should encode as message pack format" do
       subject.on_event do |event, data|
-        fields = MessagePack.unpack(data)
-        expect(fields[0]).to eq("log")
-        expect(fields[2]["name"]).to eq("foo")
+        @unpacker.feed_each(data) do |fields|
+          expect(fields[0]).to eq("log")
+          expect(fields[2]["name"]).to eq("foo")
+        end
+      end
+      subject.encode(event)
+    end
+
+  end
+
+  describe "event encoding with EventTime" do
+    subject { LogStash::Plugin.lookup("codec", "fluent").new({"nanosecond_precision" => true}) }
+
+    it "should encode as message pack format" do
+      subject.on_event do |event, data|
+        @unpacker.feed_each(data) do |fields|
+          expect(fields[0]).to eq("log")
+          expect(fields[2]["name"]).to eq("foo")
+        end
       end
       subject.encode(event)
     end
@@ -32,8 +56,27 @@ describe LogStash::Codecs::Fluent do
     let(:epochtime) { event.timestamp.to_i }
     let(:data)      { LogStash::Util.normalize(event.to_hash) }
     let(:message) do
-      MessagePack.pack([tag, epochtime, data.merge(LogStash::Event::TIMESTAMP => event.timestamp.to_iso8601)])
+      @packer.pack([tag, epochtime, data.merge(LogStash::Event::TIMESTAMP => event.timestamp.to_iso8601)])
+    end
+
+    it "should decode without errors" do
+      subject.decode(message) do |event|
+        expect(event.get("name")).to eq("foo")
+      end
+    end
+
+  end
+
+  describe "event decoding with EventTime" do
+
+    let(:tag)       { "mytag" }
+    let(:epochtime) { LogStash::Codecs::Fluent::EventTime.new(event.timestamp.to_i,
+                                                              event.timestamp.usec * 1000)  }
+    let(:data)      { LogStash::Util.normalize(event.to_hash) }
+    let(:message) do
+      @packer.pack([tag, epochtime, data.merge(LogStash::Event::TIMESTAMP => event.timestamp.to_iso8601)])
     end
+    subject { LogStash::Plugin.lookup("codec", "fluent").new({"nanosecond_precision" => true}) }
 
     it "should decode without errors" do
       subject.decode(message) do |event|
@@ -87,13 +130,13 @@ describe LogStash::Codecs::Fluent do
     let(:epochtime) { event.timestamp.to_i }
     let(:data)      { LogStash::Util.normalize(event.to_hash) }
     let(:message) do
-      MessagePack.pack([tag,
-                        [
-                          [epochtime, data.merge(LogStash::Event::TIMESTAMP => event.timestamp.to_iso8601)],
-                          [epochtime, data.merge(LogStash::Event::TIMESTAMP => event.timestamp.to_iso8601)],
-                          [epochtime, data.merge(LogStash::Event::TIMESTAMP => event.timestamp.to_iso8601)]
-                        ]
-                       ])
+      @packer.pack([tag,
+                    [
+                      [epochtime, data.merge(LogStash::Event::TIMESTAMP => event.timestamp.to_iso8601)],
+                      [epochtime, data.merge(LogStash::Event::TIMESTAMP => event.timestamp.to_iso8601)],
+                      [epochtime, data.merge(LogStash::Event::TIMESTAMP => event.timestamp.to_iso8601)]
+                    ]
+                   ])
     end
 
     it "should decode without errors" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-fluent
 version: !ruby/object:Gem::Version
-  version: 3.2.0
+  version: 3.3.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-02-06 00:00:00.000000000 Z
+date: 2019-11-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -74,6 +74,7 @@ files:
 - README.md
 - docs/index.asciidoc
 - lib/logstash/codecs/fluent.rb
+- lib/logstash/codecs/fluent/event_time.rb
 - logstash-codec-fluent.gemspec
 - spec/codecs/fluent_spec.rb
 - spec/spec_helper.rb