logstash-codec-fluent 3.4.0-java → 3.4.2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e17d4af78cd9df1b4534f747813c106d9c1c4e3072a2f74989bfd78413e3173c
-  data.tar.gz: 4c77861536c3a5ea3c4e8eef9d97bdc0c986a42991261b9143a80bfa7ede0254
+  metadata.gz: 24e4641948b23a723d4822aacde3922e21b5b89f5158f1fcfe4293e7cf205de0
+  data.tar.gz: 1a7ce136c2bdda235b35d4f562744909406e9d1cc8ab610a83f07a5adb973c6f
 SHA512:
-  metadata.gz: 8e876b1ee27ca9f647baff747432261a10a41d1217bc228f458d7c45641a93d70a6765eb312a1e141e05f6413a8866f4dee731bf215d7c5f8466583e7e2f7888
-  data.tar.gz: ffcfe5a551b868a02eb53ebefece63e97bd57c70f31a1b925d4a8e80093c92186fbfe1eeee061d1def481930f244b97f3061cc750a8880fb807f804c164982e1
+  metadata.gz: 194d5afb6af9b32bfa02c81227222a0417ee2afcb01bf7f6551372fcc36dd968fb6a642b02f960a2f84eb010457812926fa06a2505738bc396c97021c8a24d7a
+  data.tar.gz: ca8b9aba97f93d56bb4aedc3d3e568f30b77ffc7ef03da7b368b59c531fd56c89def9dd45778e7ae2f331937fab4364e3b60bd79f777121275106b0a6a5303c4

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 3.4.2
+  - Fix: Convert LogStash::Timestamp values to iso-8601 to resolve crash issue with `msgpack` serialization [#30](https://github.com/logstash-plugins/logstash-codec-fluent/pull/30)
+
+## 3.4.1
+  - Fix: handle multiple PackForward-encoded messages in a single payload [#28](https://github.com/logstash-plugins/logstash-codec-fluent/pull/28)
+
 ## 3.4.0
   - Feat: added target configuration + event-factory support [#27](https://github.com/logstash-plugins/logstash-codec-fluent/pull/27)
   - Fix: decoding of time's nano-second precision 

data/lib/logstash/codecs/fluent.rb

@@ -75,10 +75,9 @@ class LogStash::Codecs::Fluent < LogStash::Codecs::Base
     # use normalize to make sure returned Hash is pure Ruby for
     # MessagePack#pack which relies on pure Ruby object recognition
     data = LogStash::Util.normalize(event.to_hash)
-    # timestamp is serialized as a iso8601 string
-    # merge to avoid modifying data which could have side effects if multiple outputs
+
     @packer.clear
-    @on_event.call(event, @packer.pack([tag, epochtime, data.merge(LogStash::Event::TIMESTAMP => event.timestamp.to_iso8601)]))
+    @on_event.call(event, @packer.pack([tag, epochtime, normalize_timestamps(data)]))
   end # def encode
 
   def forwardable_tag(event)
@@ -117,7 +116,7 @@ class LogStash::Codecs::Fluent < LogStash::Codecs::Base
         raise(LogStash::Error, "PackedForward with compression is not supported")
       end
 
-      entries_decoder = @decoder
+      entries_decoder = @factory.unpacker
       entries_decoder.feed_each(entries) do |entry|
         yield generate_event(entry[1], entry[0], tag)
       end
@@ -145,4 +144,20 @@ class LogStash::Codecs::Fluent < LogStash::Codecs::Base
     event
   end
 
+  ## Serializes timestamp as a iso8601 string, otherwise fluentd complains when packing the data
+  # @param object any type of data such as Hash, Array, etc...
+  # @return same shape of input with iso8061 serialized timestamps
+  def normalize_timestamps(object)
+    case object
+    when Hash
+      object.inject({}){|result, (key, value)| result[key] = normalize_timestamps(value); result}
+    when Array
+      object.map{|element| normalize_timestamps(element)}
+    when LogStash::Timestamp
+      object.to_iso8601
+    else
+      object
+    end
+  end
+
 end # class LogStash::Codecs::Fluent

data/logstash-codec-fluent.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-fluent'
-  s.version         = '3.4.0'
+  s.version         = '3.4.2'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads the `fluentd` `msgpack` schema"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/codecs/fluent_spec.rb

@@ -13,7 +13,7 @@ describe LogStash::Codecs::Fluent do
     @unpacker = @factory.unpacker
   end
 
-  subject { LogStash::Codecs::Fluent.new(config) }
+  subject(:fluent_codec) { LogStash::Codecs::Fluent.new(config) }
 
   let(:config) { Hash.new }
 
@@ -47,6 +47,28 @@ describe LogStash::Codecs::Fluent do
 
   end
 
+  describe "when encoding timestamps" do
+    let(:properties) { {:name => "foo", :custom_time => Time.now(), :time => { :custom_time => Time.now() }, :custom_times => [Time.now(), Time.now(), "2023-03-28"] } }
+
+    it "converts to iso8601 and encodes without issue" do
+      subject.on_event do |event, data|
+        @unpacker.feed_each(data) do |fields|
+          expect(fields[0]).to eq("log")
+          expect(fields[2]["name"]).to eq("foo")
+          expect(fields[2]["custom_time"]).not_to be_nil
+          expect(fields[2]["custom_time"].class).to eql(String) # iso8601 string
+          expect(fields[2]["time"]).not_to be_nil
+          expect(fields[2]["time"]["custom_time"]).not_to be_nil
+          expect(fields[2]["time"]["custom_time"].class).to eql(String)
+          expect(fields[2]["custom_times"]).not_to be_nil
+          expect(fields[2]["custom_times"][0].class).to eql(String)
+          expect(fields[2]["custom_times"][2]).to eql("2023-03-28")
+        end
+      end
+      subject.encode(event)
+    end
+  end
+
   describe "event encoding with EventTime" do
 
     let(:config) { super().merge "nanosecond_precision" => true }
@@ -202,6 +224,46 @@ describe LogStash::Codecs::Fluent do
 
   end
 
+  describe "event decoding (multiple PackForward messages)" do
+    def pack_events(logstash_events)
+      packer = @factory.packer
+      logstash_events.map {|logstash_event| [logstash_event.timestamp.to_i, logstash_event.to_hash.merge(LogStash::Event::TIMESTAMP => logstash_event.timestamp.to_iso8601)] }
+                     .each {|fluentd_event_tuple| packer.pack(fluentd_event_tuple) }
+      packer.to_s
+    end
+
+    def generate_event(idx)
+      LogStash::Event.new("idx" => idx)
+    end
+
+    def generate_events(idx_range)
+      idx_range.map {|idx| generate_event(idx) }
+    end
+
+    # our message needs to contain _multiple_ PackForward events, at least
+    # one of which contains multiple Events in its pack. This ensures we don't
+    # cross wires with our pack buffers.
+    let(:message) do
+      @factory.packer.pack([tag, pack_events(generate_events(000...100))])
+                     .pack([tag, pack_events(generate_events(100...117))])
+                     .pack([tag, pack_events([generate_event(117)])])
+                     .pack([tag, pack_events([generate_event(118)])])
+                     .pack([tag, pack_events(generate_events(119...199))])
+                     .to_s
+    end
+
+    it 'decodes packed events without errors' do
+      seen = []
+
+      fluent_codec.decode(message) do |event|
+        seen << event.get("idx")
+        expect(event.get("tags")).to be_a_kind_of(Array).and(include(tag))
+      end
+
+      expect(seen).to match_array((000...199).to_a) # unordered
+    end
+  end
+
   describe "event decoding (broken package)" do
 
     let(:epochtime) { timestamp.to_s }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-fluent
 version: !ruby/object:Gem::Version
-  version: 3.4.0
+  version: 3.4.2
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-09 00:00:00.000000000 Z
+date: 2023-04-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -20,8 +20,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.99'
   name: logstash-core-plugin-api
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -37,8 +37,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
   name: logstash-mixin-event_support
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -51,8 +51,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
   name: logstash-mixin-validator_support
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -65,8 +65,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.1'
   name: msgpack
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -79,8 +79,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 1.0.0
   name: logstash-devutils
-  type: :development
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -127,7 +127,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Reads the `fluentd` `msgpack` schema