logstash-codec-fluent 3.4.0-java → 3.4.2-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: e17d4af78cd9df1b4534f747813c106d9c1c4e3072a2f74989bfd78413e3173c
4
- data.tar.gz: 4c77861536c3a5ea3c4e8eef9d97bdc0c986a42991261b9143a80bfa7ede0254
3
+ metadata.gz: 24e4641948b23a723d4822aacde3922e21b5b89f5158f1fcfe4293e7cf205de0
4
+ data.tar.gz: 1a7ce136c2bdda235b35d4f562744909406e9d1cc8ab610a83f07a5adb973c6f
5
5
  SHA512:
6
- metadata.gz: 8e876b1ee27ca9f647baff747432261a10a41d1217bc228f458d7c45641a93d70a6765eb312a1e141e05f6413a8866f4dee731bf215d7c5f8466583e7e2f7888
7
- data.tar.gz: ffcfe5a551b868a02eb53ebefece63e97bd57c70f31a1b925d4a8e80093c92186fbfe1eeee061d1def481930f244b97f3061cc750a8880fb807f804c164982e1
6
+ metadata.gz: 194d5afb6af9b32bfa02c81227222a0417ee2afcb01bf7f6551372fcc36dd968fb6a642b02f960a2f84eb010457812926fa06a2505738bc396c97021c8a24d7a
7
+ data.tar.gz: ca8b9aba97f93d56bb4aedc3d3e568f30b77ffc7ef03da7b368b59c531fd56c89def9dd45778e7ae2f331937fab4364e3b60bd79f777121275106b0a6a5303c4
data/CHANGELOG.md CHANGED
@@ -1,3 +1,9 @@
1
+ ## 3.4.2
2
+ - Fix: Convert LogStash::Timestamp values to iso-8601 to resolve crash issue with `msgpack` serialization [#30](https://github.com/logstash-plugins/logstash-codec-fluent/pull/30)
3
+
4
+ ## 3.4.1
5
+ - Fix: handle multiple PackForward-encoded messages in a single payload [#28](https://github.com/logstash-plugins/logstash-codec-fluent/pull/28)
6
+
1
7
  ## 3.4.0
2
8
  - Feat: added target configuration + event-factory support [#27](https://github.com/logstash-plugins/logstash-codec-fluent/pull/27)
3
9
  - Fix: decoding of time's nano-second precision
@@ -75,10 +75,9 @@ class LogStash::Codecs::Fluent < LogStash::Codecs::Base
75
75
  # use normalize to make sure returned Hash is pure Ruby for
76
76
  # MessagePack#pack which relies on pure Ruby object recognition
77
77
  data = LogStash::Util.normalize(event.to_hash)
78
- # timestamp is serialized as a iso8601 string
79
- # merge to avoid modifying data which could have side effects if multiple outputs
78
+
80
79
  @packer.clear
81
- @on_event.call(event, @packer.pack([tag, epochtime, data.merge(LogStash::Event::TIMESTAMP => event.timestamp.to_iso8601)]))
80
+ @on_event.call(event, @packer.pack([tag, epochtime, normalize_timestamps(data)]))
82
81
  end # def encode
83
82
 
84
83
  def forwardable_tag(event)
@@ -117,7 +116,7 @@ class LogStash::Codecs::Fluent < LogStash::Codecs::Base
117
116
  raise(LogStash::Error, "PackedForward with compression is not supported")
118
117
  end
119
118
 
120
- entries_decoder = @decoder
119
+ entries_decoder = @factory.unpacker
121
120
  entries_decoder.feed_each(entries) do |entry|
122
121
  yield generate_event(entry[1], entry[0], tag)
123
122
  end
@@ -145,4 +144,20 @@ class LogStash::Codecs::Fluent < LogStash::Codecs::Base
145
144
  event
146
145
  end
147
146
 
147
+ ## Serializes timestamp as a iso8601 string, otherwise fluentd complains when packing the data
148
+ # @param object any type of data such as Hash, Array, etc...
149
+ # @return same shape of input with iso8061 serialized timestamps
150
+ def normalize_timestamps(object)
151
+ case object
152
+ when Hash
153
+ object.inject({}){|result, (key, value)| result[key] = normalize_timestamps(value); result}
154
+ when Array
155
+ object.map{|element| normalize_timestamps(element)}
156
+ when LogStash::Timestamp
157
+ object.to_iso8601
158
+ else
159
+ object
160
+ end
161
+ end
162
+
148
163
  end # class LogStash::Codecs::Fluent
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-codec-fluent'
4
- s.version = '3.4.0'
4
+ s.version = '3.4.2'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "Reads the `fluentd` `msgpack` schema"
7
7
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -13,7 +13,7 @@ describe LogStash::Codecs::Fluent do
13
13
  @unpacker = @factory.unpacker
14
14
  end
15
15
 
16
- subject { LogStash::Codecs::Fluent.new(config) }
16
+ subject(:fluent_codec) { LogStash::Codecs::Fluent.new(config) }
17
17
 
18
18
  let(:config) { Hash.new }
19
19
 
@@ -47,6 +47,28 @@ describe LogStash::Codecs::Fluent do
47
47
 
48
48
  end
49
49
 
50
+ describe "when encoding timestamps" do
51
+ let(:properties) { {:name => "foo", :custom_time => Time.now(), :time => { :custom_time => Time.now() }, :custom_times => [Time.now(), Time.now(), "2023-03-28"] } }
52
+
53
+ it "converts to iso8601 and encodes without issue" do
54
+ subject.on_event do |event, data|
55
+ @unpacker.feed_each(data) do |fields|
56
+ expect(fields[0]).to eq("log")
57
+ expect(fields[2]["name"]).to eq("foo")
58
+ expect(fields[2]["custom_time"]).not_to be_nil
59
+ expect(fields[2]["custom_time"].class).to eql(String) # iso8601 string
60
+ expect(fields[2]["time"]).not_to be_nil
61
+ expect(fields[2]["time"]["custom_time"]).not_to be_nil
62
+ expect(fields[2]["time"]["custom_time"].class).to eql(String)
63
+ expect(fields[2]["custom_times"]).not_to be_nil
64
+ expect(fields[2]["custom_times"][0].class).to eql(String)
65
+ expect(fields[2]["custom_times"][2]).to eql("2023-03-28")
66
+ end
67
+ end
68
+ subject.encode(event)
69
+ end
70
+ end
71
+
50
72
  describe "event encoding with EventTime" do
51
73
 
52
74
  let(:config) { super().merge "nanosecond_precision" => true }
@@ -202,6 +224,46 @@ describe LogStash::Codecs::Fluent do
202
224
 
203
225
  end
204
226
 
227
+ describe "event decoding (multiple PackForward messages)" do
228
+ def pack_events(logstash_events)
229
+ packer = @factory.packer
230
+ logstash_events.map {|logstash_event| [logstash_event.timestamp.to_i, logstash_event.to_hash.merge(LogStash::Event::TIMESTAMP => logstash_event.timestamp.to_iso8601)] }
231
+ .each {|fluentd_event_tuple| packer.pack(fluentd_event_tuple) }
232
+ packer.to_s
233
+ end
234
+
235
+ def generate_event(idx)
236
+ LogStash::Event.new("idx" => idx)
237
+ end
238
+
239
+ def generate_events(idx_range)
240
+ idx_range.map {|idx| generate_event(idx) }
241
+ end
242
+
243
+ # our message needs to contain _multiple_ PackForward events, at least
244
+ # one of which contains multiple Events in its pack. This ensures we don't
245
+ # cross wires with our pack buffers.
246
+ let(:message) do
247
+ @factory.packer.pack([tag, pack_events(generate_events(000...100))])
248
+ .pack([tag, pack_events(generate_events(100...117))])
249
+ .pack([tag, pack_events([generate_event(117)])])
250
+ .pack([tag, pack_events([generate_event(118)])])
251
+ .pack([tag, pack_events(generate_events(119...199))])
252
+ .to_s
253
+ end
254
+
255
+ it 'decodes packed events without errors' do
256
+ seen = []
257
+
258
+ fluent_codec.decode(message) do |event|
259
+ seen << event.get("idx")
260
+ expect(event.get("tags")).to be_a_kind_of(Array).and(include(tag))
261
+ end
262
+
263
+ expect(seen).to match_array((000...199).to_a) # unordered
264
+ end
265
+ end
266
+
205
267
  describe "event decoding (broken package)" do
206
268
 
207
269
  let(:epochtime) { timestamp.to_s }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-codec-fluent
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.4.0
4
+ version: 3.4.2
5
5
  platform: java
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-08-09 00:00:00.000000000 Z
11
+ date: 2023-04-04 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -20,8 +20,8 @@ dependencies:
20
20
  - !ruby/object:Gem::Version
21
21
  version: '2.99'
22
22
  name: logstash-core-plugin-api
23
- type: :runtime
24
23
  prerelease: false
24
+ type: :runtime
25
25
  version_requirements: !ruby/object:Gem::Requirement
26
26
  requirements:
27
27
  - - ">="
@@ -37,8 +37,8 @@ dependencies:
37
37
  - !ruby/object:Gem::Version
38
38
  version: '1.0'
39
39
  name: logstash-mixin-event_support
40
- type: :runtime
41
40
  prerelease: false
41
+ type: :runtime
42
42
  version_requirements: !ruby/object:Gem::Requirement
43
43
  requirements:
44
44
  - - "~>"
@@ -51,8 +51,8 @@ dependencies:
51
51
  - !ruby/object:Gem::Version
52
52
  version: '1.0'
53
53
  name: logstash-mixin-validator_support
54
- type: :runtime
55
54
  prerelease: false
55
+ type: :runtime
56
56
  version_requirements: !ruby/object:Gem::Requirement
57
57
  requirements:
58
58
  - - "~>"
@@ -65,8 +65,8 @@ dependencies:
65
65
  - !ruby/object:Gem::Version
66
66
  version: '1.1'
67
67
  name: msgpack
68
- type: :runtime
69
68
  prerelease: false
69
+ type: :runtime
70
70
  version_requirements: !ruby/object:Gem::Requirement
71
71
  requirements:
72
72
  - - "~>"
@@ -79,8 +79,8 @@ dependencies:
79
79
  - !ruby/object:Gem::Version
80
80
  version: 1.0.0
81
81
  name: logstash-devutils
82
- type: :development
83
82
  prerelease: false
83
+ type: :development
84
84
  version_requirements: !ruby/object:Gem::Requirement
85
85
  requirements:
86
86
  - - ">="
@@ -127,7 +127,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
127
127
  - !ruby/object:Gem::Version
128
128
  version: '0'
129
129
  requirements: []
130
- rubygems_version: 3.0.6
130
+ rubygems_version: 3.1.6
131
131
  signing_key:
132
132
  specification_version: 4
133
133
  summary: Reads the `fluentd` `msgpack` schema