logstash-codec-fluent 3.4.0-java → 3.4.2-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/lib/logstash/codecs/fluent.rb +19 -4
- data/logstash-codec-fluent.gemspec +1 -1
- data/spec/codecs/fluent_spec.rb +63 -1
- metadata +8 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 24e4641948b23a723d4822aacde3922e21b5b89f5158f1fcfe4293e7cf205de0
|
4
|
+
data.tar.gz: 1a7ce136c2bdda235b35d4f562744909406e9d1cc8ab610a83f07a5adb973c6f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 194d5afb6af9b32bfa02c81227222a0417ee2afcb01bf7f6551372fcc36dd968fb6a642b02f960a2f84eb010457812926fa06a2505738bc396c97021c8a24d7a
|
7
|
+
data.tar.gz: ca8b9aba97f93d56bb4aedc3d3e568f30b77ffc7ef03da7b368b59c531fd56c89def9dd45778e7ae2f331937fab4364e3b60bd79f777121275106b0a6a5303c4
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
## 3.4.2
|
2
|
+
- Fix: Convert LogStash::Timestamp values to iso-8601 to resolve crash issue with `msgpack` serialization [#30](https://github.com/logstash-plugins/logstash-codec-fluent/pull/30)
|
3
|
+
|
4
|
+
## 3.4.1
|
5
|
+
- Fix: handle multiple PackForward-encoded messages in a single payload [#28](https://github.com/logstash-plugins/logstash-codec-fluent/pull/28)
|
6
|
+
|
1
7
|
## 3.4.0
|
2
8
|
- Feat: added target configuration + event-factory support [#27](https://github.com/logstash-plugins/logstash-codec-fluent/pull/27)
|
3
9
|
- Fix: decoding of time's nano-second precision
|
@@ -75,10 +75,9 @@ class LogStash::Codecs::Fluent < LogStash::Codecs::Base
|
|
75
75
|
# use normalize to make sure returned Hash is pure Ruby for
|
76
76
|
# MessagePack#pack which relies on pure Ruby object recognition
|
77
77
|
data = LogStash::Util.normalize(event.to_hash)
|
78
|
-
|
79
|
-
# merge to avoid modifying data which could have side effects if multiple outputs
|
78
|
+
|
80
79
|
@packer.clear
|
81
|
-
@on_event.call(event, @packer.pack([tag, epochtime, data
|
80
|
+
@on_event.call(event, @packer.pack([tag, epochtime, normalize_timestamps(data)]))
|
82
81
|
end # def encode
|
83
82
|
|
84
83
|
def forwardable_tag(event)
|
@@ -117,7 +116,7 @@ class LogStash::Codecs::Fluent < LogStash::Codecs::Base
|
|
117
116
|
raise(LogStash::Error, "PackedForward with compression is not supported")
|
118
117
|
end
|
119
118
|
|
120
|
-
entries_decoder = @
|
119
|
+
entries_decoder = @factory.unpacker
|
121
120
|
entries_decoder.feed_each(entries) do |entry|
|
122
121
|
yield generate_event(entry[1], entry[0], tag)
|
123
122
|
end
|
@@ -145,4 +144,20 @@ class LogStash::Codecs::Fluent < LogStash::Codecs::Base
|
|
145
144
|
event
|
146
145
|
end
|
147
146
|
|
147
|
+
## Serializes timestamp as a iso8601 string, otherwise fluentd complains when packing the data
|
148
|
+
# @param object any type of data such as Hash, Array, etc...
|
149
|
+
# @return same shape of input with iso8061 serialized timestamps
|
150
|
+
def normalize_timestamps(object)
|
151
|
+
case object
|
152
|
+
when Hash
|
153
|
+
object.inject({}){|result, (key, value)| result[key] = normalize_timestamps(value); result}
|
154
|
+
when Array
|
155
|
+
object.map{|element| normalize_timestamps(element)}
|
156
|
+
when LogStash::Timestamp
|
157
|
+
object.to_iso8601
|
158
|
+
else
|
159
|
+
object
|
160
|
+
end
|
161
|
+
end
|
162
|
+
|
148
163
|
end # class LogStash::Codecs::Fluent
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-codec-fluent'
|
4
|
-
s.version = '3.4.
|
4
|
+
s.version = '3.4.2'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Reads the `fluentd` `msgpack` schema"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
data/spec/codecs/fluent_spec.rb
CHANGED
@@ -13,7 +13,7 @@ describe LogStash::Codecs::Fluent do
|
|
13
13
|
@unpacker = @factory.unpacker
|
14
14
|
end
|
15
15
|
|
16
|
-
subject { LogStash::Codecs::Fluent.new(config) }
|
16
|
+
subject(:fluent_codec) { LogStash::Codecs::Fluent.new(config) }
|
17
17
|
|
18
18
|
let(:config) { Hash.new }
|
19
19
|
|
@@ -47,6 +47,28 @@ describe LogStash::Codecs::Fluent do
|
|
47
47
|
|
48
48
|
end
|
49
49
|
|
50
|
+
describe "when encoding timestamps" do
|
51
|
+
let(:properties) { {:name => "foo", :custom_time => Time.now(), :time => { :custom_time => Time.now() }, :custom_times => [Time.now(), Time.now(), "2023-03-28"] } }
|
52
|
+
|
53
|
+
it "converts to iso8601 and encodes without issue" do
|
54
|
+
subject.on_event do |event, data|
|
55
|
+
@unpacker.feed_each(data) do |fields|
|
56
|
+
expect(fields[0]).to eq("log")
|
57
|
+
expect(fields[2]["name"]).to eq("foo")
|
58
|
+
expect(fields[2]["custom_time"]).not_to be_nil
|
59
|
+
expect(fields[2]["custom_time"].class).to eql(String) # iso8601 string
|
60
|
+
expect(fields[2]["time"]).not_to be_nil
|
61
|
+
expect(fields[2]["time"]["custom_time"]).not_to be_nil
|
62
|
+
expect(fields[2]["time"]["custom_time"].class).to eql(String)
|
63
|
+
expect(fields[2]["custom_times"]).not_to be_nil
|
64
|
+
expect(fields[2]["custom_times"][0].class).to eql(String)
|
65
|
+
expect(fields[2]["custom_times"][2]).to eql("2023-03-28")
|
66
|
+
end
|
67
|
+
end
|
68
|
+
subject.encode(event)
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
50
72
|
describe "event encoding with EventTime" do
|
51
73
|
|
52
74
|
let(:config) { super().merge "nanosecond_precision" => true }
|
@@ -202,6 +224,46 @@ describe LogStash::Codecs::Fluent do
|
|
202
224
|
|
203
225
|
end
|
204
226
|
|
227
|
+
describe "event decoding (multiple PackForward messages)" do
|
228
|
+
def pack_events(logstash_events)
|
229
|
+
packer = @factory.packer
|
230
|
+
logstash_events.map {|logstash_event| [logstash_event.timestamp.to_i, logstash_event.to_hash.merge(LogStash::Event::TIMESTAMP => logstash_event.timestamp.to_iso8601)] }
|
231
|
+
.each {|fluentd_event_tuple| packer.pack(fluentd_event_tuple) }
|
232
|
+
packer.to_s
|
233
|
+
end
|
234
|
+
|
235
|
+
def generate_event(idx)
|
236
|
+
LogStash::Event.new("idx" => idx)
|
237
|
+
end
|
238
|
+
|
239
|
+
def generate_events(idx_range)
|
240
|
+
idx_range.map {|idx| generate_event(idx) }
|
241
|
+
end
|
242
|
+
|
243
|
+
# our message needs to contain _multiple_ PackForward events, at least
|
244
|
+
# one of which contains multiple Events in its pack. This ensures we don't
|
245
|
+
# cross wires with our pack buffers.
|
246
|
+
let(:message) do
|
247
|
+
@factory.packer.pack([tag, pack_events(generate_events(000...100))])
|
248
|
+
.pack([tag, pack_events(generate_events(100...117))])
|
249
|
+
.pack([tag, pack_events([generate_event(117)])])
|
250
|
+
.pack([tag, pack_events([generate_event(118)])])
|
251
|
+
.pack([tag, pack_events(generate_events(119...199))])
|
252
|
+
.to_s
|
253
|
+
end
|
254
|
+
|
255
|
+
it 'decodes packed events without errors' do
|
256
|
+
seen = []
|
257
|
+
|
258
|
+
fluent_codec.decode(message) do |event|
|
259
|
+
seen << event.get("idx")
|
260
|
+
expect(event.get("tags")).to be_a_kind_of(Array).and(include(tag))
|
261
|
+
end
|
262
|
+
|
263
|
+
expect(seen).to match_array((000...199).to_a) # unordered
|
264
|
+
end
|
265
|
+
end
|
266
|
+
|
205
267
|
describe "event decoding (broken package)" do
|
206
268
|
|
207
269
|
let(:epochtime) { timestamp.to_s }
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-codec-fluent
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.4.
|
4
|
+
version: 3.4.2
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2023-04-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -20,8 +20,8 @@ dependencies:
|
|
20
20
|
- !ruby/object:Gem::Version
|
21
21
|
version: '2.99'
|
22
22
|
name: logstash-core-plugin-api
|
23
|
-
type: :runtime
|
24
23
|
prerelease: false
|
24
|
+
type: :runtime
|
25
25
|
version_requirements: !ruby/object:Gem::Requirement
|
26
26
|
requirements:
|
27
27
|
- - ">="
|
@@ -37,8 +37,8 @@ dependencies:
|
|
37
37
|
- !ruby/object:Gem::Version
|
38
38
|
version: '1.0'
|
39
39
|
name: logstash-mixin-event_support
|
40
|
-
type: :runtime
|
41
40
|
prerelease: false
|
41
|
+
type: :runtime
|
42
42
|
version_requirements: !ruby/object:Gem::Requirement
|
43
43
|
requirements:
|
44
44
|
- - "~>"
|
@@ -51,8 +51,8 @@ dependencies:
|
|
51
51
|
- !ruby/object:Gem::Version
|
52
52
|
version: '1.0'
|
53
53
|
name: logstash-mixin-validator_support
|
54
|
-
type: :runtime
|
55
54
|
prerelease: false
|
55
|
+
type: :runtime
|
56
56
|
version_requirements: !ruby/object:Gem::Requirement
|
57
57
|
requirements:
|
58
58
|
- - "~>"
|
@@ -65,8 +65,8 @@ dependencies:
|
|
65
65
|
- !ruby/object:Gem::Version
|
66
66
|
version: '1.1'
|
67
67
|
name: msgpack
|
68
|
-
type: :runtime
|
69
68
|
prerelease: false
|
69
|
+
type: :runtime
|
70
70
|
version_requirements: !ruby/object:Gem::Requirement
|
71
71
|
requirements:
|
72
72
|
- - "~>"
|
@@ -79,8 +79,8 @@ dependencies:
|
|
79
79
|
- !ruby/object:Gem::Version
|
80
80
|
version: 1.0.0
|
81
81
|
name: logstash-devutils
|
82
|
-
type: :development
|
83
82
|
prerelease: false
|
83
|
+
type: :development
|
84
84
|
version_requirements: !ruby/object:Gem::Requirement
|
85
85
|
requirements:
|
86
86
|
- - ">="
|
@@ -127,7 +127,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
127
127
|
- !ruby/object:Gem::Version
|
128
128
|
version: '0'
|
129
129
|
requirements: []
|
130
|
-
rubygems_version: 3.
|
130
|
+
rubygems_version: 3.1.6
|
131
131
|
signing_key:
|
132
132
|
specification_version: 4
|
133
133
|
summary: Reads the `fluentd` `msgpack` schema
|