logstash-codec-fluent 3.2.0-java → 3.3.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/lib/logstash/codecs/fluent.rb +35 -11
- data/lib/logstash/codecs/fluent/event_time.rb +28 -0
- data/logstash-codec-fluent.gemspec +1 -1
- data/spec/codecs/fluent_spec.rb +54 -11
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 03c1ce6a081dedf60755a7a37842e814dd3a1ffc3af4d433a00cdaba91f9bc69
|
4
|
+
data.tar.gz: bb1c16c6a60754f41fb7285a5b9fe422dbf4c1d85bfd2e5040793cffd810ec5c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a2bf6928d517ebeca49942829d246965f14d1efaf08628e5b1767e5059f33df17e64f56338db35a5033b163fd87820388fd7d958a98179f80f3676af67baad29
|
7
|
+
data.tar.gz: 820ae450a849d4aaa24eb5cd0814ed066bfc1f7abb98c24428f676c87beb10e745c7d5c9cb96d8549bb7c9a8749e02955e4f00bf88340127cd0680df90967401
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
## 3.3.0
|
2
|
+
- Handle EventTime msgpack extension to handle nanosecond precision time and add its parameter [#18](https://github.com/logstash-plugins/logstash-codec-fluent/pull/18)
|
3
|
+
|
1
4
|
## 3.2.0
|
2
5
|
- Encode tags as fluent forward protocol tags. Ref: https://github.com/logstash-plugins/logstash-codec-fluent/pull/21
|
3
6
|
|
@@ -10,7 +10,9 @@ require "logstash/util"
|
|
10
10
|
# [source,ruby]
|
11
11
|
# input {
|
12
12
|
# tcp {
|
13
|
-
# codec => fluent
|
13
|
+
# codec => fluent {
|
14
|
+
# nanosecond_precision => true
|
15
|
+
# }
|
14
16
|
# port => 4000
|
15
17
|
# }
|
16
18
|
# }
|
@@ -22,15 +24,23 @@ require "logstash/util"
|
|
22
24
|
#
|
23
25
|
# Notes:
|
24
26
|
#
|
25
|
-
# *
|
26
|
-
# subsecond precision on events processed by this codec.
|
27
|
+
# * to handle EventTime msgpack extension, you must specify nanosecond_precision parameter as true.
|
27
28
|
#
|
28
29
|
class LogStash::Codecs::Fluent < LogStash::Codecs::Base
|
30
|
+
require "logstash/codecs/fluent/event_time"
|
31
|
+
|
29
32
|
config_name "fluent"
|
30
33
|
|
34
|
+
config :nanosecond_precision, :validate => :boolean, :default => false
|
35
|
+
|
31
36
|
def register
|
32
37
|
require "msgpack"
|
33
|
-
@
|
38
|
+
@factory = MessagePack::Factory.new
|
39
|
+
if @nanosecond_precision
|
40
|
+
@factory.register_type(EventTime::TYPE, EventTime)
|
41
|
+
end
|
42
|
+
@packer = @factory.packer
|
43
|
+
@decoder = @factory.unpacker
|
34
44
|
end
|
35
45
|
|
36
46
|
def decode(data, &block)
|
@@ -43,14 +53,19 @@ class LogStash::Codecs::Fluent < LogStash::Codecs::Base
|
|
43
53
|
# Ensure tag to "tag1.tag2.tag3" style string.
|
44
54
|
# Fluentd cannot handle Array class value in forward protocol's tag.
|
45
55
|
tag = forwardable_tag(event)
|
46
|
-
epochtime =
|
56
|
+
epochtime = if @nanosecond_precision
|
57
|
+
EventTime.new(event.timestamp.to_i, event.timestamp.usec * 1000)
|
58
|
+
else
|
59
|
+
event.timestamp.to_i
|
60
|
+
end
|
47
61
|
|
48
62
|
# use normalize to make sure returned Hash is pure Ruby for
|
49
63
|
# MessagePack#pack which relies on pure Ruby object recognition
|
50
64
|
data = LogStash::Util.normalize(event.to_hash)
|
51
65
|
# timestamp is serialized as a iso8601 string
|
52
66
|
# merge to avoid modifying data which could have side effects if multiple outputs
|
53
|
-
@
|
67
|
+
@packer.clear
|
68
|
+
@on_event.call(event, @packer.pack([tag, epochtime, data.merge(LogStash::Event::TIMESTAMP => event.timestamp.to_iso8601)]))
|
54
69
|
end # def encode
|
55
70
|
|
56
71
|
def forwardable_tag(event)
|
@@ -67,6 +82,15 @@ class LogStash::Codecs::Fluent < LogStash::Codecs::Base
|
|
67
82
|
|
68
83
|
private
|
69
84
|
|
85
|
+
def decode_fluent_time(fluent_time)
|
86
|
+
case fluent_time
|
87
|
+
when Fixnum
|
88
|
+
fluent_time
|
89
|
+
when EventTime
|
90
|
+
Time.at(fluent_time.sec, fluent_time.nsec)
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
70
94
|
def decode_event(data, &block)
|
71
95
|
tag = data[0]
|
72
96
|
entries = data[1]
|
@@ -80,9 +104,9 @@ class LogStash::Codecs::Fluent < LogStash::Codecs::Base
|
|
80
104
|
raise(LogStash::Error, "PackedForward with compression is not supported")
|
81
105
|
end
|
82
106
|
|
83
|
-
entries_decoder =
|
107
|
+
entries_decoder = @decoder
|
84
108
|
entries_decoder.feed_each(entries) do |entry|
|
85
|
-
epochtime = entry[0]
|
109
|
+
epochtime = decode_fluent_time(entry[0])
|
86
110
|
map = entry[1]
|
87
111
|
event = LogStash::Event.new(map.merge(
|
88
112
|
LogStash::Event::TIMESTAMP => LogStash::Timestamp.at(epochtime),
|
@@ -93,7 +117,7 @@ class LogStash::Codecs::Fluent < LogStash::Codecs::Base
|
|
93
117
|
when Array
|
94
118
|
# Forward
|
95
119
|
entries.each do |entry|
|
96
|
-
epochtime = entry[0]
|
120
|
+
epochtime = decode_fluent_time(entry[0])
|
97
121
|
map = entry[1]
|
98
122
|
event = LogStash::Event.new(map.merge(
|
99
123
|
LogStash::Event::TIMESTAMP => LogStash::Timestamp.at(epochtime),
|
@@ -101,9 +125,9 @@ class LogStash::Codecs::Fluent < LogStash::Codecs::Base
|
|
101
125
|
))
|
102
126
|
yield event
|
103
127
|
end
|
104
|
-
when Fixnum
|
128
|
+
when Fixnum, EventTime
|
105
129
|
# Message
|
106
|
-
epochtime = entries
|
130
|
+
epochtime = decode_fluent_time(entries)
|
107
131
|
map = data[2]
|
108
132
|
event = LogStash::Event.new(map.merge(
|
109
133
|
LogStash::Event::TIMESTAMP => LogStash::Timestamp.at(epochtime),
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module LogStash; module Codecs; class Fluent;
|
2
|
+
class EventTime
|
3
|
+
attr_reader :sec, :nsec
|
4
|
+
|
5
|
+
TYPE = 0
|
6
|
+
|
7
|
+
def initialize(sec, nsec = 0)
|
8
|
+
@sec = sec
|
9
|
+
@nsec = nsec
|
10
|
+
end
|
11
|
+
|
12
|
+
def to_msgpack(io = nil)
|
13
|
+
@sec.to_msgpack(io)
|
14
|
+
end
|
15
|
+
|
16
|
+
def to_msgpack_ext
|
17
|
+
[@sec, @nsec].pack('NN')
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.from_msgpack_ext(data)
|
21
|
+
new(*data.unpack('NN'))
|
22
|
+
end
|
23
|
+
|
24
|
+
def to_json(*args)
|
25
|
+
@sec
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end; end; end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-codec-fluent'
|
4
|
-
s.version = '3.
|
4
|
+
s.version = '3.3.0'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Reads the `fluentd` `msgpack` schema"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
data/spec/codecs/fluent_spec.rb
CHANGED
@@ -2,8 +2,16 @@
|
|
2
2
|
require_relative "../spec_helper"
|
3
3
|
require "logstash/plugin"
|
4
4
|
require "logstash/event"
|
5
|
+
require "msgpack"
|
5
6
|
|
6
7
|
describe LogStash::Codecs::Fluent do
|
8
|
+
before do
|
9
|
+
@factory = MessagePack::Factory.new
|
10
|
+
@factory.register_type(LogStash::Codecs::Fluent::EventTime::TYPE,
|
11
|
+
LogStash::Codecs::Fluent::EventTime)
|
12
|
+
@packer = @factory.packer
|
13
|
+
@unpacker = @factory.unpacker
|
14
|
+
end
|
7
15
|
|
8
16
|
let(:properties) { {:name => "foo" } }
|
9
17
|
let(:event) { LogStash::Event.new(properties) }
|
@@ -17,9 +25,25 @@ describe LogStash::Codecs::Fluent do
|
|
17
25
|
|
18
26
|
it "should encode as message pack format" do
|
19
27
|
subject.on_event do |event, data|
|
20
|
-
|
21
|
-
|
22
|
-
|
28
|
+
@unpacker.feed_each(data) do |fields|
|
29
|
+
expect(fields[0]).to eq("log")
|
30
|
+
expect(fields[2]["name"]).to eq("foo")
|
31
|
+
end
|
32
|
+
end
|
33
|
+
subject.encode(event)
|
34
|
+
end
|
35
|
+
|
36
|
+
end
|
37
|
+
|
38
|
+
describe "event encoding with EventTime" do
|
39
|
+
subject { LogStash::Plugin.lookup("codec", "fluent").new({"nanosecond_precision" => true}) }
|
40
|
+
|
41
|
+
it "should encode as message pack format" do
|
42
|
+
subject.on_event do |event, data|
|
43
|
+
@unpacker.feed_each(data) do |fields|
|
44
|
+
expect(fields[0]).to eq("log")
|
45
|
+
expect(fields[2]["name"]).to eq("foo")
|
46
|
+
end
|
23
47
|
end
|
24
48
|
subject.encode(event)
|
25
49
|
end
|
@@ -32,8 +56,27 @@ describe LogStash::Codecs::Fluent do
|
|
32
56
|
let(:epochtime) { event.timestamp.to_i }
|
33
57
|
let(:data) { LogStash::Util.normalize(event.to_hash) }
|
34
58
|
let(:message) do
|
35
|
-
|
59
|
+
@packer.pack([tag, epochtime, data.merge(LogStash::Event::TIMESTAMP => event.timestamp.to_iso8601)])
|
60
|
+
end
|
61
|
+
|
62
|
+
it "should decode without errors" do
|
63
|
+
subject.decode(message) do |event|
|
64
|
+
expect(event.get("name")).to eq("foo")
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
end
|
69
|
+
|
70
|
+
describe "event decoding with EventTime" do
|
71
|
+
|
72
|
+
let(:tag) { "mytag" }
|
73
|
+
let(:epochtime) { LogStash::Codecs::Fluent::EventTime.new(event.timestamp.to_i,
|
74
|
+
event.timestamp.usec * 1000) }
|
75
|
+
let(:data) { LogStash::Util.normalize(event.to_hash) }
|
76
|
+
let(:message) do
|
77
|
+
@packer.pack([tag, epochtime, data.merge(LogStash::Event::TIMESTAMP => event.timestamp.to_iso8601)])
|
36
78
|
end
|
79
|
+
subject { LogStash::Plugin.lookup("codec", "fluent").new({"nanosecond_precision" => true}) }
|
37
80
|
|
38
81
|
it "should decode without errors" do
|
39
82
|
subject.decode(message) do |event|
|
@@ -87,13 +130,13 @@ describe LogStash::Codecs::Fluent do
|
|
87
130
|
let(:epochtime) { event.timestamp.to_i }
|
88
131
|
let(:data) { LogStash::Util.normalize(event.to_hash) }
|
89
132
|
let(:message) do
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
133
|
+
@packer.pack([tag,
|
134
|
+
[
|
135
|
+
[epochtime, data.merge(LogStash::Event::TIMESTAMP => event.timestamp.to_iso8601)],
|
136
|
+
[epochtime, data.merge(LogStash::Event::TIMESTAMP => event.timestamp.to_iso8601)],
|
137
|
+
[epochtime, data.merge(LogStash::Event::TIMESTAMP => event.timestamp.to_iso8601)]
|
138
|
+
]
|
139
|
+
])
|
97
140
|
end
|
98
141
|
|
99
142
|
it "should decode without errors" do
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-codec-fluent
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.3.0
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2019-11-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -74,6 +74,7 @@ files:
|
|
74
74
|
- README.md
|
75
75
|
- docs/index.asciidoc
|
76
76
|
- lib/logstash/codecs/fluent.rb
|
77
|
+
- lib/logstash/codecs/fluent/event_time.rb
|
77
78
|
- logstash-codec-fluent.gemspec
|
78
79
|
- spec/codecs/fluent_spec.rb
|
79
80
|
- spec/spec_helper.rb
|