fluent-plugin-concat 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +5 -0
- data/fluent-plugin-concat.gemspec +1 -1
- data/lib/fluent/plugin/filter_concat.rb +22 -15
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: aba37ca7a793288cddef2c986f31caec1c492b4f
|
4
|
+
data.tar.gz: beaa009f9c3bc35bc38e39390918bdd29e53361b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f62bffeed9583c683da11a8b6b7f58f4ab45fd1d39905e46a8cfa11747e96a185f5464933c9cc989aa466dd199215855b4ca2cb5a4510d9b54b13b51f6f1eb59
|
7
|
+
data.tar.gz: 569e6b2393a83a0303cab98319ddcd412f6c11245ac26dde9524ff5bf12cbd7c82fae9765e901a84cfa834b8514978d0219f726ea5b8b34418b2b26bfeea4ea5
|
data/README.md
CHANGED
@@ -52,6 +52,11 @@ The key to determine which stream an event belongs to.
|
|
52
52
|
**flush\_interval**
|
53
53
|
|
54
54
|
The number of seconds after which the last received event log will be flushed.
|
55
|
+
If specified 0, wait for next line forever.
|
56
|
+
|
57
|
+
**use\_first\_timestamp**
|
58
|
+
|
59
|
+
Use timestamp of first record when buffer is flushed.
|
55
60
|
|
56
61
|
## Usage
|
57
62
|
|
@@ -14,10 +14,12 @@ module Fluent
|
|
14
14
|
config_param :multiline_end_regexp, :string, default: nil
|
15
15
|
desc "The key to determine which stream an event belongs to"
|
16
16
|
config_param :stream_identity_key, :string, default: nil
|
17
|
-
desc "The interval between data flushes"
|
17
|
+
desc "The interval between data flushes, 0 means disable timeout"
|
18
18
|
config_param :flush_interval, :time, default: 60
|
19
19
|
desc "The label name to handle timeout"
|
20
20
|
config_param :timeout_label, :string, default: nil
|
21
|
+
desc "Use timestamp of first record when buffer is flushed"
|
22
|
+
config_param :use_first_timestamp, :bool, default: false
|
21
23
|
|
22
24
|
class TimeoutError < StandardError
|
23
25
|
end
|
@@ -74,8 +76,11 @@ module Fluent
|
|
74
76
|
new_es = MultiEventStream.new
|
75
77
|
es.each do |time, record|
|
76
78
|
begin
|
77
|
-
new_record = process(tag, time, record)
|
78
|
-
|
79
|
+
new_time, new_record = process(tag, time, record)
|
80
|
+
if new_record
|
81
|
+
time = new_time if @use_first_timestamp
|
82
|
+
new_es.add(time, record.merge(new_record))
|
83
|
+
end
|
79
84
|
rescue => e
|
80
85
|
router.emit_error_event(tag, time, record, e)
|
81
86
|
end
|
@@ -86,6 +91,7 @@ module Fluent
|
|
86
91
|
private
|
87
92
|
|
88
93
|
def on_timer
|
94
|
+
return if @flush_interval <= 0
|
89
95
|
return if @finished
|
90
96
|
flush_timeout_buffer
|
91
97
|
end
|
@@ -104,19 +110,20 @@ module Fluent
|
|
104
110
|
return flush_buffer(stream_identity)
|
105
111
|
end
|
106
112
|
when :regexp
|
107
|
-
|
108
|
-
when firstline?(record[@key])
|
113
|
+
if firstline?(record[@key])
|
109
114
|
if @buffer[stream_identity].empty?
|
110
115
|
@buffer[stream_identity] << [tag, time, record]
|
111
116
|
else
|
112
117
|
return flush_buffer(stream_identity, [tag, time, record])
|
113
118
|
end
|
114
|
-
|
115
|
-
|
119
|
+
end
|
120
|
+
if lastline?(record[@key])
|
121
|
+
@buffer[stream_identity] << [tag, time, record] unless firstline?(record[@key])
|
116
122
|
return flush_buffer(stream_identity)
|
117
|
-
|
123
|
+
end
|
124
|
+
if !firstline?(record[@key]) && !lastline?(record[@key])
|
118
125
|
if @buffer[stream_identity].empty?
|
119
|
-
return record
|
126
|
+
return [time, record]
|
120
127
|
else
|
121
128
|
# Continuation of the previous line
|
122
129
|
@buffer[stream_identity] << [tag, time, record]
|
@@ -127,7 +134,7 @@ module Fluent
|
|
127
134
|
end
|
128
135
|
|
129
136
|
def firstline?(text)
|
130
|
-
!!@multiline_start_regexp.match(text)
|
137
|
+
@multiline_start_regexp && !!@multiline_start_regexp.match(text)
|
131
138
|
end
|
132
139
|
|
133
140
|
def lastline?(text)
|
@@ -136,13 +143,13 @@ module Fluent
|
|
136
143
|
|
137
144
|
def flush_buffer(stream_identity, new_element = nil)
|
138
145
|
lines = @buffer[stream_identity].map {|_tag, _time, record| record[@key] }
|
139
|
-
_tag,
|
146
|
+
_tag, time, first_record = @buffer[stream_identity].first
|
140
147
|
new_record = {
|
141
148
|
@key => lines.join(@separator)
|
142
149
|
}
|
143
150
|
@buffer[stream_identity] = []
|
144
151
|
@buffer[stream_identity] << new_element if new_element
|
145
|
-
|
152
|
+
[time, first_record.merge(new_record)]
|
146
153
|
end
|
147
154
|
|
148
155
|
def flush_timeout_buffer
|
@@ -150,11 +157,11 @@ module Fluent
|
|
150
157
|
timeout_stream_identities = []
|
151
158
|
@timeout_map.each do |stream_identity, previous_timestamp|
|
152
159
|
next if @flush_interval > (now - previous_timestamp)
|
153
|
-
flushed_record = flush_buffer(stream_identity)
|
160
|
+
time, flushed_record = flush_buffer(stream_identity)
|
154
161
|
timeout_stream_identities << stream_identity
|
155
162
|
tag = stream_identity.split(":").first
|
156
163
|
message = "Timeout flush: #{stream_identity}"
|
157
|
-
handle_timeout_error(tag, now, flushed_record, message)
|
164
|
+
handle_timeout_error(tag, @use_first_timestamp ? time : now, flushed_record, message)
|
158
165
|
log.info(message)
|
159
166
|
end
|
160
167
|
@timeout_map.reject! do |stream_identity, _|
|
@@ -170,7 +177,7 @@ module Fluent
|
|
170
177
|
new_record = {
|
171
178
|
@key => lines.join(@separator)
|
172
179
|
}
|
173
|
-
tag, time, record = elements.
|
180
|
+
tag, time, record = elements.first
|
174
181
|
message = "Flush remaining buffer: #{stream_identity}"
|
175
182
|
handle_timeout_error(tag, time, record.merge(new_record), message)
|
176
183
|
log.info(message)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-concat
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kenji Okimoto
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-08-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -139,7 +139,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
139
139
|
version: '0'
|
140
140
|
requirements: []
|
141
141
|
rubyforge_project:
|
142
|
-
rubygems_version: 2.
|
142
|
+
rubygems_version: 2.6.4
|
143
143
|
signing_key:
|
144
144
|
specification_version: 4
|
145
145
|
summary: Fluentd Filter plugin to concat multiple event messages
|