fluent-plugin-concat 2.2.3 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/NEWS.md +6 -0
- data/README.md +66 -6
- data/fluent-plugin-concat.gemspec +1 -1
- data/lib/fluent/plugin/filter_concat.rb +37 -4
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ee5e1936b898d90227550ba4baadba4f3d64d31577a1b0923aa634a0b3a49e6c
|
4
|
+
data.tar.gz: 59be67a9067f3589e9728702a4b4a8bb9f25e8915eedd4aa308dc297e710ba46
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2827392627d1d6d1eba547bb1d167fd86d9ebb88cb0c5aa98db6cb124a254da0af841de55a9ef4f437548a6c4175202f8bd751647ee711d40b2fd3607400f829
|
7
|
+
data.tar.gz: c827c2d9b14b509abd801428ac9aae29f34d99b12e6359e2152eb4b4716a7204dddf3638a9e9124da0eaee0d17fb8edd5e2ff18e6c3ff7fba5be4b000fad097f
|
data/NEWS.md
CHANGED
data/README.md
CHANGED
@@ -29,16 +29,16 @@ Or install it yourself as:
|
|
29
29
|
|
30
30
|
## Configuration
|
31
31
|
|
32
|
-
**key** (required)
|
32
|
+
**key** (string) (required)
|
33
33
|
|
34
34
|
The key for part of multiline log.
|
35
35
|
|
36
|
-
**separator**
|
36
|
+
**separator** (string)
|
37
37
|
|
38
38
|
The separator of lines.
|
39
39
|
Default value is `"\n"`.
|
40
40
|
|
41
|
-
**n\_lines**
|
41
|
+
**n\_lines** (integer)
|
42
42
|
|
43
43
|
The number of lines.
|
44
44
|
This is exclusive with `multiline_start_regex`.
|
@@ -58,18 +58,33 @@ This is exclusive with `n_lines.`
|
|
58
58
|
The regexp to match continuous lines.
|
59
59
|
This is exclusive with `n_lines.`
|
60
60
|
|
61
|
-
**stream\_identity\_key**
|
61
|
+
**stream\_identity\_key** (string)
|
62
62
|
|
63
63
|
The key to determine which stream an event belongs to.
|
64
64
|
|
65
|
-
**flush\_interval**
|
65
|
+
**flush\_interval** (integer)
|
66
66
|
|
67
67
|
The number of seconds after which the last received event log will be flushed.
|
68
68
|
If specified 0, wait for next line forever.
|
69
69
|
|
70
|
-
**use\_first\_timestamp**
|
70
|
+
**use\_first\_timestamp** (bool)
|
71
71
|
|
72
72
|
Use timestamp of first record when buffer is flushed.
|
73
|
+
Default value is `false`.
|
74
|
+
|
75
|
+
**partial\_key** (string)
|
76
|
+
|
77
|
+
The field name that is the reference to concatenate records
|
78
|
+
|
79
|
+
**partial\_value** (string)
|
80
|
+
|
81
|
+
The value stored in the field specified by partial_key that represent partial log
|
82
|
+
|
83
|
+
**keep\_partial\_key** (bool)
|
84
|
+
|
85
|
+
If true, keep partial_key in concatenated records
|
86
|
+
Default value is `false`.
|
87
|
+
|
73
88
|
|
74
89
|
## Usage
|
75
90
|
|
@@ -137,6 +152,51 @@ Handle single line JSON from Docker containers.
|
|
137
152
|
</filter>
|
138
153
|
```
|
139
154
|
|
155
|
+
Handle Docker's `partial_message`.
|
156
|
+
|
157
|
+
```aconf
|
158
|
+
<filter>
|
159
|
+
@type concat
|
160
|
+
key message
|
161
|
+
partial_key partial_message
|
162
|
+
partial_value true
|
163
|
+
</filter>
|
164
|
+
```
|
165
|
+
|
166
|
+
Handle containerd/cri in Kubernetes.
|
167
|
+
|
168
|
+
```aconf
|
169
|
+
<source>
|
170
|
+
@type tail
|
171
|
+
path /var/log/containers/*.log
|
172
|
+
<parse>
|
173
|
+
@type regexp
|
174
|
+
expression /^(?<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z) (?<output>\w+) (?<partial_flag>[FP]) (?<message>.+)$/
|
175
|
+
</parse>
|
176
|
+
tag k8s
|
177
|
+
@label @CONCAT
|
178
|
+
</source>
|
179
|
+
|
180
|
+
<label @CONCAT>
|
181
|
+
<filter k8s>
|
182
|
+
@type concat
|
183
|
+
key message
|
184
|
+
partial_key partial_flag
|
185
|
+
partial_value P
|
186
|
+
</filter>
|
187
|
+
<match k8s>
|
188
|
+
@type relabel
|
189
|
+
@label @OUTPUT
|
190
|
+
</match>
|
191
|
+
</label>
|
192
|
+
|
193
|
+
<label @OUTPUT>
|
194
|
+
<match>
|
195
|
+
@type stdout
|
196
|
+
</match>
|
197
|
+
</label>
|
198
|
+
```
|
199
|
+
|
140
200
|
## Contributing
|
141
201
|
|
142
202
|
1. Fork it
|
@@ -26,6 +26,12 @@ module Fluent::Plugin
|
|
26
26
|
config_param :timeout_label, :string, default: nil
|
27
27
|
desc "Use timestamp of first record when buffer is flushed"
|
28
28
|
config_param :use_first_timestamp, :bool, default: false
|
29
|
+
desc "The field name that is the reference to concatenate records"
|
30
|
+
config_param :partial_key, :string, default: nil
|
31
|
+
desc "The value stored in the field specified by partial_key that represent partial log"
|
32
|
+
config_param :partial_value, :string, default: nil
|
33
|
+
desc "If true, keep partial_key in concatenated records"
|
34
|
+
config_param :keep_partial_key, :bool, default: false
|
29
35
|
|
30
36
|
class TimeoutError < StandardError
|
31
37
|
end
|
@@ -43,17 +49,28 @@ module Fluent::Plugin
|
|
43
49
|
def configure(conf)
|
44
50
|
super
|
45
51
|
|
46
|
-
if @n_lines && (@multiline_start_regexp || @multiline_end_regexp
|
47
|
-
raise Fluent::ConfigError, "n_lines and multiline_start_regexp/multiline_end_regexp
|
52
|
+
if @n_lines && (@multiline_start_regexp || @multiline_end_regexp)
|
53
|
+
raise Fluent::ConfigError, "n_lines and multiline_start_regexp/multiline_end_regexp are exclusive"
|
48
54
|
end
|
49
|
-
if @n_lines.nil? && @multiline_start_regexp.nil? && @multiline_end_regexp.nil?
|
55
|
+
if @n_lines.nil? && @multiline_start_regexp.nil? && @multiline_end_regexp.nil? && @partial_key.nil?
|
50
56
|
raise Fluent::ConfigError, "Either n_lines or multiline_start_regexp or multiline_end_regexp is required"
|
51
57
|
end
|
58
|
+
if @partial_key && @n_lines
|
59
|
+
raise Fluent::ConfigError, "partial_key and n_lines are exclusive"
|
60
|
+
end
|
61
|
+
if @partial_key && (@multiline_start_regexp || @multiline_end_regexp)
|
62
|
+
raise Fluent::ConfigError, "partial_key and multiline_start_regexp/multiline_end_regexp are exclusive"
|
63
|
+
end
|
64
|
+
if @partial_key && @partial_value.nil?
|
65
|
+
raise Fluent::ConfigError, "partial_value is required when partial_key is specified"
|
66
|
+
end
|
52
67
|
|
53
68
|
@mode = nil
|
54
69
|
case
|
55
70
|
when @n_lines
|
56
71
|
@mode = :line
|
72
|
+
when @partial_key
|
73
|
+
@mode = :partial
|
57
74
|
when @multiline_start_regexp || @multiline_end_regexp
|
58
75
|
@mode = :regexp
|
59
76
|
if @multiline_start_regexp
|
@@ -96,7 +113,9 @@ module Fluent::Plugin
|
|
96
113
|
unless flushed_es.empty?
|
97
114
|
flushed_es.each do |_time, new_record|
|
98
115
|
time = _time if @use_first_timestamp
|
99
|
-
|
116
|
+
merged_record = record.merge(new_record)
|
117
|
+
merged_record.delete(@partial_key) unless @keep_partial_key
|
118
|
+
new_es.add(time, merged_record)
|
100
119
|
end
|
101
120
|
end
|
102
121
|
rescue => e
|
@@ -128,6 +147,8 @@ module Fluent::Plugin
|
|
128
147
|
case @mode
|
129
148
|
when :line
|
130
149
|
process_line(stream_identity, tag, time, record)
|
150
|
+
when :partial
|
151
|
+
process_partial(stream_identity, tag, time, record)
|
131
152
|
when :regexp
|
132
153
|
process_regexp(stream_identity, tag, time, record)
|
133
154
|
end
|
@@ -144,6 +165,18 @@ module Fluent::Plugin
|
|
144
165
|
new_es
|
145
166
|
end
|
146
167
|
|
168
|
+
def process_partial(stream_identity, tag, time, record)
|
169
|
+
new_es = Fluent::MultiEventStream.new
|
170
|
+
@buffer[stream_identity] << [tag, time, record]
|
171
|
+
unless @partial_value == record[@partial_key]
|
172
|
+
new_time, new_record = flush_buffer(stream_identity)
|
173
|
+
time = new_time if @use_first_timestamp
|
174
|
+
new_record.delete(@partial_key)
|
175
|
+
new_es.add(time, new_record)
|
176
|
+
end
|
177
|
+
new_es
|
178
|
+
end
|
179
|
+
|
147
180
|
def process_regexp(stream_identity, tag, time, record)
|
148
181
|
new_es = Fluent::MultiEventStream.new
|
149
182
|
case
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-concat
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kenji Okimoto
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-07-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|