fluent-plugin-concat 2.4.0 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/linux.yml +30 -0
- data/.github/workflows/windows.yml +26 -0
- data/NEWS.md +6 -0
- data/README.md +79 -69
- data/fluent-plugin-concat.gemspec +2 -1
- data/lib/fluent/plugin/filter_concat.rb +89 -21
- metadata +22 -7
- data/.travis.yml +0 -19
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0fa8ad39f08cb1de3cf3137fd5c504e285cc82107b10b3faf9455a8b08c002ca
|
4
|
+
data.tar.gz: d4f0fbfa8974961fcb2e4e7128e8d823c5a607a4087fc063d936c90896e267c9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 173c3b6dcb69212174c1bca085e44b26da8137c0c4cd935e622b54ba019150a720e1eb863a4c3aff7503b8ee752aa992e694e321423c0cc2e3d846ccf2d79215
|
7
|
+
data.tar.gz: 11f274a4e8e4ae32f8e8f91d4eabd7ba72d98c794ff202b5e111b475024b40cc61fae65698eaebf645cc161fa8b3a9c632ff1f6de778fa369afe9fa57459c126
|
@@ -0,0 +1,30 @@
|
|
1
|
+
name: Testing on Ubuntu
|
2
|
+
on:
|
3
|
+
- push
|
4
|
+
- pull_request
|
5
|
+
jobs:
|
6
|
+
build:
|
7
|
+
runs-on: ${{ matrix.os }}
|
8
|
+
strategy:
|
9
|
+
fail-fast: false
|
10
|
+
matrix:
|
11
|
+
ruby:
|
12
|
+
- 2.5
|
13
|
+
- 2.6
|
14
|
+
- 2.7
|
15
|
+
- 3.0
|
16
|
+
os:
|
17
|
+
- ubuntu-latest
|
18
|
+
name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
|
19
|
+
steps:
|
20
|
+
- uses: actions/checkout@v2
|
21
|
+
- uses: ruby/setup-ruby@v1
|
22
|
+
with:
|
23
|
+
ruby-version: ${{ matrix.ruby }}
|
24
|
+
- name: unit testing
|
25
|
+
env:
|
26
|
+
CI: true
|
27
|
+
run: |
|
28
|
+
gem install bundler rake
|
29
|
+
bundle install --jobs 4 --retry 3
|
30
|
+
bundle exec rake test
|
@@ -0,0 +1,26 @@
|
|
1
|
+
name: Testing on Windows
|
2
|
+
on:
|
3
|
+
- push
|
4
|
+
- pull_request
|
5
|
+
jobs:
|
6
|
+
build:
|
7
|
+
runs-on: ${{ matrix.os }}
|
8
|
+
strategy:
|
9
|
+
fail-fast: false
|
10
|
+
matrix:
|
11
|
+
ruby: [ '2.5', '2.6', '2.7', '3.0' ]
|
12
|
+
os:
|
13
|
+
- windows-latest
|
14
|
+
name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
|
15
|
+
steps:
|
16
|
+
- uses: actions/checkout@v2
|
17
|
+
- uses: ruby/setup-ruby@v1
|
18
|
+
with:
|
19
|
+
ruby-version: ${{ matrix.ruby }}
|
20
|
+
- name: unit testing
|
21
|
+
env:
|
22
|
+
CI: true
|
23
|
+
run: |
|
24
|
+
gem install bundler rake
|
25
|
+
bundle install --jobs 4 --retry 3
|
26
|
+
bundle exec rake test
|
data/NEWS.md
CHANGED
data/README.md
CHANGED
@@ -29,73 +29,56 @@ Or install it yourself as:
|
|
29
29
|
|
30
30
|
## Configuration
|
31
31
|
|
32
|
-
|
32
|
+
### Example
|
33
33
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
The regexp to match continuous lines.
|
59
|
-
This is exclusive with `n_lines.`
|
60
|
-
|
61
|
-
**stream\_identity\_key** (string) (optional)
|
62
|
-
|
63
|
-
The key to determine which stream an event belongs to.
|
64
|
-
|
65
|
-
**flush\_interval** (integer) (optional)
|
66
|
-
|
67
|
-
The number of seconds after which the last received event log will be flushed.
|
68
|
-
If specified 0, wait for next line forever.
|
69
|
-
|
70
|
-
**timeout\_label** (string) (optional)
|
71
|
-
|
72
|
-
The label name to handle events caused by timeout.
|
73
|
-
|
74
|
-
**use\_first\_timestamp** (bool) (optional)
|
75
|
-
|
76
|
-
Use timestamp of first record when buffer is flushed.
|
77
|
-
Default value is `false`.
|
78
|
-
|
79
|
-
**partial\_key** (string) (optional)
|
80
|
-
|
81
|
-
The field name that is the reference to concatenate records
|
82
|
-
|
83
|
-
**partial\_value** (string) (optional)
|
84
|
-
|
85
|
-
The value stored in the field specified by partial_key that represent partial log
|
86
|
-
|
87
|
-
**keep\_partial\_key** (bool) (optional)
|
88
|
-
|
89
|
-
If true, keep partial_key in concatenated records
|
90
|
-
Default value is `false`.
|
91
|
-
|
92
|
-
**use\_partial\_metadata** (bool) (optional)
|
93
|
-
|
94
|
-
Use partial metadata to concatenate multiple records
|
95
|
-
|
96
|
-
**keep\_partial\_metadata** (bool) (optional)
|
34
|
+
```
|
35
|
+
<filter docker.log>
|
36
|
+
@type concat
|
37
|
+
key loga
|
38
|
+
#separator "\n"
|
39
|
+
n_lines 10
|
40
|
+
#multiline_start_regexp /^Start/
|
41
|
+
#multiline_end_regexp /^End/
|
42
|
+
#continuous_line_regexp nil
|
43
|
+
#stream_identity_key nil
|
44
|
+
#flush_interval 60
|
45
|
+
#timeout_label nil
|
46
|
+
#use_first_timestamp false
|
47
|
+
#partial_key nil
|
48
|
+
#partial_value nil
|
49
|
+
#keep_partial_key false
|
50
|
+
#use_partial_metadata false
|
51
|
+
#keep_partial_metadata false
|
52
|
+
#partial\_metadata\_format docker-fluentd
|
53
|
+
#use\_partial\_cri\_logtag false
|
54
|
+
#partial\_cri\_logtag\_key nil
|
55
|
+
#partial\_cri\_stream\_key stream
|
56
|
+
</filter>
|
57
|
+
```
|
97
58
|
|
98
|
-
|
59
|
+
### Parameter
|
60
|
+
|
61
|
+
|parameter|description|default|
|
62
|
+
|---|---|---|
|
63
|
+
|key|The key for part of multiline log||
|
64
|
+
|separator|The separator of lines|`"\n"`|
|
65
|
+
|n\_lines|The number of lines. This is exclusive with `multiline_start_regex`|nil|
|
66
|
+
|multiline\_start\_regexp|The regexp to match beginning of multiline. This is exclusive with `n_lines`|nil|
|
67
|
+
|multiline\_end\_regexp|The regexp to match ending of multiline.This is exclusive with `n_lines`|nil|
|
68
|
+
|continuous\_line\_regexp|The regexp to match continuous lines.This is exclusive with `n_lines`|nil|
|
69
|
+
|stream\_identity\_key|The key to determine which stream an event belongs to|nil|
|
70
|
+
|flush\_interval|The number of seconds after which the last received event log will be flushed.If specified 0, wait for next line foreverr|60|
|
71
|
+
|timeout\_label|The label name to handle events caused by timeout|nil|
|
72
|
+
|use\_first\_timestamp|Use timestamp of first record when buffer is flushed|`false`|
|
73
|
+
|partial\_key|The field name that is the reference to concatenate records|nil|
|
74
|
+
|partial\_value|The value stored in the field specified by partial_key that represent partial log|nil|
|
75
|
+
|keep\_partial\_key|If true, keep partial_key in concatenated records|`false`|
|
76
|
+
|use\_partial\_metadata|Use partial metadata to concatenate multiple records|`false`|
|
77
|
+
|keep\_partial\_metadata|If true, keep partial metadata|`false`|
|
78
|
+
|partial\_metadata\_format|Input format of the partial metadata (fluentd or journald docker log driver) ( `docker-fluentd`, `docker-journald`, `docker-journald-lowercase`)<br>Configure based on the input plugin, that is used. <br>The docker fluentd and journald log drivers are behaving differently, so the plugin needs to know, what to look for.<br>Use `docker-journald-lowercase`, if you have `fields_lowercase true` in the `journald` source config |`docker-fluentd`|
|
79
|
+
|use\_partial\_cri\_logtag|bool (optional)|Use cri log tag to concatenate multiple records||
|
80
|
+
|partial\_cri\_logtag\_key|string (optional)|The key name that is referred to concatenate records on cri log||
|
81
|
+
|partial\_cri\_stream\_key|string (optional)|The key name that is referred to detect stream name on cri log|`stream`|
|
99
82
|
|
100
83
|
## Usage
|
101
84
|
|
@@ -168,13 +151,38 @@ Handle Docker logs splitted in several parts (using `partial_message`), and do n
|
|
168
151
|
```aconf
|
169
152
|
<filter>
|
170
153
|
@type concat
|
171
|
-
key
|
154
|
+
key log
|
172
155
|
partial_key partial_message
|
173
156
|
partial_value true
|
174
157
|
separator ""
|
175
158
|
</filter>
|
176
159
|
```
|
177
160
|
|
161
|
+
(Docker v19.03+) Handle Docker logs splitted in several parts (using `use_partial_metadata`), and do not add new line between parts.
|
162
|
+
|
163
|
+
```aconf
|
164
|
+
<filter>
|
165
|
+
@type concat
|
166
|
+
key log
|
167
|
+
use_partial_metadata true
|
168
|
+
separator ""
|
169
|
+
</filter>
|
170
|
+
```
|
171
|
+
|
172
|
+
(Docker v20.10+) Handle Docker logs splitted in several parts (using `use_partial_metadata`), and do not add new line between parts.
|
173
|
+
|
174
|
+
Docker v20.10 improved partial message handling by adding better metadata in the journald log driver, this works now similarily to the fluentd log driver, but requires one additional setting
|
175
|
+
|
176
|
+
```aconf
|
177
|
+
<filter>
|
178
|
+
@type concat
|
179
|
+
key log
|
180
|
+
use_partial_metadata true
|
181
|
+
partial_metadata_format docker-journald
|
182
|
+
separator ""
|
183
|
+
</filter>
|
184
|
+
```
|
185
|
+
|
178
186
|
Handle Docker logs splitted in several parts (using newline detection), and do not add new line between parts (prior to Docker 18.06).
|
179
187
|
|
180
188
|
```aconf
|
@@ -194,7 +202,8 @@ Handle containerd/cri in Kubernetes.
|
|
194
202
|
path /var/log/containers/*.log
|
195
203
|
<parse>
|
196
204
|
@type regexp
|
197
|
-
expression /^(?<
|
205
|
+
expression /^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<message>.*)$/
|
206
|
+
time_format %Y-%m-%dT%H:%M:%S.%L%z
|
198
207
|
</parse>
|
199
208
|
tag k8s
|
200
209
|
@label @CONCAT
|
@@ -204,8 +213,9 @@ Handle containerd/cri in Kubernetes.
|
|
204
213
|
<filter k8s>
|
205
214
|
@type concat
|
206
215
|
key message
|
207
|
-
|
208
|
-
|
216
|
+
use_partial_cri_logtag true
|
217
|
+
partial_cri_logtag_key logtag
|
218
|
+
partial_cri_stream_key stream
|
209
219
|
</filter>
|
210
220
|
<match k8s>
|
211
221
|
@type relabel
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
6
|
spec.name = "fluent-plugin-concat"
|
7
|
-
spec.version = "2.
|
7
|
+
spec.version = "2.5.0"
|
8
8
|
spec.authors = ["Kenji Okimoto"]
|
9
9
|
spec.email = ["okimoto@clear-code.com"]
|
10
10
|
|
@@ -22,4 +22,5 @@ Gem::Specification.new do |spec|
|
|
22
22
|
spec.add_development_dependency "test-unit", ">= 3.1.0"
|
23
23
|
spec.add_development_dependency "test-unit-rr"
|
24
24
|
spec.add_development_dependency "appraisal"
|
25
|
+
spec.add_development_dependency "webrick"
|
25
26
|
end
|
@@ -34,8 +34,16 @@ module Fluent::Plugin
|
|
34
34
|
config_param :keep_partial_key, :bool, default: false
|
35
35
|
desc "Use partial metadata to concatenate multiple records"
|
36
36
|
config_param :use_partial_metadata, :bool, default: false
|
37
|
+
desc "Input format of the partial metadata (fluentd or journald docker log driver)"
|
38
|
+
config_param :partial_metadata_format, :enum, list: [:"docker-fluentd", :"docker-journald", :"docker-journald-lowercase"], default: :"docker-fluentd"
|
37
39
|
desc "If true, keep partial metadata"
|
38
40
|
config_param :keep_partial_metadata, :bool, default: false
|
41
|
+
desc "Use cri log tag to concatenate multiple records"
|
42
|
+
config_param :use_partial_cri_logtag, :bool, default: false
|
43
|
+
desc "The key name that is referred to concatenate records on cri log"
|
44
|
+
config_param :partial_cri_logtag_key, :string, default: nil
|
45
|
+
desc "The key name that is referred to detect stream name on cri log"
|
46
|
+
config_param :partial_cri_stream_key, :string, default: "stream"
|
39
47
|
|
40
48
|
class TimeoutError < StandardError
|
41
49
|
end
|
@@ -50,11 +58,18 @@ module Fluent::Plugin
|
|
50
58
|
end
|
51
59
|
end
|
52
60
|
|
61
|
+
def required_params
|
62
|
+
params = [@n_lines.nil?, @multiline_start_regexp.nil?, @multiline_end_regexp.nil?, @partial_key.nil?, !@use_partial_metadata, !@use_partial_cri_logtag]
|
63
|
+
names = ["n_lines", "multiline_start_regexp", "multiline_end_regexp", "partial_key", "use_partial_metadata", "use_partial_cri_logtag"]
|
64
|
+
return params, names
|
65
|
+
end
|
66
|
+
|
53
67
|
def configure(conf)
|
54
68
|
super
|
55
69
|
|
56
|
-
|
57
|
-
|
70
|
+
params, names = required_params
|
71
|
+
if params.all?
|
72
|
+
raise Fluent::ConfigError, "Either #{[names[0..-2].join(", "), names[-1]].join(" or ")} is required"
|
58
73
|
end
|
59
74
|
if @n_lines && (@multiline_start_regexp || @multiline_end_regexp)
|
60
75
|
raise Fluent::ConfigError, "n_lines and multiline_start_regexp/multiline_end_regexp are exclusive"
|
@@ -69,13 +84,22 @@ module Fluent::Plugin
|
|
69
84
|
raise Fluent::ConfigError, "partial_value is required when partial_key is specified"
|
70
85
|
end
|
71
86
|
if @use_partial_metadata && @n_lines
|
72
|
-
raise Fluent::ConfigError, "
|
87
|
+
raise Fluent::ConfigError, "use_partial_metadata and n_lines are exclusive"
|
73
88
|
end
|
74
89
|
if @use_partial_metadata && (@multiline_start_regexp || @multiline_end_regexp)
|
75
|
-
raise Fluent::ConfigError, "
|
90
|
+
raise Fluent::ConfigError, "use_partial_metadata and multiline_start_regexp/multiline_end_regexp are exclusive"
|
76
91
|
end
|
77
92
|
if @use_partial_metadata && @partial_key
|
78
|
-
raise Fluent::ConfigError, "
|
93
|
+
raise Fluent::ConfigError, "use_partial_metadata and partial_key are exclusive"
|
94
|
+
end
|
95
|
+
if @use_partial_cri_logtag && @n_lines
|
96
|
+
raise Fluent::ConfigError, "use_partial_cri_logtag and n_lines are exclusive"
|
97
|
+
end
|
98
|
+
if @use_partial_cri_logtag && (@multiline_start_regexp || @multiline_end_regexp)
|
99
|
+
raise Fluent::ConfigError, "use_partial_cri_logtag and multiline_start_regexp/multiline_end_regexp are exclusive"
|
100
|
+
end
|
101
|
+
if @use_partial_cri_logtag && @partial_key
|
102
|
+
raise Fluent::ConfigError, "use_partial_cri_logtag and partial_key are exclusive"
|
79
103
|
end
|
80
104
|
|
81
105
|
@mode = nil
|
@@ -86,6 +110,33 @@ module Fluent::Plugin
|
|
86
110
|
@mode = :partial
|
87
111
|
when @use_partial_metadata
|
88
112
|
@mode = :partial_metadata
|
113
|
+
case @partial_metadata_format
|
114
|
+
when :"docker-fluentd"
|
115
|
+
@partial_message_field = "partial_message".freeze
|
116
|
+
@partial_id_field = "partial_id".freeze
|
117
|
+
@partial_ordinal_field = "partial_ordinal".freeze
|
118
|
+
@partial_last_field = "partial_last".freeze
|
119
|
+
@partial_message_indicator = @partial_message_field
|
120
|
+
when :"docker-journald"
|
121
|
+
@partial_message_field = "CONTAINER_PARTIAL_MESSAGE".freeze
|
122
|
+
@partial_id_field = "CONTAINER_PARTIAL_ID".freeze
|
123
|
+
@partial_ordinal_field = "CONTAINER_PARTIAL_ORDINAL".freeze
|
124
|
+
@partial_last_field = "CONTAINER_PARTIAL_LAST".freeze
|
125
|
+
# the journald log driver does not add CONTAINER_PARTIAL_MESSAGE to the last message
|
126
|
+
# so we help ourself by using another indicator
|
127
|
+
@partial_message_indicator = @partial_id_field
|
128
|
+
when :"docker-journald-lowercase"
|
129
|
+
@partial_message_field = "container_partial_message".freeze
|
130
|
+
@partial_id_field = "container_partial_id".freeze
|
131
|
+
@partial_ordinal_field = "container_partial_ordinal".freeze
|
132
|
+
@partial_last_field = "container_partial_last".freeze
|
133
|
+
@partial_message_indicator = @partial_id_field
|
134
|
+
end
|
135
|
+
when @use_partial_cri_logtag
|
136
|
+
@mode = :partial_cri
|
137
|
+
@partial_logtag_delimiter = ":".freeze
|
138
|
+
@partial_logtag_continue = "P".freeze
|
139
|
+
@partial_logtag_full = "F".freeze
|
89
140
|
when @multiline_start_regexp || @multiline_end_regexp
|
90
141
|
@mode = :regexp
|
91
142
|
if @multiline_start_regexp
|
@@ -113,12 +164,12 @@ module Fluent::Plugin
|
|
113
164
|
end
|
114
165
|
|
115
166
|
def filter_stream(tag, es)
|
167
|
+
if /\Afluent\.(?:trace|debug|info|warn|error|fatal)\z/ =~ tag
|
168
|
+
return es
|
169
|
+
end
|
170
|
+
|
116
171
|
new_es = Fluent::MultiEventStream.new
|
117
172
|
es.each do |time, record|
|
118
|
-
if /\Afluent\.(?:trace|debug|info|warn|error|fatal)\z/ =~ tag
|
119
|
-
new_es.add(time, record)
|
120
|
-
next
|
121
|
-
end
|
122
173
|
unless record.key?(@key)
|
123
174
|
new_es.add(time, record)
|
124
175
|
next
|
@@ -130,7 +181,7 @@ module Fluent::Plugin
|
|
130
181
|
end
|
131
182
|
end
|
132
183
|
if @mode == :partial_metadata
|
133
|
-
unless record.key?(
|
184
|
+
unless record.key?(@partial_message_indicator)
|
134
185
|
new_es.add(time, record)
|
135
186
|
next
|
136
187
|
end
|
@@ -146,11 +197,14 @@ module Fluent::Plugin
|
|
146
197
|
merged_record.delete(@partial_key) unless @keep_partial_key
|
147
198
|
when :partial_metadata
|
148
199
|
unless @keep_partial_metadata
|
149
|
-
merged_record.delete(
|
150
|
-
merged_record.delete(
|
151
|
-
merged_record.delete(
|
152
|
-
merged_record.delete(
|
200
|
+
merged_record.delete(@partial_message_field)
|
201
|
+
merged_record.delete(@partial_id_field)
|
202
|
+
merged_record.delete(@partial_ordinal_field)
|
203
|
+
merged_record.delete(@partial_last_field)
|
153
204
|
end
|
205
|
+
when :partial_cri
|
206
|
+
merged_record.delete(@partial_cri_logtag_key) unless @keep_partial_key
|
207
|
+
merged_record.delete(@partial_cri_stream_key)
|
154
208
|
end
|
155
209
|
new_es.add(time, merged_record)
|
156
210
|
end
|
@@ -175,9 +229,9 @@ module Fluent::Plugin
|
|
175
229
|
def process(tag, time, record)
|
176
230
|
if @mode == :partial_metadata
|
177
231
|
if @stream_identity_key
|
178
|
-
stream_identity = %Q(#{tag}:#{record[@stream_identity_key]}#{record[
|
232
|
+
stream_identity = %Q(#{tag}:#{record[@stream_identity_key]}#{record[@partial_id_field]})
|
179
233
|
else
|
180
|
-
stream_identity = %Q(#{tag}:#{record[
|
234
|
+
stream_identity = %Q(#{tag}:#{record[@partial_id_field]})
|
181
235
|
end
|
182
236
|
else
|
183
237
|
if @stream_identity_key
|
@@ -196,6 +250,8 @@ module Fluent::Plugin
|
|
196
250
|
process_partial(stream_identity, tag, time, record)
|
197
251
|
when :partial_metadata
|
198
252
|
process_partial_metadata(stream_identity, tag, time, record)
|
253
|
+
when :partial_cri
|
254
|
+
process_partial_cri(stream_identity, tag, time, record)
|
199
255
|
when :regexp
|
200
256
|
process_regexp(stream_identity, tag, time, record)
|
201
257
|
end
|
@@ -224,10 +280,22 @@ module Fluent::Plugin
|
|
224
280
|
new_es
|
225
281
|
end
|
226
282
|
|
283
|
+
def process_partial_cri(stream_identity, tag, time, record)
|
284
|
+
new_es = Fluent::MultiEventStream.new
|
285
|
+
@buffer[stream_identity] << [tag, time, record]
|
286
|
+
if record[@partial_cri_logtag_key].split(@partial_logtag_delimiter)[0] == @partial_logtag_full
|
287
|
+
new_time, new_record = flush_buffer(stream_identity)
|
288
|
+
time = new_time if @use_first_timestamp
|
289
|
+
new_record.delete(@partial_cri_logtag_key)
|
290
|
+
new_es.add(time, new_record)
|
291
|
+
end
|
292
|
+
new_es
|
293
|
+
end
|
294
|
+
|
227
295
|
def process_partial_metadata(stream_identity, tag, time, record)
|
228
296
|
new_es = Fluent::MultiEventStream.new
|
229
297
|
@buffer[stream_identity] << [tag, time, record]
|
230
|
-
if record[
|
298
|
+
if record[@partial_last_field] == "true"
|
231
299
|
new_time, new_record = flush_buffer(stream_identity)
|
232
300
|
time = new_time if @use_first_timestamp
|
233
301
|
new_record.delete(@partial_key)
|
@@ -288,16 +356,16 @@ module Fluent::Plugin
|
|
288
356
|
end
|
289
357
|
|
290
358
|
def firstline?(text)
|
291
|
-
@multiline_start_regexp &&
|
359
|
+
@multiline_start_regexp && @multiline_start_regexp.match?(text)
|
292
360
|
end
|
293
361
|
|
294
362
|
def lastline?(text)
|
295
|
-
@multiline_end_regexp &&
|
363
|
+
@multiline_end_regexp && @multiline_end_regexp.match?(text)
|
296
364
|
end
|
297
365
|
|
298
366
|
def continuous_line?(text)
|
299
367
|
if @continuous_line_regexp
|
300
|
-
|
368
|
+
@continuous_line_regexp.match?(text)
|
301
369
|
else
|
302
370
|
true
|
303
371
|
end
|
@@ -306,7 +374,7 @@ module Fluent::Plugin
|
|
306
374
|
def flush_buffer(stream_identity, new_element = nil)
|
307
375
|
lines = if @mode == :partial_metadata
|
308
376
|
@buffer[stream_identity]
|
309
|
-
.sort_by {|_tag, _time, record| record[
|
377
|
+
.sort_by {|_tag, _time, record| record[@partial_ordinal_field].to_i }
|
310
378
|
.map {|_tag, _time, record| record[@key] }
|
311
379
|
else
|
312
380
|
@buffer[stream_identity].map {|_tag, _time, record| record[@key] }
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-concat
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kenji Okimoto
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-05-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -100,6 +100,20 @@ dependencies:
|
|
100
100
|
- - ">="
|
101
101
|
- !ruby/object:Gem::Version
|
102
102
|
version: '0'
|
103
|
+
- !ruby/object:Gem::Dependency
|
104
|
+
name: webrick
|
105
|
+
requirement: !ruby/object:Gem::Requirement
|
106
|
+
requirements:
|
107
|
+
- - ">="
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: '0'
|
110
|
+
type: :development
|
111
|
+
prerelease: false
|
112
|
+
version_requirements: !ruby/object:Gem::Requirement
|
113
|
+
requirements:
|
114
|
+
- - ">="
|
115
|
+
- !ruby/object:Gem::Version
|
116
|
+
version: '0'
|
103
117
|
description: Fluentd Filter plugin to concat multiple event messages
|
104
118
|
email:
|
105
119
|
- okimoto@clear-code.com
|
@@ -108,9 +122,10 @@ extensions: []
|
|
108
122
|
extra_rdoc_files: []
|
109
123
|
files:
|
110
124
|
- ".github/ISSUE_TEMPLATE.md"
|
125
|
+
- ".github/workflows/linux.yml"
|
126
|
+
- ".github/workflows/windows.yml"
|
111
127
|
- ".gitignore"
|
112
128
|
- ".rubocop.yml"
|
113
|
-
- ".travis.yml"
|
114
129
|
- Appraisals
|
115
130
|
- Gemfile
|
116
131
|
- LICENSE.txt
|
@@ -123,7 +138,7 @@ homepage: https://github.com/fluent-plugins-nursery/fluent-plugin-concat
|
|
123
138
|
licenses:
|
124
139
|
- MIT
|
125
140
|
metadata: {}
|
126
|
-
post_install_message:
|
141
|
+
post_install_message:
|
127
142
|
rdoc_options: []
|
128
143
|
require_paths:
|
129
144
|
- lib
|
@@ -138,8 +153,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
138
153
|
- !ruby/object:Gem::Version
|
139
154
|
version: '0'
|
140
155
|
requirements: []
|
141
|
-
rubygems_version: 3.
|
142
|
-
signing_key:
|
156
|
+
rubygems_version: 3.2.3
|
157
|
+
signing_key:
|
143
158
|
specification_version: 4
|
144
159
|
summary: Fluentd Filter plugin to concat multiple event messages
|
145
160
|
test_files: []
|
data/.travis.yml
DELETED
@@ -1,19 +0,0 @@
|
|
1
|
-
sudo: false
|
2
|
-
language: ruby
|
3
|
-
|
4
|
-
notifications:
|
5
|
-
webhooks:
|
6
|
-
urls:
|
7
|
-
secure: NhOBulc3nFAnJrCr9G0/3IGd1pOb/Zk0Wyk9Jqrf7PHnI4mZOIgttGnewZx1sG96NWrs0Braue5vXwFBTrJ60J6EW3+Q33XzhaZR+Q95jEb+5vcdIP69TYs0TgOIiOouo69Y4rIE1ANYuF/1cBfCKxGulpOIKUCW5LYPkyUEwQy+7VccULctCYCha2OYHQau5SUITCQQqZMfYCvZ+9FRubhR+OiiAnwduuzixKm5b2sa2K4lCKy2vsz3JZtjvQ6/7WM5JsipErtuUY2fDlrWSr/9SehRu8LCD119FjV/HcaAffFMokX+8vJm6RLGcyx/0mAejEqaBofqf9vi1KQJKsKuEe8N9IUpoLrl36RYVfDDRNDqm0pSkrz/KaNxSSe5sDNsCFY+uEFhtiPMHJiSlyEA+BVqnUeoM9pSWp3pcTsIHAjq5VGfQ6u9IooUODCASBlOsLuPu+FYEWG3dC3oiOLEIqQ+WywJg2P7I4dJNhyivti5j61czipTGj28ReFiydp5macKabkvAFBvOW8q07QBO4cvjixhr8hiUgMEVdo5y5dqNhLvS182by3OpmtaSQfpuI8QXytiN9629lWzB7qu0Q7Pji/5wSqEuCdaEHjF5uCVYLFE2LaGJdlTLBg9n0vFfFpJXnP1q0nehq2qRs30pNcgkcCKfEIPJgFgCac=
|
8
|
-
on_success: change
|
9
|
-
on_failure: always
|
10
|
-
on_start: never
|
11
|
-
|
12
|
-
matrix:
|
13
|
-
include:
|
14
|
-
- rvm: 2.4.5
|
15
|
-
gemfile: Gemfile
|
16
|
-
- rvm: 2.5.5
|
17
|
-
gemfile: Gemfile
|
18
|
-
- rvm: 2.6.2
|
19
|
-
gemfile: Gemfile
|