fluent-plugin-concat 2.5.0 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/ISSUE_TEMPLATE/bug_report.yml +67 -0
- data/.github/ISSUE_TEMPLATE/config.yml +5 -0
- data/.github/ISSUE_TEMPLATE/feature_request.yml +35 -0
- data/.github/workflows/add-to-project.yml +17 -0
- data/.github/workflows/linux.yml +7 -4
- data/.github/workflows/windows.yml +3 -2
- data/NEWS.md +6 -0
- data/README.md +22 -1
- data/fluent-plugin-concat.gemspec +1 -1
- data/lib/fluent/plugin/filter_concat.rb +61 -3
- metadata +7 -7
- data/.github/ISSUE_TEMPLATE.md +0 -19
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 31897cab290a08b88945f1a78571bc4bfac24affae42aa2fb79eee0c6a5df937
|
4
|
+
data.tar.gz: 689b42a5ec25b1400d51af0c4f59961023e4fd01dd9b56c2d87b6ff83a2c3ddd
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 501acf92258e36fbb603c27bd4a861fff446304b33b3ccb425fb2fa281bc4b6ab05c357cc051196b17da66fd57c4de4e2e7d55999b194adfc3ce4dac9694b130
|
7
|
+
data.tar.gz: 369632414f573689ca37ea34c2f17b88a74476dab41ea7edfe9d8ee394d42b20fed5d27bef7306489729b7eb9ee96f8a1429adaca88bf1b0066d7ccbcc25b22a
|
@@ -0,0 +1,67 @@
|
|
1
|
+
name: Bug Report
|
2
|
+
description: Create a report with a procedure for reproducing the bug
|
3
|
+
labels: "waiting-for-triage"
|
4
|
+
body:
|
5
|
+
- type: textarea
|
6
|
+
id: description
|
7
|
+
attributes:
|
8
|
+
label: Describe the bug
|
9
|
+
description: A clear and concise description of what the bug is
|
10
|
+
validations:
|
11
|
+
required: true
|
12
|
+
- type: textarea
|
13
|
+
id: reproduce
|
14
|
+
attributes:
|
15
|
+
label: To Reproduce
|
16
|
+
description: Steps to reproduce the behavior
|
17
|
+
validations:
|
18
|
+
required: true
|
19
|
+
- type: textarea
|
20
|
+
id: expected
|
21
|
+
attributes:
|
22
|
+
label: Expected behavior
|
23
|
+
description: A clear and concise description of what you expected to happen
|
24
|
+
validations:
|
25
|
+
required: true
|
26
|
+
- type: textarea
|
27
|
+
id: environment
|
28
|
+
attributes:
|
29
|
+
label: Your Environment
|
30
|
+
description: |
|
31
|
+
- Fluentd or its package version: `fluentd --version` (Fluentd, fluent-package) or `td-agent --version` (td-agent)
|
32
|
+
- Operating system: `cat /etc/os-release`
|
33
|
+
- Kernel version: `uname -r`
|
34
|
+
|
35
|
+
Tip: If you hit the problem with older fluent-plugin-concat version, try latest version first.
|
36
|
+
value: |
|
37
|
+
- Fluentd version:
|
38
|
+
- Package version:
|
39
|
+
- Operating system:
|
40
|
+
- Kernel version:
|
41
|
+
render: markdown
|
42
|
+
validations:
|
43
|
+
required: true
|
44
|
+
- type: textarea
|
45
|
+
id: configuration
|
46
|
+
attributes:
|
47
|
+
label: Your Configuration
|
48
|
+
description: |
|
49
|
+
Write your configuration here. Minimum reproducible fluentd.conf is recommended.
|
50
|
+
render: apache
|
51
|
+
validations:
|
52
|
+
required: true
|
53
|
+
- type: textarea
|
54
|
+
id: logs
|
55
|
+
attributes:
|
56
|
+
label: Your Error Log
|
57
|
+
description: Write your ALL error log here
|
58
|
+
render: shell
|
59
|
+
validations:
|
60
|
+
required: true
|
61
|
+
- type: textarea
|
62
|
+
id: addtional-context
|
63
|
+
attributes:
|
64
|
+
label: Additional context
|
65
|
+
description: Add any other context about the problem here.
|
66
|
+
validations:
|
67
|
+
required: false
|
@@ -0,0 +1,35 @@
|
|
1
|
+
name: Feature request
|
2
|
+
description: Suggest an idea for this project
|
3
|
+
labels: ["waiting-for-triage", "enhancement"]
|
4
|
+
body:
|
5
|
+
- type: textarea
|
6
|
+
id: description
|
7
|
+
attributes:
|
8
|
+
label: Is your feature request related to a problem? Please describe.
|
9
|
+
description: |
|
10
|
+
A clear and concise description of what the problem is.
|
11
|
+
Ex. I'm always frustrated when [...]
|
12
|
+
validations:
|
13
|
+
required: true
|
14
|
+
- type: textarea
|
15
|
+
id: solution
|
16
|
+
attributes:
|
17
|
+
label: Describe the solution you'd like
|
18
|
+
description: A clear and concise description of what you want to happen.
|
19
|
+
validations:
|
20
|
+
required: true
|
21
|
+
- type: textarea
|
22
|
+
id: alternative
|
23
|
+
attributes:
|
24
|
+
label: Describe alternatives you've considered
|
25
|
+
description: A clear and concise description of any alternative solutions or features you've considered.
|
26
|
+
validations:
|
27
|
+
required: true
|
28
|
+
- type: textarea
|
29
|
+
id: addtional-context
|
30
|
+
attributes:
|
31
|
+
label: Additional context
|
32
|
+
description: Add any other context or screenshots about the feature request here.
|
33
|
+
validations:
|
34
|
+
required: false
|
35
|
+
|
@@ -0,0 +1,17 @@
|
|
1
|
+
name: Add bugs to fluent project
|
2
|
+
|
3
|
+
on:
|
4
|
+
issues:
|
5
|
+
types:
|
6
|
+
- opened
|
7
|
+
|
8
|
+
jobs:
|
9
|
+
add-to-project:
|
10
|
+
name: Add issue to project
|
11
|
+
runs-on: ubuntu-latest
|
12
|
+
steps:
|
13
|
+
- uses: actions/add-to-project@v1.0.2
|
14
|
+
with:
|
15
|
+
project-url: https://github.com/orgs/fluent/projects/4
|
16
|
+
github-token: ${{ secrets.ADD_TO_PROJECT_PAT }}
|
17
|
+
labeled: waiting-for-triage
|
data/.github/workflows/linux.yml
CHANGED
@@ -9,18 +9,21 @@ jobs:
|
|
9
9
|
fail-fast: false
|
10
10
|
matrix:
|
11
11
|
ruby:
|
12
|
-
- 2.5
|
13
|
-
- 2.6
|
14
12
|
- 2.7
|
15
|
-
- 3.0
|
13
|
+
- '3.0'
|
14
|
+
- 3.1
|
15
|
+
- 3.2
|
16
|
+
- 3.3
|
17
|
+
- 3.4
|
16
18
|
os:
|
17
19
|
- ubuntu-latest
|
18
20
|
name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
|
19
21
|
steps:
|
20
|
-
- uses: actions/checkout@
|
22
|
+
- uses: actions/checkout@v4
|
21
23
|
- uses: ruby/setup-ruby@v1
|
22
24
|
with:
|
23
25
|
ruby-version: ${{ matrix.ruby }}
|
26
|
+
rubygems: latest
|
24
27
|
- name: unit testing
|
25
28
|
env:
|
26
29
|
CI: true
|
@@ -8,15 +8,16 @@ jobs:
|
|
8
8
|
strategy:
|
9
9
|
fail-fast: false
|
10
10
|
matrix:
|
11
|
-
ruby: [ '2.
|
11
|
+
ruby: [ '2.7', '3.0', '3.1', '3.2', '3.3', '3.4' ]
|
12
12
|
os:
|
13
13
|
- windows-latest
|
14
14
|
name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
|
15
15
|
steps:
|
16
|
-
- uses: actions/checkout@
|
16
|
+
- uses: actions/checkout@v4
|
17
17
|
- uses: ruby/setup-ruby@v1
|
18
18
|
with:
|
19
19
|
ruby-version: ${{ matrix.ruby }}
|
20
|
+
rubygems: latest
|
20
21
|
- name: unit testing
|
21
22
|
env:
|
22
23
|
CI: true
|
data/NEWS.md
CHANGED
@@ -6,6 +6,12 @@
|
|
6
6
|
|
7
7
|
* adds `partial_metadata_format` to support Docker 20.10 journald log driver improvements [moby/moby#41407](https://github.com/moby/moby/pull/41407)
|
8
8
|
|
9
|
+
## v2.6.0
|
10
|
+
|
11
|
+
### Improvements
|
12
|
+
|
13
|
+
* Support `buffer_limit_size`, `buffer_overflow_method` to change behavior when buffer overflow. See #68
|
14
|
+
|
9
15
|
## v2.3.0
|
10
16
|
|
11
17
|
### Improvements
|
data/README.md
CHANGED
@@ -27,6 +27,13 @@ Or install it yourself as:
|
|
27
27
|
|
28
28
|
$ gem install fluent-plugin-concat
|
29
29
|
|
30
|
+
## Plugin helpers
|
31
|
+
|
32
|
+
* [timer](https://docs.fluentd.org/v1.0/articles/api-plugin-helper-timer)
|
33
|
+
* [event_emitter](https://docs.fluentd.org/v1.0/articles/api-plugin-helper-event_emitter)
|
34
|
+
|
35
|
+
* See also: [Filter Plugin Overview](https://docs.fluentd.org/filter#overview)
|
36
|
+
|
30
37
|
## Configuration
|
31
38
|
|
32
39
|
### Example
|
@@ -67,7 +74,7 @@ Or install it yourself as:
|
|
67
74
|
|multiline\_end\_regexp|The regexp to match ending of multiline.This is exclusive with `n_lines`|nil|
|
68
75
|
|continuous\_line\_regexp|The regexp to match continuous lines.This is exclusive with `n_lines`|nil|
|
69
76
|
|stream\_identity\_key|The key to determine which stream an event belongs to|nil|
|
70
|
-
|flush\_interval|The number of seconds after which the last received event log will be flushed.If specified 0, wait for next line
|
77
|
+
|flush\_interval|The number of seconds after which the last received event log will be flushed.If specified 0, wait for next line forever|60|
|
71
78
|
|timeout\_label|The label name to handle events caused by timeout|nil|
|
72
79
|
|use\_first\_timestamp|Use timestamp of first record when buffer is flushed|`false`|
|
73
80
|
|partial\_key|The field name that is the reference to concatenate records|nil|
|
@@ -79,6 +86,20 @@ Or install it yourself as:
|
|
79
86
|
|use\_partial\_cri\_logtag|bool (optional)|Use cri log tag to concatenate multiple records||
|
80
87
|
|partial\_cri\_logtag\_key|string (optional)|The key name that is referred to concatenate records on cri log||
|
81
88
|
|partial\_cri\_stream\_key|string (optional)|The key name that is referred to detect stream name on cri log|`stream`|
|
89
|
+
|buffer\_limit\_size|The max size of each buffer|`512000`|
|
90
|
+
|
91
|
+
|parameter|description|available values|default|
|
92
|
+
|---|---|---|---|
|
93
|
+
|buffer\_overflow\_method|The method if overflow buffer|`ignore`, `truncate`, `drop`, `new`|`ignore`|
|
94
|
+
|
95
|
+
* `ignore`
|
96
|
+
* Concatenate the current record in the buffer
|
97
|
+
* `truncate`
|
98
|
+
* Drop the current record and flush the buffer
|
99
|
+
* `drop`
|
100
|
+
* Drop the current record and clear the buffer
|
101
|
+
* `new`
|
102
|
+
* Flush the buffer and store the current record in next buffer
|
82
103
|
|
83
104
|
## Usage
|
84
105
|
|
@@ -22,7 +22,7 @@ module Fluent::Plugin
|
|
22
22
|
config_param :stream_identity_key, :string, default: nil
|
23
23
|
desc "The interval between data flushes, 0 means disable timeout"
|
24
24
|
config_param :flush_interval, :time, default: 60
|
25
|
-
desc "The label name to handle timeout"
|
25
|
+
desc "The label name to handle events caused by timeout"
|
26
26
|
config_param :timeout_label, :string, default: nil
|
27
27
|
desc "Use timestamp of first record when buffer is flushed"
|
28
28
|
config_param :use_first_timestamp, :bool, default: false
|
@@ -44,6 +44,10 @@ module Fluent::Plugin
|
|
44
44
|
config_param :partial_cri_logtag_key, :string, default: nil
|
45
45
|
desc "The key name that is referred to detect stream name on cri log"
|
46
46
|
config_param :partial_cri_stream_key, :string, default: "stream"
|
47
|
+
desc "The max size of each buffer"
|
48
|
+
config_param :buffer_limit_size, :size, default: 500 * 1024 # 500k
|
49
|
+
desc "The method if overflow buffer"
|
50
|
+
config_param :buffer_overflow_method, :enum, list: [:ignore, :truncate, :drop, :new], default: :ignore
|
47
51
|
|
48
52
|
class TimeoutError < StandardError
|
49
53
|
end
|
@@ -52,6 +56,7 @@ module Fluent::Plugin
|
|
52
56
|
super
|
53
57
|
|
54
58
|
@buffer = Hash.new {|h, k| h[k] = [] }
|
59
|
+
@buffer_size = Hash.new(0)
|
55
60
|
@timeout_map_mutex = Thread::Mutex.new
|
56
61
|
@timeout_map_mutex.synchronize do
|
57
62
|
@timeout_map = Hash.new {|h, k| h[k] = Fluent::Engine.now }
|
@@ -270,13 +275,39 @@ module Fluent::Plugin
|
|
270
275
|
|
271
276
|
def process_partial(stream_identity, tag, time, record)
|
272
277
|
new_es = Fluent::MultiEventStream.new
|
273
|
-
|
274
|
-
|
278
|
+
force_flush = false
|
279
|
+
if overflow?(stream_identity, record)
|
280
|
+
force_flush = case @buffer_overflow_method
|
281
|
+
when :ignore
|
282
|
+
@buffer[stream_identity] << [tag, time, record]
|
283
|
+
false
|
284
|
+
when :truncate
|
285
|
+
true
|
286
|
+
when :drop
|
287
|
+
@buffer[stream_identity] = []
|
288
|
+
false
|
289
|
+
when :new
|
290
|
+
true
|
291
|
+
end
|
292
|
+
else
|
293
|
+
@buffer[stream_identity] << [tag, time, record]
|
294
|
+
end
|
295
|
+
if force_flush || @partial_value != record[@partial_key]
|
275
296
|
new_time, new_record = flush_buffer(stream_identity)
|
276
297
|
time = new_time if @use_first_timestamp
|
277
298
|
new_record.delete(@partial_key)
|
278
299
|
new_es.add(time, new_record)
|
279
300
|
end
|
301
|
+
if force_flush && @buffer_overflow_method == :new
|
302
|
+
@buffer[stream_identity] << [tag, time, record]
|
303
|
+
@buffer_size[stream_identity] = record.keys.sum(&:bytesize) + record.values.sum(&:bytesize)
|
304
|
+
if @partial_value != record[@partial_key]
|
305
|
+
new_time, new_record = flush_buffer(stream_identity)
|
306
|
+
time = new_time if @use_first_timestamp
|
307
|
+
new_record.delete(@partial_key)
|
308
|
+
new_es.add(time, new_record)
|
309
|
+
end
|
310
|
+
end
|
280
311
|
new_es
|
281
312
|
end
|
282
313
|
|
@@ -371,6 +402,17 @@ module Fluent::Plugin
|
|
371
402
|
end
|
372
403
|
end
|
373
404
|
|
405
|
+
def overflow?(stream_identity, record)
|
406
|
+
size = record.keys.sum(&:bytesize) + record.values.sum(&:bytesize)
|
407
|
+
if @buffer_size[stream_identity] + size > @buffer_limit_size
|
408
|
+
@buffer_size[stream_identity] = 0
|
409
|
+
true
|
410
|
+
else
|
411
|
+
@buffer_size[stream_identity] += size
|
412
|
+
false
|
413
|
+
end
|
414
|
+
end
|
415
|
+
|
374
416
|
def flush_buffer(stream_identity, new_element = nil)
|
375
417
|
lines = if @mode == :partial_metadata
|
376
418
|
@buffer[stream_identity]
|
@@ -385,6 +427,7 @@ module Fluent::Plugin
|
|
385
427
|
}
|
386
428
|
@buffer[stream_identity] = []
|
387
429
|
@buffer[stream_identity] << new_element if new_element
|
430
|
+
@buffer_size[stream_identity] = 0
|
388
431
|
[time, first_record.merge(new_record)]
|
389
432
|
end
|
390
433
|
|
@@ -434,3 +477,18 @@ module Fluent::Plugin
|
|
434
477
|
end
|
435
478
|
end
|
436
479
|
end
|
480
|
+
|
481
|
+
class Array
|
482
|
+
# Support Ruby 2.3 or earlier
|
483
|
+
unless [].respond_to?(:sum)
|
484
|
+
def sum
|
485
|
+
inject(0) do |memo, value|
|
486
|
+
if block_given?
|
487
|
+
memo + yield(value)
|
488
|
+
else
|
489
|
+
memo + value
|
490
|
+
end
|
491
|
+
end
|
492
|
+
end
|
493
|
+
end
|
494
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-concat
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kenji Okimoto
|
8
|
-
autorequire:
|
9
8
|
bindir: bin
|
10
9
|
cert_chain: []
|
11
|
-
date:
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
12
11
|
dependencies:
|
13
12
|
- !ruby/object:Gem::Dependency
|
14
13
|
name: fluentd
|
@@ -121,7 +120,10 @@ executables: []
|
|
121
120
|
extensions: []
|
122
121
|
extra_rdoc_files: []
|
123
122
|
files:
|
124
|
-
- ".github/ISSUE_TEMPLATE.
|
123
|
+
- ".github/ISSUE_TEMPLATE/bug_report.yml"
|
124
|
+
- ".github/ISSUE_TEMPLATE/config.yml"
|
125
|
+
- ".github/ISSUE_TEMPLATE/feature_request.yml"
|
126
|
+
- ".github/workflows/add-to-project.yml"
|
125
127
|
- ".github/workflows/linux.yml"
|
126
128
|
- ".github/workflows/windows.yml"
|
127
129
|
- ".gitignore"
|
@@ -138,7 +140,6 @@ homepage: https://github.com/fluent-plugins-nursery/fluent-plugin-concat
|
|
138
140
|
licenses:
|
139
141
|
- MIT
|
140
142
|
metadata: {}
|
141
|
-
post_install_message:
|
142
143
|
rdoc_options: []
|
143
144
|
require_paths:
|
144
145
|
- lib
|
@@ -153,8 +154,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
153
154
|
- !ruby/object:Gem::Version
|
154
155
|
version: '0'
|
155
156
|
requirements: []
|
156
|
-
rubygems_version: 3.
|
157
|
-
signing_key:
|
157
|
+
rubygems_version: 3.6.8
|
158
158
|
specification_version: 4
|
159
159
|
summary: Fluentd Filter plugin to concat multiple event messages
|
160
160
|
test_files: []
|
data/.github/ISSUE_TEMPLATE.md
DELETED
@@ -1,19 +0,0 @@
|
|
1
|
-
#### Problem
|
2
|
-
|
3
|
-
...
|
4
|
-
|
5
|
-
#### Steps to replicate
|
6
|
-
|
7
|
-
Provide example config and message
|
8
|
-
|
9
|
-
#### Expected Behavior
|
10
|
-
|
11
|
-
...
|
12
|
-
|
13
|
-
#### Your environment
|
14
|
-
|
15
|
-
* OS version
|
16
|
-
* paste result of ``fluentd --version`` or ``td-agent --version``
|
17
|
-
* plugin version
|
18
|
-
* paste boot log of fluentd or td-agent
|
19
|
-
* paste result of ``fluent-gem list``, ``td-agent-gem list`` or your Gemfile.lock
|