fluent-plugin-concat 2.5.0 → 2.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 0fa8ad39f08cb1de3cf3137fd5c504e285cc82107b10b3faf9455a8b08c002ca
4
- data.tar.gz: d4f0fbfa8974961fcb2e4e7128e8d823c5a607a4087fc063d936c90896e267c9
3
+ metadata.gz: 31897cab290a08b88945f1a78571bc4bfac24affae42aa2fb79eee0c6a5df937
4
+ data.tar.gz: 689b42a5ec25b1400d51af0c4f59961023e4fd01dd9b56c2d87b6ff83a2c3ddd
5
5
  SHA512:
6
- metadata.gz: 173c3b6dcb69212174c1bca085e44b26da8137c0c4cd935e622b54ba019150a720e1eb863a4c3aff7503b8ee752aa992e694e321423c0cc2e3d846ccf2d79215
7
- data.tar.gz: 11f274a4e8e4ae32f8e8f91d4eabd7ba72d98c794ff202b5e111b475024b40cc61fae65698eaebf645cc161fa8b3a9c632ff1f6de778fa369afe9fa57459c126
6
+ metadata.gz: 501acf92258e36fbb603c27bd4a861fff446304b33b3ccb425fb2fa281bc4b6ab05c357cc051196b17da66fd57c4de4e2e7d55999b194adfc3ce4dac9694b130
7
+ data.tar.gz: 369632414f573689ca37ea34c2f17b88a74476dab41ea7edfe9d8ee394d42b20fed5d27bef7306489729b7eb9ee96f8a1429adaca88bf1b0066d7ccbcc25b22a
@@ -0,0 +1,67 @@
1
+ name: Bug Report
2
+ description: Create a report with a procedure for reproducing the bug
3
+ labels: "waiting-for-triage"
4
+ body:
5
+ - type: textarea
6
+ id: description
7
+ attributes:
8
+ label: Describe the bug
9
+ description: A clear and concise description of what the bug is
10
+ validations:
11
+ required: true
12
+ - type: textarea
13
+ id: reproduce
14
+ attributes:
15
+ label: To Reproduce
16
+ description: Steps to reproduce the behavior
17
+ validations:
18
+ required: true
19
+ - type: textarea
20
+ id: expected
21
+ attributes:
22
+ label: Expected behavior
23
+ description: A clear and concise description of what you expected to happen
24
+ validations:
25
+ required: true
26
+ - type: textarea
27
+ id: environment
28
+ attributes:
29
+ label: Your Environment
30
+ description: |
31
+ - Fluentd or its package version: `fluentd --version` (Fluentd, fluent-package) or `td-agent --version` (td-agent)
32
+ - Operating system: `cat /etc/os-release`
33
+ - Kernel version: `uname -r`
34
+
35
+ Tip: If you hit the problem with older fluent-plugin-concat version, try latest version first.
36
+ value: |
37
+ - Fluentd version:
38
+ - Package version:
39
+ - Operating system:
40
+ - Kernel version:
41
+ render: markdown
42
+ validations:
43
+ required: true
44
+ - type: textarea
45
+ id: configuration
46
+ attributes:
47
+ label: Your Configuration
48
+ description: |
49
+ Write your configuration here. Minimum reproducible fluentd.conf is recommended.
50
+ render: apache
51
+ validations:
52
+ required: true
53
+ - type: textarea
54
+ id: logs
55
+ attributes:
56
+ label: Your Error Log
57
+ description: Write your ALL error log here
58
+ render: shell
59
+ validations:
60
+ required: true
61
+ - type: textarea
62
+ id: addtional-context
63
+ attributes:
64
+ label: Additional context
65
+ description: Add any other context about the problem here.
66
+ validations:
67
+ required: false
@@ -0,0 +1,5 @@
1
+ blank_issues_enabled: false
2
+ contact_links:
3
+ - name: Ask a Question
4
+ url: https://github.com/fluent/fluentd/discussions
5
+ about: I have questions about Fluentd and plugins. Please ask and answer questions at https://github.com/fluent/fluentd/discussions
@@ -0,0 +1,35 @@
1
+ name: Feature request
2
+ description: Suggest an idea for this project
3
+ labels: ["waiting-for-triage", "enhancement"]
4
+ body:
5
+ - type: textarea
6
+ id: description
7
+ attributes:
8
+ label: Is your feature request related to a problem? Please describe.
9
+ description: |
10
+ A clear and concise description of what the problem is.
11
+ Ex. I'm always frustrated when [...]
12
+ validations:
13
+ required: true
14
+ - type: textarea
15
+ id: solution
16
+ attributes:
17
+ label: Describe the solution you'd like
18
+ description: A clear and concise description of what you want to happen.
19
+ validations:
20
+ required: true
21
+ - type: textarea
22
+ id: alternative
23
+ attributes:
24
+ label: Describe alternatives you've considered
25
+ description: A clear and concise description of any alternative solutions or features you've considered.
26
+ validations:
27
+ required: true
28
+ - type: textarea
29
+ id: addtional-context
30
+ attributes:
31
+ label: Additional context
32
+ description: Add any other context or screenshots about the feature request here.
33
+ validations:
34
+ required: false
35
+
@@ -0,0 +1,17 @@
1
+ name: Add bugs to fluent project
2
+
3
+ on:
4
+ issues:
5
+ types:
6
+ - opened
7
+
8
+ jobs:
9
+ add-to-project:
10
+ name: Add issue to project
11
+ runs-on: ubuntu-latest
12
+ steps:
13
+ - uses: actions/add-to-project@v1.0.2
14
+ with:
15
+ project-url: https://github.com/orgs/fluent/projects/4
16
+ github-token: ${{ secrets.ADD_TO_PROJECT_PAT }}
17
+ labeled: waiting-for-triage
@@ -9,18 +9,21 @@ jobs:
9
9
  fail-fast: false
10
10
  matrix:
11
11
  ruby:
12
- - 2.5
13
- - 2.6
14
12
  - 2.7
15
- - 3.0
13
+ - '3.0'
14
+ - 3.1
15
+ - 3.2
16
+ - 3.3
17
+ - 3.4
16
18
  os:
17
19
  - ubuntu-latest
18
20
  name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
19
21
  steps:
20
- - uses: actions/checkout@v2
22
+ - uses: actions/checkout@v4
21
23
  - uses: ruby/setup-ruby@v1
22
24
  with:
23
25
  ruby-version: ${{ matrix.ruby }}
26
+ rubygems: latest
24
27
  - name: unit testing
25
28
  env:
26
29
  CI: true
@@ -8,15 +8,16 @@ jobs:
8
8
  strategy:
9
9
  fail-fast: false
10
10
  matrix:
11
- ruby: [ '2.5', '2.6', '2.7', '3.0' ]
11
+ ruby: [ '2.7', '3.0', '3.1', '3.2', '3.3', '3.4' ]
12
12
  os:
13
13
  - windows-latest
14
14
  name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
15
15
  steps:
16
- - uses: actions/checkout@v2
16
+ - uses: actions/checkout@v4
17
17
  - uses: ruby/setup-ruby@v1
18
18
  with:
19
19
  ruby-version: ${{ matrix.ruby }}
20
+ rubygems: latest
20
21
  - name: unit testing
21
22
  env:
22
23
  CI: true
data/NEWS.md CHANGED
@@ -6,6 +6,12 @@
6
6
 
7
7
  * adds `partial_metadata_format` to support Docker 20.10 journald log driver improvements [moby/moby#41407](https://github.com/moby/moby/pull/41407)
8
8
 
9
+ ## v2.6.0
10
+
11
+ ### Improvements
12
+
13
+ * Support `buffer_limit_size`, `buffer_overflow_method` to change behavior when buffer overflow. See #68
14
+
9
15
  ## v2.3.0
10
16
 
11
17
  ### Improvements
data/README.md CHANGED
@@ -27,6 +27,13 @@ Or install it yourself as:
27
27
 
28
28
  $ gem install fluent-plugin-concat
29
29
 
30
+ ## Plugin helpers
31
+
32
+ * [timer](https://docs.fluentd.org/v1.0/articles/api-plugin-helper-timer)
33
+ * [event_emitter](https://docs.fluentd.org/v1.0/articles/api-plugin-helper-event_emitter)
34
+
35
+ * See also: [Filter Plugin Overview](https://docs.fluentd.org/filter#overview)
36
+
30
37
  ## Configuration
31
38
 
32
39
  ### Example
@@ -67,7 +74,7 @@ Or install it yourself as:
67
74
  |multiline\_end\_regexp|The regexp to match ending of multiline.This is exclusive with `n_lines`|nil|
68
75
  |continuous\_line\_regexp|The regexp to match continuous lines.This is exclusive with `n_lines`|nil|
69
76
  |stream\_identity\_key|The key to determine which stream an event belongs to|nil|
70
- |flush\_interval|The number of seconds after which the last received event log will be flushed.If specified 0, wait for next line foreverr|60|
77
+ |flush\_interval|The number of seconds after which the last received event log will be flushed.If specified 0, wait for next line forever|60|
71
78
  |timeout\_label|The label name to handle events caused by timeout|nil|
72
79
  |use\_first\_timestamp|Use timestamp of first record when buffer is flushed|`false`|
73
80
  |partial\_key|The field name that is the reference to concatenate records|nil|
@@ -79,6 +86,20 @@ Or install it yourself as:
79
86
  |use\_partial\_cri\_logtag|bool (optional)|Use cri log tag to concatenate multiple records||
80
87
  |partial\_cri\_logtag\_key|string (optional)|The key name that is referred to concatenate records on cri log||
81
88
  |partial\_cri\_stream\_key|string (optional)|The key name that is referred to detect stream name on cri log|`stream`|
89
+ |buffer\_limit\_size|The max size of each buffer|`512000`|
90
+
91
+ |parameter|description|available values|default|
92
+ |---|---|---|---|
93
+ |buffer\_overflow\_method|The method if overflow buffer|`ignore`, `truncate`, `drop`, `new`|`ignore`|
94
+
95
+ * `ignore`
96
+ * Concatenate the current record in the buffer
97
+ * `truncate`
98
+ * Drop the current record and flush the buffer
99
+ * `drop`
100
+ * Drop the current record and clear the buffer
101
+ * `new`
102
+ * Flush the buffer and store the current record in next buffer
82
103
 
83
104
  ## Usage
84
105
 
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |spec|
6
6
  spec.name = "fluent-plugin-concat"
7
- spec.version = "2.5.0"
7
+ spec.version = "2.6.0"
8
8
  spec.authors = ["Kenji Okimoto"]
9
9
  spec.email = ["okimoto@clear-code.com"]
10
10
 
@@ -22,7 +22,7 @@ module Fluent::Plugin
22
22
  config_param :stream_identity_key, :string, default: nil
23
23
  desc "The interval between data flushes, 0 means disable timeout"
24
24
  config_param :flush_interval, :time, default: 60
25
- desc "The label name to handle timeout"
25
+ desc "The label name to handle events caused by timeout"
26
26
  config_param :timeout_label, :string, default: nil
27
27
  desc "Use timestamp of first record when buffer is flushed"
28
28
  config_param :use_first_timestamp, :bool, default: false
@@ -44,6 +44,10 @@ module Fluent::Plugin
44
44
  config_param :partial_cri_logtag_key, :string, default: nil
45
45
  desc "The key name that is referred to detect stream name on cri log"
46
46
  config_param :partial_cri_stream_key, :string, default: "stream"
47
+ desc "The max size of each buffer"
48
+ config_param :buffer_limit_size, :size, default: 500 * 1024 # 500k
49
+ desc "The method if overflow buffer"
50
+ config_param :buffer_overflow_method, :enum, list: [:ignore, :truncate, :drop, :new], default: :ignore
47
51
 
48
52
  class TimeoutError < StandardError
49
53
  end
@@ -52,6 +56,7 @@ module Fluent::Plugin
52
56
  super
53
57
 
54
58
  @buffer = Hash.new {|h, k| h[k] = [] }
59
+ @buffer_size = Hash.new(0)
55
60
  @timeout_map_mutex = Thread::Mutex.new
56
61
  @timeout_map_mutex.synchronize do
57
62
  @timeout_map = Hash.new {|h, k| h[k] = Fluent::Engine.now }
@@ -270,13 +275,39 @@ module Fluent::Plugin
270
275
 
271
276
  def process_partial(stream_identity, tag, time, record)
272
277
  new_es = Fluent::MultiEventStream.new
273
- @buffer[stream_identity] << [tag, time, record]
274
- unless @partial_value == record[@partial_key]
278
+ force_flush = false
279
+ if overflow?(stream_identity, record)
280
+ force_flush = case @buffer_overflow_method
281
+ when :ignore
282
+ @buffer[stream_identity] << [tag, time, record]
283
+ false
284
+ when :truncate
285
+ true
286
+ when :drop
287
+ @buffer[stream_identity] = []
288
+ false
289
+ when :new
290
+ true
291
+ end
292
+ else
293
+ @buffer[stream_identity] << [tag, time, record]
294
+ end
295
+ if force_flush || @partial_value != record[@partial_key]
275
296
  new_time, new_record = flush_buffer(stream_identity)
276
297
  time = new_time if @use_first_timestamp
277
298
  new_record.delete(@partial_key)
278
299
  new_es.add(time, new_record)
279
300
  end
301
+ if force_flush && @buffer_overflow_method == :new
302
+ @buffer[stream_identity] << [tag, time, record]
303
+ @buffer_size[stream_identity] = record.keys.sum(&:bytesize) + record.values.sum(&:bytesize)
304
+ if @partial_value != record[@partial_key]
305
+ new_time, new_record = flush_buffer(stream_identity)
306
+ time = new_time if @use_first_timestamp
307
+ new_record.delete(@partial_key)
308
+ new_es.add(time, new_record)
309
+ end
310
+ end
280
311
  new_es
281
312
  end
282
313
 
@@ -371,6 +402,17 @@ module Fluent::Plugin
371
402
  end
372
403
  end
373
404
 
405
+ def overflow?(stream_identity, record)
406
+ size = record.keys.sum(&:bytesize) + record.values.sum(&:bytesize)
407
+ if @buffer_size[stream_identity] + size > @buffer_limit_size
408
+ @buffer_size[stream_identity] = 0
409
+ true
410
+ else
411
+ @buffer_size[stream_identity] += size
412
+ false
413
+ end
414
+ end
415
+
374
416
  def flush_buffer(stream_identity, new_element = nil)
375
417
  lines = if @mode == :partial_metadata
376
418
  @buffer[stream_identity]
@@ -385,6 +427,7 @@ module Fluent::Plugin
385
427
  }
386
428
  @buffer[stream_identity] = []
387
429
  @buffer[stream_identity] << new_element if new_element
430
+ @buffer_size[stream_identity] = 0
388
431
  [time, first_record.merge(new_record)]
389
432
  end
390
433
 
@@ -434,3 +477,18 @@ module Fluent::Plugin
434
477
  end
435
478
  end
436
479
  end
480
+
481
+ class Array
482
+ # Support Ruby 2.3 or earlier
483
+ unless [].respond_to?(:sum)
484
+ def sum
485
+ inject(0) do |memo, value|
486
+ if block_given?
487
+ memo + yield(value)
488
+ else
489
+ memo + value
490
+ end
491
+ end
492
+ end
493
+ end
494
+ end
metadata CHANGED
@@ -1,14 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-concat
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.5.0
4
+ version: 2.6.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Kenji Okimoto
8
- autorequire:
9
8
  bindir: bin
10
9
  cert_chain: []
11
- date: 2021-05-25 00:00:00.000000000 Z
10
+ date: 1980-01-02 00:00:00.000000000 Z
12
11
  dependencies:
13
12
  - !ruby/object:Gem::Dependency
14
13
  name: fluentd
@@ -121,7 +120,10 @@ executables: []
121
120
  extensions: []
122
121
  extra_rdoc_files: []
123
122
  files:
124
- - ".github/ISSUE_TEMPLATE.md"
123
+ - ".github/ISSUE_TEMPLATE/bug_report.yml"
124
+ - ".github/ISSUE_TEMPLATE/config.yml"
125
+ - ".github/ISSUE_TEMPLATE/feature_request.yml"
126
+ - ".github/workflows/add-to-project.yml"
125
127
  - ".github/workflows/linux.yml"
126
128
  - ".github/workflows/windows.yml"
127
129
  - ".gitignore"
@@ -138,7 +140,6 @@ homepage: https://github.com/fluent-plugins-nursery/fluent-plugin-concat
138
140
  licenses:
139
141
  - MIT
140
142
  metadata: {}
141
- post_install_message:
142
143
  rdoc_options: []
143
144
  require_paths:
144
145
  - lib
@@ -153,8 +154,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
153
154
  - !ruby/object:Gem::Version
154
155
  version: '0'
155
156
  requirements: []
156
- rubygems_version: 3.2.3
157
- signing_key:
157
+ rubygems_version: 3.6.8
158
158
  specification_version: 4
159
159
  summary: Fluentd Filter plugin to concat multiple event messages
160
160
  test_files: []
@@ -1,19 +0,0 @@
1
- #### Problem
2
-
3
- ...
4
-
5
- #### Steps to replicate
6
-
7
- Provide example config and message
8
-
9
- #### Expected Behavior
10
-
11
- ...
12
-
13
- #### Your environment
14
-
15
- * OS version
16
- * paste result of ``fluentd --version`` or ``td-agent --version``
17
- * plugin version
18
- * paste boot log of fluentd or td-agent
19
- * paste result of ``fluent-gem list``, ``td-agent-gem list`` or your Gemfile.lock