fluent-plugin-concat 2.5.0 → 2.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0fa8ad39f08cb1de3cf3137fd5c504e285cc82107b10b3faf9455a8b08c002ca
-  data.tar.gz: d4f0fbfa8974961fcb2e4e7128e8d823c5a607a4087fc063d936c90896e267c9
+  metadata.gz: 31897cab290a08b88945f1a78571bc4bfac24affae42aa2fb79eee0c6a5df937
+  data.tar.gz: 689b42a5ec25b1400d51af0c4f59961023e4fd01dd9b56c2d87b6ff83a2c3ddd
 SHA512:
-  metadata.gz: 173c3b6dcb69212174c1bca085e44b26da8137c0c4cd935e622b54ba019150a720e1eb863a4c3aff7503b8ee752aa992e694e321423c0cc2e3d846ccf2d79215
-  data.tar.gz: 11f274a4e8e4ae32f8e8f91d4eabd7ba72d98c794ff202b5e111b475024b40cc61fae65698eaebf645cc161fa8b3a9c632ff1f6de778fa369afe9fa57459c126
+  metadata.gz: 501acf92258e36fbb603c27bd4a861fff446304b33b3ccb425fb2fa281bc4b6ab05c357cc051196b17da66fd57c4de4e2e7d55999b194adfc3ce4dac9694b130
+  data.tar.gz: 369632414f573689ca37ea34c2f17b88a74476dab41ea7edfe9d8ee394d42b20fed5d27bef7306489729b7eb9ee96f8a1429adaca88bf1b0066d7ccbcc25b22a

data/.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,67 @@
+name: Bug Report
+description: Create a report with a procedure for reproducing the bug
+labels: "waiting-for-triage"
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: Describe the bug
+      description: A clear and concise description of what the bug is
+    validations:
+      required: true
+  - type: textarea
+    id: reproduce
+    attributes:
+      label: To Reproduce
+      description: Steps to reproduce the behavior
+    validations:
+      required: true
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected behavior
+      description: A clear and concise description of what you expected to happen
+    validations:
+      required: true
+  - type: textarea
+    id: environment
+    attributes:
+      label: Your Environment
+      description: |
+        - Fluentd or its package version: `fluentd --version` (Fluentd, fluent-package) or `td-agent --version` (td-agent)
+        - Operating system: `cat /etc/os-release`
+        - Kernel version: `uname -r`
+
+        Tip: If you hit the problem with older fluent-plugin-concat version, try latest version first.
+      value: |
+        - Fluentd version:
+        - Package version:
+        - Operating system:
+        - Kernel version:
+      render: markdown
+    validations:
+      required: true
+  - type: textarea
+    id: configuration
+    attributes:
+      label: Your Configuration
+      description: |
+        Write your configuration here. Minimum reproducible fluentd.conf is recommended.
+      render: apache
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Your Error Log
+      description: Write your ALL error log here
+      render: shell
+    validations:
+      required: true
+  - type: textarea
+    id: addtional-context
+    attributes:
+      label: Additional context
+      description: Add any other context about the problem here.
+    validations:
+      required: false

data/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Ask a Question
+    url: https://github.com/fluent/fluentd/discussions
+    about: I have questions about Fluentd and plugins. Please ask and answer questions at https://github.com/fluent/fluentd/discussions

data/.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,35 @@
+name: Feature request
+description: Suggest an idea for this project
+labels: ["waiting-for-triage", "enhancement"]
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: Is your feature request related to a problem? Please describe.
+      description: |
+        A clear and concise description of what the problem is.
+        Ex. I'm always frustrated when [...]
+    validations:
+      required: true
+  - type: textarea
+    id: solution
+    attributes:
+      label: Describe the solution you'd like
+      description: A clear and concise description of what you want to happen.
+    validations:
+      required: true
+  - type: textarea
+    id: alternative
+    attributes:
+      label: Describe alternatives you've considered
+      description: A clear and concise description of any alternative solutions or features you've considered.
+    validations:
+      required: true
+  - type: textarea
+    id: addtional-context
+    attributes:
+      label: Additional context
+      description: Add any other context or screenshots about the feature request here.
+    validations:
+      required: false
+

data/.github/workflows/add-to-project.yml

@@ -0,0 +1,17 @@
+name: Add bugs to fluent project
+
+on:
+  issues:
+    types:
+      - opened
+
+jobs:
+  add-to-project:
+    name: Add issue to project
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/add-to-project@v1.0.2
+        with:
+          project-url: https://github.com/orgs/fluent/projects/4
+          github-token: ${{ secrets.ADD_TO_PROJECT_PAT }}
+          labeled: waiting-for-triage

data/.github/workflows/linux.yml

@@ -9,18 +9,21 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
-          - 2.5
-          - 2.6
           - 2.7
-          - 3.0
+          - '3.0'
+          - 3.1
+          - 3.2
+          - 3.3
+          - 3.4
         os:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}
+        rubygems: latest
     - name: unit testing
       env:
         CI: true

data/.github/workflows/windows.yml

@@ -8,15 +8,16 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.5', '2.6', '2.7', '3.0' ]
+        ruby: [ '2.7', '3.0', '3.1', '3.2', '3.3', '3.4' ]
         os:
           - windows-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}
+        rubygems: latest
     - name: unit testing
       env:
         CI: true

data/NEWS.md

@@ -6,6 +6,12 @@
 
 * adds `partial_metadata_format` to support Docker 20.10 journald log driver improvements [moby/moby#41407](https://github.com/moby/moby/pull/41407) 
 
+## v2.6.0
+
+### Improvements
+
+* Support `buffer_limit_size`, `buffer_overflow_method` to change behavior when buffer overflow. See #68
+
 ## v2.3.0
 
 ### Improvements

data/README.md

@@ -27,6 +27,13 @@ Or install it yourself as:
 
     $ gem install fluent-plugin-concat
 
+## Plugin helpers
+
+* [timer](https://docs.fluentd.org/v1.0/articles/api-plugin-helper-timer)
+* [event_emitter](https://docs.fluentd.org/v1.0/articles/api-plugin-helper-event_emitter)
+
+* See also: [Filter Plugin Overview](https://docs.fluentd.org/filter#overview)
+
 ## Configuration
 
 ### Example
@@ -67,7 +74,7 @@ Or install it yourself as:
 |multiline\_end\_regexp|The regexp to match ending of multiline.This is exclusive with `n_lines`|nil|
 |continuous\_line\_regexp|The regexp to match continuous lines.This is exclusive with `n_lines`|nil|
 |stream\_identity\_key|The key to determine which stream an event belongs to|nil|
-|flush\_interval|The number of seconds after which the last received event log will be flushed.If specified 0, wait for next line foreverr|60|
+|flush\_interval|The number of seconds after which the last received event log will be flushed.If specified 0, wait for next line forever|60|
 |timeout\_label|The label name to handle events caused by timeout|nil|
 |use\_first\_timestamp|Use timestamp of first record when buffer is flushed|`false`|
 |partial\_key|The field name that is the reference to concatenate records|nil|
@@ -79,6 +86,20 @@ Or install it yourself as:
 |use\_partial\_cri\_logtag|bool (optional)|Use cri log tag to concatenate multiple records||
 |partial\_cri\_logtag\_key|string (optional)|The key name that is referred to concatenate records on cri log||
 |partial\_cri\_stream\_key|string (optional)|The key name that is referred to detect stream name on cri log|`stream`|
+|buffer\_limit\_size|The max size of each buffer|`512000`|
+
+|parameter|description|available values|default|
+|---|---|---|---|
+|buffer\_overflow\_method|The method if overflow buffer|`ignore`, `truncate`, `drop`, `new`|`ignore`|
+
+* `ignore`
+  * Concatenate the current record in the buffer
+* `truncate`
+  * Drop the current record and flush the buffer
+* `drop`
+  * Drop the current record and clear the buffer
+* `new`
+  * Flush the buffer and store the current record in next buffer
 
 ## Usage
 

data/fluent-plugin-concat.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-concat"
-  spec.version       = "2.5.0"
+  spec.version       = "2.6.0"
   spec.authors       = ["Kenji Okimoto"]
   spec.email         = ["okimoto@clear-code.com"]
 

data/lib/fluent/plugin/filter_concat.rb

@@ -22,7 +22,7 @@ module Fluent::Plugin
     config_param :stream_identity_key, :string, default: nil
     desc "The interval between data flushes, 0 means disable timeout"
     config_param :flush_interval, :time, default: 60
-    desc "The label name to handle timeout"
+    desc "The label name to handle events caused by timeout"
     config_param :timeout_label, :string, default: nil
     desc "Use timestamp of first record when buffer is flushed"
     config_param :use_first_timestamp, :bool, default: false
@@ -44,6 +44,10 @@ module Fluent::Plugin
     config_param :partial_cri_logtag_key, :string, default: nil
     desc "The key name that is referred to detect stream name on cri log"
     config_param :partial_cri_stream_key, :string, default: "stream"
+    desc "The max size of each buffer"
+    config_param :buffer_limit_size, :size, default: 500 * 1024 # 500k
+    desc "The method if overflow buffer"
+    config_param :buffer_overflow_method, :enum, list: [:ignore, :truncate, :drop, :new], default: :ignore
 
     class TimeoutError < StandardError
     end
@@ -52,6 +56,7 @@ module Fluent::Plugin
       super
 
       @buffer = Hash.new {|h, k| h[k] = [] }
+      @buffer_size = Hash.new(0)
       @timeout_map_mutex = Thread::Mutex.new
       @timeout_map_mutex.synchronize do
         @timeout_map = Hash.new {|h, k| h[k] = Fluent::Engine.now }
@@ -270,13 +275,39 @@ module Fluent::Plugin
 
     def process_partial(stream_identity, tag, time, record)
       new_es = Fluent::MultiEventStream.new
-      @buffer[stream_identity] << [tag, time, record]
-      unless @partial_value == record[@partial_key]
+      force_flush = false
+      if overflow?(stream_identity, record)
+        force_flush = case @buffer_overflow_method
+                      when :ignore
+                        @buffer[stream_identity] << [tag, time, record]
+                        false
+                      when :truncate
+                        true
+                      when :drop
+                        @buffer[stream_identity] = []
+                        false
+                      when :new
+                        true
+                      end
+      else
+        @buffer[stream_identity] << [tag, time, record]
+      end
+      if force_flush || @partial_value != record[@partial_key]
         new_time, new_record = flush_buffer(stream_identity)
         time = new_time if @use_first_timestamp
         new_record.delete(@partial_key)
         new_es.add(time, new_record)
       end
+      if force_flush && @buffer_overflow_method == :new
+        @buffer[stream_identity] << [tag, time, record]
+        @buffer_size[stream_identity] = record.keys.sum(&:bytesize) + record.values.sum(&:bytesize)
+        if @partial_value != record[@partial_key]
+          new_time, new_record = flush_buffer(stream_identity)
+          time = new_time if @use_first_timestamp
+          new_record.delete(@partial_key)
+          new_es.add(time, new_record)
+        end
+      end
       new_es
     end
 
@@ -371,6 +402,17 @@ module Fluent::Plugin
       end
     end
 
+    def overflow?(stream_identity, record)
+      size = record.keys.sum(&:bytesize) + record.values.sum(&:bytesize)
+      if @buffer_size[stream_identity] + size > @buffer_limit_size
+        @buffer_size[stream_identity] = 0
+        true
+      else
+        @buffer_size[stream_identity] += size
+        false
+      end
+    end
+
     def flush_buffer(stream_identity, new_element = nil)
       lines = if @mode == :partial_metadata
                 @buffer[stream_identity]
@@ -385,6 +427,7 @@ module Fluent::Plugin
       }
       @buffer[stream_identity] = []
       @buffer[stream_identity] << new_element if new_element
+      @buffer_size[stream_identity] = 0
       [time, first_record.merge(new_record)]
     end
 
@@ -434,3 +477,18 @@ module Fluent::Plugin
     end
   end
 end
+
+class Array
+  # Support Ruby 2.3 or earlier
+  unless [].respond_to?(:sum)
+    def sum
+      inject(0) do |memo, value|
+        if block_given?
+          memo + yield(value)
+        else
+          memo + value
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-concat
 version: !ruby/object:Gem::Version
-  version: 2.5.0
+  version: 2.6.0
 platform: ruby
 authors:
 - Kenji Okimoto
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-05-25 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -121,7 +120,10 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".github/ISSUE_TEMPLATE.md"
+- ".github/ISSUE_TEMPLATE/bug_report.yml"
+- ".github/ISSUE_TEMPLATE/config.yml"
+- ".github/ISSUE_TEMPLATE/feature_request.yml"
+- ".github/workflows/add-to-project.yml"
 - ".github/workflows/linux.yml"
 - ".github/workflows/windows.yml"
 - ".gitignore"
@@ -138,7 +140,6 @@ homepage: https://github.com/fluent-plugins-nursery/fluent-plugin-concat
 licenses:
 - MIT
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -153,8 +154,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
-signing_key:
+rubygems_version: 3.6.8
 specification_version: 4
 summary: Fluentd Filter plugin to concat multiple event messages
 test_files: []

data/.github/ISSUE_TEMPLATE.md

@@ -1,19 +0,0 @@
-#### Problem
-
-...
-
-#### Steps to replicate
-
-Provide example config and message
-
-#### Expected Behavior
-
-...
-
-#### Your environment
-
-* OS version
-* paste result of ``fluentd --version`` or ``td-agent --version``
-* plugin version
-  * paste boot log of fluentd or td-agent
-  * paste result of ``fluent-gem list``, ``td-agent-gem list`` or your Gemfile.lock