fluent-plugin-concat 2.4.0 → 2.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d906b3c9372b21adec5b1515e4820b037d90a0fa83a32f01dad1a8d6e5e0f79c
-  data.tar.gz: 9685f06e791e53ae0fcb3bb3e71991f812405298084c31cb4cc61a4181e17d6c
+  metadata.gz: 31897cab290a08b88945f1a78571bc4bfac24affae42aa2fb79eee0c6a5df937
+  data.tar.gz: 689b42a5ec25b1400d51af0c4f59961023e4fd01dd9b56c2d87b6ff83a2c3ddd
 SHA512:
-  metadata.gz: 1683fcfe93f86fedbc89334dde6565b48fad41800631a5624ebfd26f89f8c725c0cff1aed9ef3733d863bf1ec3fb2e0e62676ce78a68dc40d332b08f80209748
-  data.tar.gz: 6bec85d758dfce0dbced83a03413d2e1c2cef6d73a9c30ccb07f03c62d5d211d13559fdd7703f373f72af6d3c631ecf753451b93737777e82b2fc053ae5cb0ef
+  metadata.gz: 501acf92258e36fbb603c27bd4a861fff446304b33b3ccb425fb2fa281bc4b6ab05c357cc051196b17da66fd57c4de4e2e7d55999b194adfc3ce4dac9694b130
+  data.tar.gz: 369632414f573689ca37ea34c2f17b88a74476dab41ea7edfe9d8ee394d42b20fed5d27bef7306489729b7eb9ee96f8a1429adaca88bf1b0066d7ccbcc25b22a

data/.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,67 @@
+name: Bug Report
+description: Create a report with a procedure for reproducing the bug
+labels: "waiting-for-triage"
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: Describe the bug
+      description: A clear and concise description of what the bug is
+    validations:
+      required: true
+  - type: textarea
+    id: reproduce
+    attributes:
+      label: To Reproduce
+      description: Steps to reproduce the behavior
+    validations:
+      required: true
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected behavior
+      description: A clear and concise description of what you expected to happen
+    validations:
+      required: true
+  - type: textarea
+    id: environment
+    attributes:
+      label: Your Environment
+      description: |
+        - Fluentd or its package version: `fluentd --version` (Fluentd, fluent-package) or `td-agent --version` (td-agent)
+        - Operating system: `cat /etc/os-release`
+        - Kernel version: `uname -r`
+
+        Tip: If you hit the problem with older fluent-plugin-concat version, try latest version first.
+      value: |
+        - Fluentd version:
+        - Package version:
+        - Operating system:
+        - Kernel version:
+      render: markdown
+    validations:
+      required: true
+  - type: textarea
+    id: configuration
+    attributes:
+      label: Your Configuration
+      description: |
+        Write your configuration here. Minimum reproducible fluentd.conf is recommended.
+      render: apache
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Your Error Log
+      description: Write your ALL error log here
+      render: shell
+    validations:
+      required: true
+  - type: textarea
+    id: addtional-context
+    attributes:
+      label: Additional context
+      description: Add any other context about the problem here.
+    validations:
+      required: false

data/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Ask a Question
+    url: https://github.com/fluent/fluentd/discussions
+    about: I have questions about Fluentd and plugins. Please ask and answer questions at https://github.com/fluent/fluentd/discussions

data/.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,35 @@
+name: Feature request
+description: Suggest an idea for this project
+labels: ["waiting-for-triage", "enhancement"]
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: Is your feature request related to a problem? Please describe.
+      description: |
+        A clear and concise description of what the problem is.
+        Ex. I'm always frustrated when [...]
+    validations:
+      required: true
+  - type: textarea
+    id: solution
+    attributes:
+      label: Describe the solution you'd like
+      description: A clear and concise description of what you want to happen.
+    validations:
+      required: true
+  - type: textarea
+    id: alternative
+    attributes:
+      label: Describe alternatives you've considered
+      description: A clear and concise description of any alternative solutions or features you've considered.
+    validations:
+      required: true
+  - type: textarea
+    id: addtional-context
+    attributes:
+      label: Additional context
+      description: Add any other context or screenshots about the feature request here.
+    validations:
+      required: false
+

data/.github/workflows/add-to-project.yml

@@ -0,0 +1,17 @@
+name: Add bugs to fluent project
+
+on:
+  issues:
+    types:
+      - opened
+
+jobs:
+  add-to-project:
+    name: Add issue to project
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/add-to-project@v1.0.2
+        with:
+          project-url: https://github.com/orgs/fluent/projects/4
+          github-token: ${{ secrets.ADD_TO_PROJECT_PAT }}
+          labeled: waiting-for-triage

data/.github/workflows/linux.yml

@@ -0,0 +1,33 @@
+name: Testing on Ubuntu
+on:
+  - push
+  - pull_request
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby:
+          - 2.7
+          - '3.0'
+          - 3.1
+          - 3.2
+          - 3.3
+          - 3.4
+        os:
+          - ubuntu-latest
+    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v4
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+        rubygems: latest
+    - name: unit testing
+      env:
+        CI: true
+      run: |
+        gem install bundler rake
+        bundle install --jobs 4 --retry 3
+        bundle exec rake test

data/.github/workflows/windows.yml

@@ -0,0 +1,27 @@
+name: Testing on Windows
+on:
+  - push
+  - pull_request
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '2.7', '3.0', '3.1', '3.2', '3.3', '3.4' ]
+        os:
+          - windows-latest
+    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v4
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+        rubygems: latest
+    - name: unit testing
+      env:
+        CI: true
+      run: |
+        gem install bundler rake
+        bundle install --jobs 4 --retry 3
+        bundle exec rake test

data/NEWS.md

@@ -1,5 +1,17 @@
 # ChangeLog
 
+## next
+
+### Improvements
+
+* adds `partial_metadata_format` to support Docker 20.10 journald log driver improvements [moby/moby#41407](https://github.com/moby/moby/pull/41407) 
+
+## v2.6.0
+
+### Improvements
+
+* Support `buffer_limit_size`, `buffer_overflow_method` to change behavior when buffer overflow. See #68
+
 ## v2.3.0
 
 ### Improvements

data/README.md

@@ -27,75 +27,79 @@ Or install it yourself as:
 
     $ gem install fluent-plugin-concat
 
-## Configuration
-
-**key** (string) (required)
-
-The key for part of multiline log.
-
-**separator** (string) (optional)
-
-The separator of lines.
-Default value is `"\n"`.
-
-**n\_lines** (integer) (optional)
-
-The number of lines.
-This is exclusive with `multiline_start_regex`.
-
-**multiline\_start\_regexp** (string) (optional)
-
-The regexp to match beginning of multiline.
-This is exclusive with `n_lines.`
-
-**multiline\_end\_regexp** (string) (optional)
-
-The regexp to match ending of multiline.
-This is exclusive with `n_lines.`
-
-**continuous\_line\_regexp** (string) (optional)
-
-The regexp to match continuous lines.
-This is exclusive with `n_lines.`
-
-**stream\_identity\_key** (string) (optional)
+## Plugin helpers
 
-The key to determine which stream an event belongs to.
+* [timer](https://docs.fluentd.org/v1.0/articles/api-plugin-helper-timer)
+* [event_emitter](https://docs.fluentd.org/v1.0/articles/api-plugin-helper-event_emitter)
 
-**flush\_interval** (integer) (optional)
+* See also: [Filter Plugin Overview](https://docs.fluentd.org/filter#overview)
 
-The number of seconds after which the last received event log will be flushed.
-If specified 0, wait for next line forever.
-
-**timeout\_label** (string) (optional)
-
-The label name to handle events caused by timeout.
-
-**use\_first\_timestamp** (bool) (optional)
-
-Use timestamp of first record when buffer is flushed.
-Default value is `false`.
-
-**partial\_key** (string) (optional)
-
-The field name that is the reference to concatenate records
-
-**partial\_value** (string) (optional)
-
-The value stored in the field specified by partial_key that represent partial log
-
-**keep\_partial\_key** (bool) (optional)
-
-If true, keep partial_key in concatenated records
-Default value is `false`.
-
-**use\_partial\_metadata** (bool) (optional)
+## Configuration
 
-Use partial metadata to concatenate multiple records
+### Example
 
-**keep\_partial\_metadata** (bool) (optional)
+```
+<filter docker.log>
+  @type concat
+  key loga
+  #separator "\n"
+  n_lines 10
+  #multiline_start_regexp /^Start/
+  #multiline_end_regexp /^End/
+  #continuous_line_regexp nil
+  #stream_identity_key nil
+  #flush_interval 60
+  #timeout_label nil
+  #use_first_timestamp false
+  #partial_key nil
+  #partial_value nil
+  #keep_partial_key false
+  #use_partial_metadata false
+  #keep_partial_metadata false
+  #partial\_metadata\_format docker-fluentd
+  #use\_partial\_cri\_logtag false
+  #partial\_cri\_logtag\_key nil
+  #partial\_cri\_stream\_key stream
+</filter>
+```
 
-If true, keep partial metadata
+### Parameter
+
+|parameter|description|default|
+|---|---|---|
+|key|The key for part of multiline log||
+|separator|The separator of lines|`"\n"`|
+|n\_lines|The number of lines. This is exclusive with `multiline_start_regex`|nil|
+|multiline\_start\_regexp|The regexp to match beginning of multiline. This is exclusive with `n_lines`|nil|
+|multiline\_end\_regexp|The regexp to match ending of multiline.This is exclusive with `n_lines`|nil|
+|continuous\_line\_regexp|The regexp to match continuous lines.This is exclusive with `n_lines`|nil|
+|stream\_identity\_key|The key to determine which stream an event belongs to|nil|
+|flush\_interval|The number of seconds after which the last received event log will be flushed.If specified 0, wait for next line forever|60|
+|timeout\_label|The label name to handle events caused by timeout|nil|
+|use\_first\_timestamp|Use timestamp of first record when buffer is flushed|`false`|
+|partial\_key|The field name that is the reference to concatenate records|nil|
+|partial\_value|The value stored in the field specified by partial_key that represent partial log|nil|
+|keep\_partial\_key|If true, keep partial_key in concatenated records|`false`|
+|use\_partial\_metadata|Use partial metadata to concatenate multiple records|`false`|
+|keep\_partial\_metadata|If true, keep partial metadata|`false`|
+|partial\_metadata\_format|Input format of the partial metadata (fluentd or journald docker log driver) ( `docker-fluentd`, `docker-journald`, `docker-journald-lowercase`)<br>Configure based on the input plugin, that is used. <br>The docker fluentd and journald log drivers are behaving differently, so the plugin needs to know, what to look for.<br>Use `docker-journald-lowercase`, if you have `fields_lowercase true` in the `journald` source config |`docker-fluentd`|
+|use\_partial\_cri\_logtag|bool (optional)|Use cri log tag to concatenate multiple records||
+|partial\_cri\_logtag\_key|string (optional)|The key name that is referred to concatenate records on cri log||
+|partial\_cri\_stream\_key|string (optional)|The key name that is referred to detect stream name on cri log|`stream`|
+|buffer\_limit\_size|The max size of each buffer|`512000`|
+
+|parameter|description|available values|default|
+|---|---|---|---|
+|buffer\_overflow\_method|The method if overflow buffer|`ignore`, `truncate`, `drop`, `new`|`ignore`|
+
+* `ignore`
+  * Concatenate the current record in the buffer
+* `truncate`
+  * Drop the current record and flush the buffer
+* `drop`
+  * Drop the current record and clear the buffer
+* `new`
+  * Flush the buffer and store the current record in next buffer
 
 ## Usage
 
@@ -168,13 +172,38 @@ Handle Docker logs splitted in several parts (using `partial_message`), and do n
 ```aconf
 <filter>
   @type concat
-  key message
+  key log
   partial_key partial_message
   partial_value true
   separator ""
 </filter>
 ```
 
+(Docker v19.03+) Handle Docker logs splitted in several parts (using `use_partial_metadata`), and do not add new line between parts.
+
+```aconf
+<filter>
+  @type concat
+  key log
+  use_partial_metadata true
+  separator ""
+</filter>
+```
+
+(Docker v20.10+) Handle Docker logs splitted in several parts (using `use_partial_metadata`), and do not add new line between parts.
+
+Docker v20.10 improved partial message handling by adding better metadata in the journald log driver, this works now similarily to the fluentd log driver, but requires one additional setting
+
+```aconf
+<filter>
+  @type concat
+  key log
+  use_partial_metadata true
+  partial_metadata_format docker-journald
+  separator ""
+</filter>
+```
+
 Handle Docker logs splitted in several parts (using newline detection), and do not add new line between parts (prior to Docker 18.06).
 
 ```aconf
@@ -194,7 +223,8 @@ Handle containerd/cri in Kubernetes.
   path /var/log/containers/*.log
   <parse>
     @type regexp
-    expression /^(?<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z) (?<output>\w+) (?<partial_flag>[FP]) (?<message>.+)$/
+    expression /^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<message>.*)$/
+    time_format %Y-%m-%dT%H:%M:%S.%L%z
   </parse>
   tag k8s
   @label @CONCAT
@@ -204,8 +234,9 @@ Handle containerd/cri in Kubernetes.
   <filter k8s>
     @type concat
     key message
-    partial_key partial_flag
-    partial_value P
+    use_partial_cri_logtag true
+    partial_cri_logtag_key logtag
+    partial_cri_stream_key stream
   </filter>
   <match k8s>
     @type relabel

data/fluent-plugin-concat.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-concat"
-  spec.version       = "2.4.0"
+  spec.version       = "2.6.0"
   spec.authors       = ["Kenji Okimoto"]
   spec.email         = ["okimoto@clear-code.com"]
 
@@ -22,4 +22,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "test-unit", ">= 3.1.0"
   spec.add_development_dependency "test-unit-rr"
   spec.add_development_dependency "appraisal"
+  spec.add_development_dependency "webrick"
 end

data/lib/fluent/plugin/filter_concat.rb

@@ -22,7 +22,7 @@ module Fluent::Plugin
     config_param :stream_identity_key, :string, default: nil
     desc "The interval between data flushes, 0 means disable timeout"
     config_param :flush_interval, :time, default: 60
-    desc "The label name to handle timeout"
+    desc "The label name to handle events caused by timeout"
     config_param :timeout_label, :string, default: nil
     desc "Use timestamp of first record when buffer is flushed"
     config_param :use_first_timestamp, :bool, default: false
@@ -34,8 +34,20 @@ module Fluent::Plugin
     config_param :keep_partial_key, :bool, default: false
     desc "Use partial metadata to concatenate multiple records"
     config_param :use_partial_metadata, :bool, default: false
+    desc "Input format of the partial metadata (fluentd or journald docker log driver)"
+    config_param :partial_metadata_format, :enum, list: [:"docker-fluentd", :"docker-journald", :"docker-journald-lowercase"], default: :"docker-fluentd"
     desc "If true, keep partial metadata"
     config_param :keep_partial_metadata, :bool, default: false
+    desc "Use cri log tag to concatenate multiple records"
+    config_param :use_partial_cri_logtag, :bool, default: false
+    desc "The key name that is referred to concatenate records on cri log"
+    config_param :partial_cri_logtag_key, :string, default: nil
+    desc "The key name that is referred to detect stream name on cri log"
+    config_param :partial_cri_stream_key, :string, default: "stream"
+    desc "The max size of each buffer"
+    config_param :buffer_limit_size, :size, default: 500 * 1024 # 500k
+    desc "The method if overflow buffer"
+    config_param :buffer_overflow_method, :enum, list: [:ignore, :truncate, :drop, :new], default: :ignore
 
     class TimeoutError < StandardError
     end
@@ -44,17 +56,25 @@ module Fluent::Plugin
       super
 
       @buffer = Hash.new {|h, k| h[k] = [] }
+      @buffer_size = Hash.new(0)
       @timeout_map_mutex = Thread::Mutex.new
       @timeout_map_mutex.synchronize do
         @timeout_map = Hash.new {|h, k| h[k] = Fluent::Engine.now }
       end
     end
 
+    def required_params
+      params = [@n_lines.nil?, @multiline_start_regexp.nil?, @multiline_end_regexp.nil?, @partial_key.nil?, !@use_partial_metadata, !@use_partial_cri_logtag]
+      names = ["n_lines", "multiline_start_regexp", "multiline_end_regexp", "partial_key", "use_partial_metadata", "use_partial_cri_logtag"]
+      return params, names
+    end
+
     def configure(conf)
       super
 
-      if @n_lines.nil? && @multiline_start_regexp.nil? && @multiline_end_regexp.nil? && @partial_key.nil? && !@use_partial_metadata
-        raise Fluent::ConfigError, "Either n_lines, multiline_start_regexp, multiline_end_regexp, partial_key or use_partial_metadata is required"
+      params, names = required_params
+      if params.all?
+        raise Fluent::ConfigError, "Either #{[names[0..-2].join(", "), names[-1]].join(" or ")} is required"
       end
       if @n_lines && (@multiline_start_regexp || @multiline_end_regexp)
         raise Fluent::ConfigError, "n_lines and multiline_start_regexp/multiline_end_regexp are exclusive"
@@ -69,13 +89,22 @@ module Fluent::Plugin
         raise Fluent::ConfigError, "partial_value is required when partial_key is specified"
       end
       if @use_partial_metadata && @n_lines
-        raise Fluent::ConfigError, "user_partial_metadata and n_lines are exclusive"
+        raise Fluent::ConfigError, "use_partial_metadata and n_lines are exclusive"
       end
       if @use_partial_metadata && (@multiline_start_regexp || @multiline_end_regexp)
-        raise Fluent::ConfigError, "user_partial_metadata and multiline_start_regexp/multiline_end_regexp are exclusive"
+        raise Fluent::ConfigError, "use_partial_metadata and multiline_start_regexp/multiline_end_regexp are exclusive"
       end
       if @use_partial_metadata && @partial_key
-        raise Fluent::ConfigError, "user_partial_metadata and partial_key are exclusive"
+        raise Fluent::ConfigError, "use_partial_metadata and partial_key are exclusive"
+      end
+      if @use_partial_cri_logtag && @n_lines
+        raise Fluent::ConfigError, "use_partial_cri_logtag and n_lines are exclusive"
+      end
+      if @use_partial_cri_logtag && (@multiline_start_regexp || @multiline_end_regexp)
+        raise Fluent::ConfigError, "use_partial_cri_logtag and multiline_start_regexp/multiline_end_regexp are exclusive"
+      end
+      if @use_partial_cri_logtag && @partial_key
+        raise Fluent::ConfigError, "use_partial_cri_logtag and partial_key are exclusive"
       end
 
       @mode = nil
@@ -86,6 +115,33 @@ module Fluent::Plugin
         @mode = :partial
       when @use_partial_metadata
         @mode = :partial_metadata
+        case @partial_metadata_format
+        when :"docker-fluentd"
+          @partial_message_field     = "partial_message".freeze
+          @partial_id_field          = "partial_id".freeze
+          @partial_ordinal_field     = "partial_ordinal".freeze
+          @partial_last_field        = "partial_last".freeze
+          @partial_message_indicator = @partial_message_field
+        when :"docker-journald"
+          @partial_message_field     = "CONTAINER_PARTIAL_MESSAGE".freeze
+          @partial_id_field          = "CONTAINER_PARTIAL_ID".freeze
+          @partial_ordinal_field     = "CONTAINER_PARTIAL_ORDINAL".freeze
+          @partial_last_field        = "CONTAINER_PARTIAL_LAST".freeze
+          # the journald log driver does not add CONTAINER_PARTIAL_MESSAGE to the last message
+          # so we help ourself by using another indicator
+          @partial_message_indicator = @partial_id_field
+        when :"docker-journald-lowercase"
+          @partial_message_field     = "container_partial_message".freeze
+          @partial_id_field          = "container_partial_id".freeze
+          @partial_ordinal_field     = "container_partial_ordinal".freeze
+          @partial_last_field        = "container_partial_last".freeze
+          @partial_message_indicator = @partial_id_field
+        end
+      when @use_partial_cri_logtag
+        @mode = :partial_cri
+        @partial_logtag_delimiter = ":".freeze
+        @partial_logtag_continue = "P".freeze
+        @partial_logtag_full = "F".freeze
       when @multiline_start_regexp || @multiline_end_regexp
         @mode = :regexp
         if @multiline_start_regexp
@@ -113,12 +169,12 @@ module Fluent::Plugin
     end
 
     def filter_stream(tag, es)
+      if /\Afluent\.(?:trace|debug|info|warn|error|fatal)\z/ =~ tag
+        return es
+      end
+
       new_es = Fluent::MultiEventStream.new
       es.each do |time, record|
-        if /\Afluent\.(?:trace|debug|info|warn|error|fatal)\z/ =~ tag
-          new_es.add(time, record)
-          next
-        end
         unless record.key?(@key)
           new_es.add(time, record)
           next
@@ -130,7 +186,7 @@ module Fluent::Plugin
           end
         end
         if @mode == :partial_metadata
-          unless record.key?("partial_message")
+          unless record.key?(@partial_message_indicator)
             new_es.add(time, record)
             next
           end
@@ -146,11 +202,14 @@ module Fluent::Plugin
                 merged_record.delete(@partial_key) unless @keep_partial_key
               when :partial_metadata
                 unless @keep_partial_metadata
-                  merged_record.delete("partial_message")
-                  merged_record.delete("partial_id")
-                  merged_record.delete("partial_ordinal")
-                  merged_record.delete("partial_last")
+                  merged_record.delete(@partial_message_field)
+                  merged_record.delete(@partial_id_field)
+                  merged_record.delete(@partial_ordinal_field)
+                  merged_record.delete(@partial_last_field)
                 end
+              when :partial_cri
+                merged_record.delete(@partial_cri_logtag_key) unless @keep_partial_key
+                merged_record.delete(@partial_cri_stream_key)
               end
               new_es.add(time, merged_record)
             end
@@ -175,9 +234,9 @@ module Fluent::Plugin
     def process(tag, time, record)
       if @mode == :partial_metadata
         if @stream_identity_key
-          stream_identity = %Q(#{tag}:#{record[@stream_identity_key]}#{record["partial_id"]})
+          stream_identity = %Q(#{tag}:#{record[@stream_identity_key]}#{record[@partial_id_field]})
         else
-          stream_identity = %Q(#{tag}:#{record["partial_id"]})
+          stream_identity = %Q(#{tag}:#{record[@partial_id_field]})
         end
       else
         if @stream_identity_key
@@ -196,6 +255,8 @@ module Fluent::Plugin
         process_partial(stream_identity, tag, time, record)
       when :partial_metadata
         process_partial_metadata(stream_identity, tag, time, record)
+      when :partial_cri
+        process_partial_cri(stream_identity, tag, time, record)
       when :regexp
         process_regexp(stream_identity, tag, time, record)
       end
@@ -214,20 +275,58 @@ module Fluent::Plugin
 
     def process_partial(stream_identity, tag, time, record)
       new_es = Fluent::MultiEventStream.new
-      @buffer[stream_identity] << [tag, time, record]
-      unless @partial_value == record[@partial_key]
+      force_flush = false
+      if overflow?(stream_identity, record)
+        force_flush = case @buffer_overflow_method
+                      when :ignore
+                        @buffer[stream_identity] << [tag, time, record]
+                        false
+                      when :truncate
+                        true
+                      when :drop
+                        @buffer[stream_identity] = []
+                        false
+                      when :new
+                        true
+                      end
+      else
+        @buffer[stream_identity] << [tag, time, record]
+      end
+      if force_flush || @partial_value != record[@partial_key]
         new_time, new_record = flush_buffer(stream_identity)
         time = new_time if @use_first_timestamp
         new_record.delete(@partial_key)
         new_es.add(time, new_record)
       end
+      if force_flush && @buffer_overflow_method == :new
+        @buffer[stream_identity] << [tag, time, record]
+        @buffer_size[stream_identity] = record.keys.sum(&:bytesize) + record.values.sum(&:bytesize)
+        if @partial_value != record[@partial_key]
+          new_time, new_record = flush_buffer(stream_identity)
+          time = new_time if @use_first_timestamp
+          new_record.delete(@partial_key)
+          new_es.add(time, new_record)
+        end
+      end
+      new_es
+    end
+
+    def process_partial_cri(stream_identity, tag, time, record)
+      new_es = Fluent::MultiEventStream.new
+      @buffer[stream_identity] << [tag, time, record]
+      if record[@partial_cri_logtag_key].split(@partial_logtag_delimiter)[0] == @partial_logtag_full
+        new_time, new_record = flush_buffer(stream_identity)
+        time = new_time if @use_first_timestamp
+        new_record.delete(@partial_cri_logtag_key)
+        new_es.add(time, new_record)
+      end
       new_es
     end
 
     def process_partial_metadata(stream_identity, tag, time, record)
       new_es = Fluent::MultiEventStream.new
       @buffer[stream_identity] << [tag, time, record]
-      if record["partial_last"] == "true"
+      if record[@partial_last_field] == "true"
         new_time, new_record = flush_buffer(stream_identity)
         time = new_time if @use_first_timestamp
         new_record.delete(@partial_key)
@@ -288,25 +387,36 @@ module Fluent::Plugin
     end
 
     def firstline?(text)
-      @multiline_start_regexp && !!@multiline_start_regexp.match(text)
+      @multiline_start_regexp && @multiline_start_regexp.match?(text)
     end
 
     def lastline?(text)
-      @multiline_end_regexp && !!@multiline_end_regexp.match(text)
+      @multiline_end_regexp && @multiline_end_regexp.match?(text)
     end
 
     def continuous_line?(text)
       if @continuous_line_regexp
-        !!@continuous_line_regexp.match(text)
+        @continuous_line_regexp.match?(text)
       else
         true
       end
     end
 
+    def overflow?(stream_identity, record)
+      size = record.keys.sum(&:bytesize) + record.values.sum(&:bytesize)
+      if @buffer_size[stream_identity] + size > @buffer_limit_size
+        @buffer_size[stream_identity] = 0
+        true
+      else
+        @buffer_size[stream_identity] += size
+        false
+      end
+    end
+
     def flush_buffer(stream_identity, new_element = nil)
       lines = if @mode == :partial_metadata
                 @buffer[stream_identity]
-                  .sort_by {|_tag, _time, record| record["partial_ordinal"].to_i }
+                  .sort_by {|_tag, _time, record| record[@partial_ordinal_field].to_i }
                   .map {|_tag, _time, record| record[@key] }
               else
                 @buffer[stream_identity].map {|_tag, _time, record| record[@key] }
@@ -317,6 +427,7 @@ module Fluent::Plugin
       }
       @buffer[stream_identity] = []
       @buffer[stream_identity] << new_element if new_element
+      @buffer_size[stream_identity] = 0
       [time, first_record.merge(new_record)]
     end
 
@@ -366,3 +477,18 @@ module Fluent::Plugin
     end
   end
 end
+
+class Array
+  # Support Ruby 2.3 or earlier
+  unless [].respond_to?(:sum)
+    def sum
+      inject(0) do |memo, value|
+        if block_given?
+          memo + yield(value)
+        else
+          memo + value
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-concat
 version: !ruby/object:Gem::Version
-  version: 2.4.0
+  version: 2.6.0
 platform: ruby
 authors:
 - Kenji Okimoto
-autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-07-26 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -100,6 +99,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Fluentd Filter plugin to concat multiple event messages
 email:
 - okimoto@clear-code.com
@@ -107,10 +120,14 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".github/ISSUE_TEMPLATE.md"
+- ".github/ISSUE_TEMPLATE/bug_report.yml"
+- ".github/ISSUE_TEMPLATE/config.yml"
+- ".github/ISSUE_TEMPLATE/feature_request.yml"
+- ".github/workflows/add-to-project.yml"
+- ".github/workflows/linux.yml"
+- ".github/workflows/windows.yml"
 - ".gitignore"
 - ".rubocop.yml"
-- ".travis.yml"
 - Appraisals
 - Gemfile
 - LICENSE.txt
@@ -123,7 +140,6 @@ homepage: https://github.com/fluent-plugins-nursery/fluent-plugin-concat
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -138,8 +154,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.6.8
 specification_version: 4
 summary: Fluentd Filter plugin to concat multiple event messages
 test_files: []

data/.github/ISSUE_TEMPLATE.md

@@ -1,19 +0,0 @@
-#### Problem
-
-...
-
-#### Steps to replicate
-
-Provide example config and message
-
-#### Expected Behavior
-
-...
-
-#### Your environment
-
-* OS version
-* paste result of ``fluentd --version`` or ``td-agent --version``
-* plugin version
-  * paste boot log of fluentd or td-agent
-  * paste result of ``fluent-gem list``, ``td-agent-gem list`` or your Gemfile.lock

data/.travis.yml

@@ -1,19 +0,0 @@
-sudo: false
-language: ruby
-
-notifications:
-  webhooks:
-    urls:
-      secure: NhOBulc3nFAnJrCr9G0/3IGd1pOb/Zk0Wyk9Jqrf7PHnI4mZOIgttGnewZx1sG96NWrs0Braue5vXwFBTrJ60J6EW3+Q33XzhaZR+Q95jEb+5vcdIP69TYs0TgOIiOouo69Y4rIE1ANYuF/1cBfCKxGulpOIKUCW5LYPkyUEwQy+7VccULctCYCha2OYHQau5SUITCQQqZMfYCvZ+9FRubhR+OiiAnwduuzixKm5b2sa2K4lCKy2vsz3JZtjvQ6/7WM5JsipErtuUY2fDlrWSr/9SehRu8LCD119FjV/HcaAffFMokX+8vJm6RLGcyx/0mAejEqaBofqf9vi1KQJKsKuEe8N9IUpoLrl36RYVfDDRNDqm0pSkrz/KaNxSSe5sDNsCFY+uEFhtiPMHJiSlyEA+BVqnUeoM9pSWp3pcTsIHAjq5VGfQ6u9IooUODCASBlOsLuPu+FYEWG3dC3oiOLEIqQ+WywJg2P7I4dJNhyivti5j61czipTGj28ReFiydp5macKabkvAFBvOW8q07QBO4cvjixhr8hiUgMEVdo5y5dqNhLvS182by3OpmtaSQfpuI8QXytiN9629lWzB7qu0Q7Pji/5wSqEuCdaEHjF5uCVYLFE2LaGJdlTLBg9n0vFfFpJXnP1q0nehq2qRs30pNcgkcCKfEIPJgFgCac=
-    on_success: change
-    on_failure: always
-    on_start: never
-
-matrix:
-  include:
-    - rvm: 2.4.5
-      gemfile: Gemfile
-    - rvm: 2.5.5
-      gemfile: Gemfile
-    - rvm: 2.6.2
-      gemfile: Gemfile