fluent-plugin-concat 2.6.0 → 2.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 31897cab290a08b88945f1a78571bc4bfac24affae42aa2fb79eee0c6a5df937
-  data.tar.gz: 689b42a5ec25b1400d51af0c4f59961023e4fd01dd9b56c2d87b6ff83a2c3ddd
+  metadata.gz: 6e3b40a3ba2937ad73962f14278075d37467c3f3a4498a0915a25736b3387e16
+  data.tar.gz: 7bdc6876112368367fc31a14b1c5b49984308a91810f15f55cb561c5b73a0d2d
 SHA512:
-  metadata.gz: 501acf92258e36fbb603c27bd4a861fff446304b33b3ccb425fb2fa281bc4b6ab05c357cc051196b17da66fd57c4de4e2e7d55999b194adfc3ce4dac9694b130
-  data.tar.gz: 369632414f573689ca37ea34c2f17b88a74476dab41ea7edfe9d8ee394d42b20fed5d27bef7306489729b7eb9ee96f8a1429adaca88bf1b0066d7ccbcc25b22a
+  metadata.gz: d21cf56c213937fc8a506a23d0f3126d5f6df484d78983aa58b2f8c58af0263447f3a51e6b1be1998d1fa9b1b695954561e0bf12ec3a20fa47fdb9ed19c0259c
+  data.tar.gz: 5482a3a6d00f598dcabd1c91bf92502fddaf69db1b60493f42dd549282e5bdeae582c297e6c2324cf6b8b4d6150dbcf86b4dceac8ae7a1cd5cadbd65b0a316e0

data/.github/dependabot.yml

@@ -0,0 +1,23 @@
+version: 2
+updates:
+  - package-ecosystem: 'github-actions'
+    directory: '/'
+    schedule:
+      interval: 'monthly'
+    groups:
+      # PR: "Security update [package] from [old] to [new]"
+      # This PR should be merged in hurry
+      security-updates:
+        applies-to: security-updates
+        patterns:
+          - '*'
+
+      # PR: "Bump [package] from [old] to [new]"
+      # No need to be merged this PR in hurry. It is enough to merge
+      # once in a month.
+      monthly-updates:
+        applies-to: version-updates
+        patterns:
+          - '*'
+    # Allow to create PR both of security and normal updates.
+    open-pull-requests-limit: 1

data/.github/workflows/add-to-project.yml

@@ -10,8 +10,15 @@ jobs:
     name: Add issue to project
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/add-to-project@v1.0.2
+      - name: Generate token
+        id: generate_token
+        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
+        with:
+          app-id: ${{ secrets.PROJECT_APP_ID }}
+          private-key: ${{ secrets.PROJECT_APP_PRIVATE_KEY }}
+          owner: fluent
+      - uses: actions/add-to-project@5afcf98fcd03f1c2f92c3c83f58ae24323cc57fd # v2.0.0
         with:
           project-url: https://github.com/orgs/fluent/projects/4
-          github-token: ${{ secrets.ADD_TO_PROJECT_PAT }}
+          github-token: ${{ steps.generate_token.outputs.token }}
           labeled: waiting-for-triage

data/.github/workflows/linux.yml

@@ -1,26 +1,32 @@
 name: Testing on Ubuntu
 on:
-  - push
-  - pull_request
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+  schedule:
+    - cron: '0 0 1 * *'
 jobs:
+  ruby-versions:
+    uses: ruby/actions/.github/workflows/ruby_versions.yml@master
+    with:
+      engine: cruby
+      min_version: 2.7
   build:
+    needs: ruby-versions
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        ruby:
-          - 2.7
-          - '3.0'
-          - 3.1
-          - 3.2
-          - 3.3
-          - 3.4
+        ruby: ${{ fromJson(needs.ruby-versions.outputs.versions) }}
+        exclude:
+          - ruby: head
         os:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v4
-    - uses: ruby/setup-ruby@v1
+    - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+    - uses: ruby/setup-ruby@afeafc3d1ab54a631816aba4c914a0081c12ff2f # v1.310.0
       with:
         ruby-version: ${{ matrix.ruby }}
         rubygems: latest
@@ -28,6 +34,5 @@ jobs:
       env:
         CI: true
       run: |
-        gem install bundler rake
         bundle install --jobs 4 --retry 3
         bundle exec rake test

data/.github/workflows/windows.yml

@@ -1,20 +1,32 @@
 name: Testing on Windows
 on:
-  - push
-  - pull_request
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+  schedule:
+    - cron: '0 0 1 * *'
 jobs:
+  ruby-versions:
+    uses: ruby/actions/.github/workflows/ruby_versions.yml@master
+    with:
+      engine: cruby
+      min_version: 2.7
   build:
+    needs: ruby-versions
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.7', '3.0', '3.1', '3.2', '3.3', '3.4' ]
+        ruby: ${{ fromJson(needs.ruby-versions.outputs.versions) }}
+        exclude:
+          - ruby: head
         os:
           - windows-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v4
-    - uses: ruby/setup-ruby@v1
+    - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+    - uses: ruby/setup-ruby@afeafc3d1ab54a631816aba4c914a0081c12ff2f # v1.310.0
       with:
         ruby-version: ${{ matrix.ruby }}
         rubygems: latest
@@ -22,6 +34,5 @@ jobs:
       env:
         CI: true
       run: |
-        gem install bundler rake
         bundle install --jobs 4 --retry 3
         bundle exec rake test

data/NEWS.md

@@ -1,10 +1,17 @@
 # ChangeLog
 
-## next
+## v2.6.2
 
-### Improvements
+### Fixes
 
-* adds `partial_metadata_format` to support Docker 20.10 journald log driver improvements [moby/moby#41407](https://github.com/moby/moby/pull/41407) 
+* Fix unbounded growth of stream state in timeout handling (#150)
+* docs: clarify timeout handling to prevent silent log loss
+
+## v2.6.1
+
+### Fixes
+
+* Fix NoMethodError when calculating bytesize of non-string record values. See #136
 
 ## v2.6.0
 
@@ -12,6 +19,12 @@
 
 * Support `buffer_limit_size`, `buffer_overflow_method` to change behavior when buffer overflow. See #68
 
+## v2.5.0
+
+### Improvements
+
+* adds `partial_metadata_format` to support Docker 20.10 journald log driver improvements [moby/moby#41407](https://github.com/moby/moby/pull/41407) 
+
 ## v2.3.0
 
 ### Improvements
@@ -63,4 +76,3 @@
 ### Incompatibilities
 
 * Flush buffer when match both `multiline_start_regexp` and `multiline_end_regexp`
-

data/README.md

@@ -1,6 +1,7 @@
 # fluent-plugin-concat
 
-[![Build Status](https://travis-ci.org/fluent-plugins-nursery/fluent-plugin-concat.svg?branch=master)](https://travis-ci.org/fluent-plugins-nursery/fluent-plugin-concat)
+[![Testing on Ubuntu](https://github.com/fluent-plugins-nursery/fluent-plugin-concat/actions/workflows/linux.yml/badge.svg)](https://github.com/fluent-plugins-nursery/fluent-plugin-concat/actions/workflows/linux.yml)
+[![Testing on Windows](https://github.com/fluent-plugins-nursery/fluent-plugin-concat/actions/workflows/windows.yml/badge.svg)](https://github.com/fluent-plugins-nursery/fluent-plugin-concat/actions/workflows/windows.yml)
 
 Fluentd Filter plugin to concatenate multiline log separated in multiple events.
 
@@ -123,40 +124,6 @@ Specify first line of multiline by regular expression.
 </filter>
 ```
 
-You can handle timeout events and remaining buffers on shutdown this plugin.
-
-```aconf
-<label @ERROR>
-  <match docker.log>
-    @type file
-    path /path/to/error.log
-  </match>
-</label>
-```
-
-Handle timeout log lines the same as normal logs.
-
-```aconf
-<filter **>
-  @type concat
-  key message
-  multiline_start_regexp /^Start/
-  flush_interval 5
-  timeout_label @NORMAL
-</filter>
-
-<match **>
-  @type relabel
-  @label @NORMAL
-</match>
-
-<label @NORMAL>
-  <match **>
-    @type stdout
-  </match>
-</label>
-```
-
 Handle single line JSON from Docker containers.
 
 ```aconf
@@ -251,6 +218,40 @@ Handle containerd/cri in Kubernetes.
 </label>
 ```
 
+### Timeout Handling and Preventing Log Loss
+
+When a multiline log is incomplete and reaches the `flush_interval`, the plugin flushes the remaining buffer. **These timeout events are routed to the `@ERROR` label by default**.
+
+**If you do not explicitly handle these timeout events, your last log lines will be silently dropped.**
+
+To prevent data loss and handle timeout logs the same as normal logs, you **must** configure `timeout_label` and set up a corresponding `<label>` block to catch and route them properly.
+
+#### Example: Safely rescuing timeout logs
+
+```aconf
+<filter **>
+  @type concat
+  key message
+  multiline_start_regexp /^Start/
+  flush_interval 5
+  # 1. Route timeout events to a specific label
+  timeout_label @NORMAL 
+</filter>
+
+# 2. Route normal events to the same label
+<match **>
+  @type relabel
+  @label @NORMAL
+</match>
+
+# 3. Handle both normal and timeout events together
+<label @NORMAL>
+  <match **>
+    @type stdout
+  </match>
+</label>
+```
+
 ## Contributing
 
 1. Fork it

data/fluent-plugin-concat.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-concat"
-  spec.version       = "2.6.0"
+  spec.version       = "2.6.2"
   spec.authors       = ["Kenji Okimoto"]
   spec.email         = ["okimoto@clear-code.com"]
 

data/lib/fluent/plugin/filter_concat.rb

@@ -300,7 +300,7 @@ module Fluent::Plugin
       end
       if force_flush && @buffer_overflow_method == :new
         @buffer[stream_identity] << [tag, time, record]
-        @buffer_size[stream_identity] = record.keys.sum(&:bytesize) + record.values.sum(&:bytesize)
+        @buffer_size[stream_identity] = record_bytesize(record)
         if @partial_value != record[@partial_key]
           new_time, new_record = flush_buffer(stream_identity)
           time = new_time if @use_first_timestamp
@@ -403,7 +403,7 @@ module Fluent::Plugin
     end
 
     def overflow?(stream_identity, record)
-      size = record.keys.sum(&:bytesize) + record.values.sum(&:bytesize)
+      size = record_bytesize(record)
       if @buffer_size[stream_identity] + size > @buffer_limit_size
         @buffer_size[stream_identity] = 0
         true
@@ -433,20 +433,27 @@ module Fluent::Plugin
 
     def flush_timeout_buffer
       now = Fluent::Engine.now
-      timeout_stream_identities = []
+      expired_stream_identities = []
       @timeout_map_mutex.synchronize do
         @timeout_map.each do |stream_identity, previous_timestamp|
           next if @flush_interval > (now - previous_timestamp)
+          expired_stream_identities << stream_identity
           next if @buffer[stream_identity].empty?
           time, flushed_record = flush_buffer(stream_identity)
-          timeout_stream_identities << stream_identity
           tag = stream_identity.split(":").first
           message = "Timeout flush: #{stream_identity}"
           handle_timeout_error(tag, @use_first_timestamp ? time : now, flushed_record, message)
           log.info(message)
         end
-        @timeout_map.reject! do |stream_identity, _|
-          timeout_stream_identities.include?(stream_identity)
+        # Purge expired streams from all the state hashes, not only the
+        # flushed ones. Streams which completed normally leave empty entries
+        # behind, so @buffer/@buffer_size/@timeout_map would otherwise grow
+        # unboundedly (notably in partial metadata mode where every split
+        # message has a unique stream identity).
+        expired_stream_identities.each do |stream_identity|
+          @timeout_map.delete(stream_identity)
+          @buffer.delete(stream_identity)
+          @buffer_size.delete(stream_identity)
         end
       end
     end
@@ -475,6 +482,10 @@ module Fluent::Plugin
         router.emit_error_event(tag, time, record, TimeoutError.new(message))
       end
     end
+
+    def record_bytesize(record)
+      record.keys.sum(&:bytesize) + record.values.sum { |v| v.to_s.bytesize }
+    end
   end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-concat
 version: !ruby/object:Gem::Version
-  version: 2.6.0
+  version: 2.6.2
 platform: ruby
 authors:
 - Kenji Okimoto
@@ -123,6 +123,7 @@ files:
 - ".github/ISSUE_TEMPLATE/bug_report.yml"
 - ".github/ISSUE_TEMPLATE/config.yml"
 - ".github/ISSUE_TEMPLATE/feature_request.yml"
+- ".github/dependabot.yml"
 - ".github/workflows/add-to-project.yml"
 - ".github/workflows/linux.yml"
 - ".github/workflows/windows.yml"
@@ -154,7 +155,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.8
+rubygems_version: 4.0.10
 specification_version: 4
 summary: Fluentd Filter plugin to concat multiple event messages
 test_files: []