fluent-plugin-concat 2.6.1 → 2.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 57c16827309de887d99790f740319040ac08b1ab8e137d8c99caa7f8c71b68f0
-  data.tar.gz: 99ac67b840a21e911128cff0123426f11911f1c4baf7324eaab4f1f615dbf160
+  metadata.gz: 6e3b40a3ba2937ad73962f14278075d37467c3f3a4498a0915a25736b3387e16
+  data.tar.gz: 7bdc6876112368367fc31a14b1c5b49984308a91810f15f55cb561c5b73a0d2d
 SHA512:
-  metadata.gz: 8fc317393e1dfbdb7cecc39e0fb045ee82ea93f4f21e3a0f377c460b0df6b0d89f0ef1cd4ff11aeb20a99f7ae3134a4ddd7c96e7166fceca46c4f4ad36f4b168
-  data.tar.gz: 20661b96b7799c0484c98a7b9ece9376b4b004d7b3e054c7eb3b3de0c9c760ba6354d72b8f3e8025284f4dad85afe2a08ec3dc69f11df8f6359493a6a15ba496
+  metadata.gz: d21cf56c213937fc8a506a23d0f3126d5f6df484d78983aa58b2f8c58af0263447f3a51e6b1be1998d1fa9b1b695954561e0bf12ec3a20fa47fdb9ed19c0259c
+  data.tar.gz: 5482a3a6d00f598dcabd1c91bf92502fddaf69db1b60493f42dd549282e5bdeae582c297e6c2324cf6b8b4d6150dbcf86b4dceac8ae7a1cd5cadbd65b0a316e0

data/.github/dependabot.yml

@@ -4,3 +4,20 @@ updates:
     directory: '/'
     schedule:
       interval: 'monthly'
+    groups:
+      # PR: "Security update [package] from [old] to [new]"
+      # This PR should be merged in hurry
+      security-updates:
+        applies-to: security-updates
+        patterns:
+          - '*'
+
+      # PR: "Bump [package] from [old] to [new]"
+      # No need to be merged this PR in hurry. It is enough to merge
+      # once in a month.
+      monthly-updates:
+        applies-to: version-updates
+        patterns:
+          - '*'
+    # Allow to create PR both of security and normal updates.
+    open-pull-requests-limit: 1

data/.github/workflows/add-to-project.yml

@@ -12,12 +12,12 @@ jobs:
     steps:
       - name: Generate token
         id: generate_token
-        uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0
+        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
         with:
           app-id: ${{ secrets.PROJECT_APP_ID }}
           private-key: ${{ secrets.PROJECT_APP_PRIVATE_KEY }}
           owner: fluent
-      - uses: actions/add-to-project@244f685bbc3b7adfa8466e08b698b5577571133e # v1.0.2
+      - uses: actions/add-to-project@5afcf98fcd03f1c2f92c3c83f58ae24323cc57fd # v2.0.0
         with:
           project-url: https://github.com/orgs/fluent/projects/4
           github-token: ${{ steps.generate_token.outputs.token }}

data/.github/workflows/linux.yml

@@ -25,8 +25,8 @@ jobs:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-    - uses: ruby/setup-ruby@6ca151fd1bfcfd6fe0c4eb6837eb0584d0134a0c # v1.290.0
+    - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+    - uses: ruby/setup-ruby@afeafc3d1ab54a631816aba4c914a0081c12ff2f # v1.310.0
       with:
         ruby-version: ${{ matrix.ruby }}
         rubygems: latest

data/.github/workflows/windows.yml

@@ -25,8 +25,8 @@ jobs:
           - windows-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-    - uses: ruby/setup-ruby@6ca151fd1bfcfd6fe0c4eb6837eb0584d0134a0c # v1.290.0
+    - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+    - uses: ruby/setup-ruby@afeafc3d1ab54a631816aba4c914a0081c12ff2f # v1.310.0
       with:
         ruby-version: ${{ matrix.ruby }}
         rubygems: latest

data/NEWS.md

@@ -1,5 +1,12 @@
 # ChangeLog
 
+## v2.6.2
+
+### Fixes
+
+* Fix unbounded growth of stream state in timeout handling (#150)
+* docs: clarify timeout handling to prevent silent log loss
+
 ## v2.6.1
 
 ### Fixes

data/README.md

@@ -1,6 +1,7 @@
 # fluent-plugin-concat
 
-[![Build Status](https://travis-ci.org/fluent-plugins-nursery/fluent-plugin-concat.svg?branch=master)](https://travis-ci.org/fluent-plugins-nursery/fluent-plugin-concat)
+[![Testing on Ubuntu](https://github.com/fluent-plugins-nursery/fluent-plugin-concat/actions/workflows/linux.yml/badge.svg)](https://github.com/fluent-plugins-nursery/fluent-plugin-concat/actions/workflows/linux.yml)
+[![Testing on Windows](https://github.com/fluent-plugins-nursery/fluent-plugin-concat/actions/workflows/windows.yml/badge.svg)](https://github.com/fluent-plugins-nursery/fluent-plugin-concat/actions/workflows/windows.yml)
 
 Fluentd Filter plugin to concatenate multiline log separated in multiple events.
 
@@ -123,40 +124,6 @@ Specify first line of multiline by regular expression.
 </filter>
 ```
 
-You can handle timeout events and remaining buffers on shutdown this plugin.
-
-```aconf
-<label @ERROR>
-  <match docker.log>
-    @type file
-    path /path/to/error.log
-  </match>
-</label>
-```
-
-Handle timeout log lines the same as normal logs.
-
-```aconf
-<filter **>
-  @type concat
-  key message
-  multiline_start_regexp /^Start/
-  flush_interval 5
-  timeout_label @NORMAL
-</filter>
-
-<match **>
-  @type relabel
-  @label @NORMAL
-</match>
-
-<label @NORMAL>
-  <match **>
-    @type stdout
-  </match>
-</label>
-```
-
 Handle single line JSON from Docker containers.
 
 ```aconf
@@ -251,6 +218,40 @@ Handle containerd/cri in Kubernetes.
 </label>
 ```
 
+### Timeout Handling and Preventing Log Loss
+
+When a multiline log is incomplete and reaches the `flush_interval`, the plugin flushes the remaining buffer. **These timeout events are routed to the `@ERROR` label by default**.
+
+**If you do not explicitly handle these timeout events, your last log lines will be silently dropped.**
+
+To prevent data loss and handle timeout logs the same as normal logs, you **must** configure `timeout_label` and set up a corresponding `<label>` block to catch and route them properly.
+
+#### Example: Safely rescuing timeout logs
+
+```aconf
+<filter **>
+  @type concat
+  key message
+  multiline_start_regexp /^Start/
+  flush_interval 5
+  # 1. Route timeout events to a specific label
+  timeout_label @NORMAL 
+</filter>
+
+# 2. Route normal events to the same label
+<match **>
+  @type relabel
+  @label @NORMAL
+</match>
+
+# 3. Handle both normal and timeout events together
+<label @NORMAL>
+  <match **>
+    @type stdout
+  </match>
+</label>
+```
+
 ## Contributing
 
 1. Fork it

data/fluent-plugin-concat.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-concat"
-  spec.version       = "2.6.1"
+  spec.version       = "2.6.2"
   spec.authors       = ["Kenji Okimoto"]
   spec.email         = ["okimoto@clear-code.com"]
 

data/lib/fluent/plugin/filter_concat.rb

@@ -433,20 +433,27 @@ module Fluent::Plugin
 
     def flush_timeout_buffer
       now = Fluent::Engine.now
-      timeout_stream_identities = []
+      expired_stream_identities = []
       @timeout_map_mutex.synchronize do
         @timeout_map.each do |stream_identity, previous_timestamp|
           next if @flush_interval > (now - previous_timestamp)
+          expired_stream_identities << stream_identity
           next if @buffer[stream_identity].empty?
           time, flushed_record = flush_buffer(stream_identity)
-          timeout_stream_identities << stream_identity
           tag = stream_identity.split(":").first
           message = "Timeout flush: #{stream_identity}"
           handle_timeout_error(tag, @use_first_timestamp ? time : now, flushed_record, message)
           log.info(message)
         end
-        @timeout_map.reject! do |stream_identity, _|
-          timeout_stream_identities.include?(stream_identity)
+        # Purge expired streams from all the state hashes, not only the
+        # flushed ones. Streams which completed normally leave empty entries
+        # behind, so @buffer/@buffer_size/@timeout_map would otherwise grow
+        # unboundedly (notably in partial metadata mode where every split
+        # message has a unique stream identity).
+        expired_stream_identities.each do |stream_identity|
+          @timeout_map.delete(stream_identity)
+          @buffer.delete(stream_identity)
+          @buffer_size.delete(stream_identity)
         end
       end
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-concat
 version: !ruby/object:Gem::Version
-  version: 2.6.1
+  version: 2.6.2
 platform: ruby
 authors:
 - Kenji Okimoto
@@ -155,7 +155,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 4.0.10
 specification_version: 4
 summary: Fluentd Filter plugin to concat multiple event messages
 test_files: []