logstash-filter-aggregate 2.10.0 → 2.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: e30c90c81bac3cd99cf2a01d8e66e010a89acf87
-  data.tar.gz: 5733c4b4a64b9a6b7032fd093f289ad0d2e84f2e
+SHA256:
+  metadata.gz: e58c8dca379dfba4361b616d161cd03b8c32211f48fd696b1acc977e4506c3d3
+  data.tar.gz: a6ec0abe65bb04db42f28c2db73095d5bf20f0f23c5c8710547a1834572975dc
 SHA512:
-  metadata.gz: 809dbbdba440d501fb460ab5d60cbaaabb397654a054cd668a2d38ed1de31b99876839896cbd0c16634572d702ba8e944f094616d95e916935009a3290de8d36
-  data.tar.gz: 6cb68ae7e22433d0bc0a49803a95e1836de98f8bb36c937812b2bbb6981d5624590df1ce27694ee34cb7fef53774ab18e9b823b0253b62c3687ea8f83895b82e
+  metadata.gz: 222a895a6d0106f19bd642a131babb8f4ec2dee84fc1b608d901a3c1865e062e252dc966971e5adb71321128c7292a4d4dd8845b6eea60878d3011ccea968406
+  data.tar.gz: e4380712a5b97a69dd7d3e53e634f2fa8caed0fc373f9fdae899d9ae73b2d5e724c07883c1ad582c8851c1f51101cd96c528639a4c0c123fe2b169d762d77de1

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 2.11.0
+  - new feature: log a warning message when the number of tasks stored in memory exceeds the configured threshold (#125)
+
 ## 2.10.0
   - new feature: add ability to generate new event during code execution (#116)
 

data/docs/index.asciidoc

@@ -26,7 +26,7 @@ include::{include_path}/plugin_header.asciidoc[]
 The aim of this filter is to aggregate information available among several events (typically log lines) belonging to a same task,
 and finally push aggregated information into final task event.
 
-You should be very careful to set Logstash filter workers to 1 (`-w 1` flag) for this filter to work correctly
+You should be very careful to set Logstash filter workers to 1 (`-w 1` in [command-line flag](https://www.elastic.co/guide/en/logstash/current/running-logstash-command-line.html#command-line-flags)) for this filter to work correctly
 otherwise events may be processed out of sequence and unexpected results will occur.
 
 
@@ -364,6 +364,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-timeout_tags>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-timeout_task_id_field>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-timeout_timestamp_field>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-map_count_warning_threshold>> |<<number,number>>|No
 |=======================================================================
 
 Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
@@ -614,6 +615,29 @@ Example:
       }
     }
 
+[id="plugins-{type}s-{plugin}-map_count_warning_threshold"]
+===== `map_count_warning_threshold`
+
+  * Value type is <<number,number>>
+  * Default value is `5000`
+
+Defines the threshold for the number of tasks in memory before a warning is logged.
+When the number of maps for a task_id pattern exceeds this threshold, a warning message is logged to help identify potential memory issues caused by unterminated tasks.
+
+The warning is repeated every 20% of the threshold events (e.g., with threshold 5000, warnings appear every 1000 events while above threshold).
+
+Set to `0` to disable memory warnings.
+
+Example:
+[source,ruby]
+    filter {
+      aggregate {
+        task_id => "%{taskid}"
+        code => "map['count'] ||= 0; map['count'] += 1"
+        map_count_warning_threshold => 10000
+      }
+    }
+
 
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]

data/lib/logstash/filters/aggregate.rb

@@ -42,6 +42,8 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
 
   config :timeout_tags, :validate => :array, :required => false, :default => []
 
+  config :map_count_warning_threshold, :validate => :number, :required => false, :default => 5000
+
 
   # ################## #
   # INSTANCE VARIABLES #
@@ -62,6 +64,9 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
   # Default timeout (in seconds) when not defined in plugin configuration
   DEFAULT_TIMEOUT = 1800
 
+  # Warning frequency divisor: warn every (threshold / divisor) events when above threshold
+  WARNING_FREQUENCY_DIVISOR = 5
+
   # Store all shared aggregate attributes per pipeline id
   @@pipelines = {}
 
@@ -138,6 +143,9 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
       @current_pipeline.aggregate_maps[@task_id] ||= {}
       update_aggregate_maps_metric()
 
+      # calculate warning frequency (warn every threshold/divisor events, minimum 1)
+      @map_count_warning_frequency = [@map_count_warning_threshold / WARNING_FREQUENCY_DIVISOR, 1].max
+
     end
   end
 
@@ -180,6 +188,8 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
     # protect aggregate_maps against concurrent access, using a mutex
     @current_pipeline.mutex.synchronize do
 
+      check_map_count_warning()
+
       # if timeout is based on event timestamp, check if task_id map is expired and should be removed
       if @timeout_timestamp_field
         event_to_yield = remove_expired_map_based_on_event_timestamp(task_id, event)
@@ -485,6 +495,26 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
     end
   end
 
+  # checks if map count exceeds warning threshold and logs a warning if so
+  def check_map_count_warning()
+    return if @map_count_warning_threshold == 0
+
+    map_count = @current_pipeline.aggregate_maps[@task_id].length
+    if map_count >= @map_count_warning_threshold
+      @events_since_last_warning ||= 0
+      if @events_since_last_warning == 0
+        @logger.warn("Aggregate filter memory warning: task_id pattern '#{@task_id}' has #{map_count} maps in memory (threshold: #{@map_count_warning_threshold}).",
+                     :task_id_pattern => @task_id,
+                     :map_count => map_count,
+                     :threshold => @map_count_warning_threshold)
+      end
+      @events_since_last_warning = (@events_since_last_warning + 1) % @map_count_warning_frequency
+    else
+      @events_since_last_warning = 0
+    end
+
+  end
+
 end # class LogStash::Filters::Aggregate
 
 # Element of "aggregate_maps"

data/logstash-filter-aggregate.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-filter-aggregate'
-  s.version = '2.10.0'
+  s.version = '2.11.0'
   s.licenses = ['Apache-2.0']
   s.summary = 'Aggregates information from several events originating with a single task'
   s.description = 'This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program'

data/spec/filters/aggregate_spec.rb

@@ -433,4 +433,37 @@ describe LogStash::Filters::Aggregate do
 
   end
 
+  context "map_count_warning_threshold option" do
+    describe "when threshold is set to 0 (disabled)" do
+      it "should not log any warning" do
+        filter = setup_filter({ "task_id" => "%{warn_id}", "code" => "", "map_count_warning_threshold" => 0 })
+        expect(filter.logger).not_to receive(:warn)
+        3.times { |i| filter.filter(event({"warn_id" => "task_#{i}"})) }
+      end
+    end
+
+    describe "when map count reaches threshold" do
+      it "should log a warning" do
+        filter = setup_filter({ "task_id" => "%{warn_id}", "code" => "", "map_count_warning_threshold" => 3 })
+        expect(filter.logger).to receive(:warn).with(
+          /Aggregate filter memory warning.*has 3 maps in memory/,
+          hash_including(:task_id_pattern => "%{warn_id}", :map_count => 3, :threshold => 3)
+        ).once
+        4.times { |i| filter.filter(event({"warn_id" => "task_#{i}"})) }
+      end
+
+      it "should log warning every 20% of map_count_warning_threshold occurrences" do
+        filter = setup_filter({ "task_id" => "%{warn_id}", "code" => "", "map_count_warning_threshold" => 5 })
+        expect(filter.logger).to receive(:warn).twice
+        7.times { |i| filter.filter(event({"warn_id" => "task_#{i}"})) }
+      end
+    end
+
+    describe "when using default threshold" do
+      it "should have default threshold of 5000" do
+        filter = setup_filter({ "task_id" => "%{warn_id}", "code" => "" })
+        expect(filter.map_count_warning_threshold).to eq(5000)
+      end
+    end
+  end
 end

metadata

@@ -1,17 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-aggregate
 version: !ruby/object:Gem::Version
-  version: 2.10.0
+  version: 2.11.0
 platform: ruby
 authors:
 - Elastic
 - Fabien Baligand
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-10-11 00:00:00.000000000 Z
+date: 2026-02-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -20,9 +20,8 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
-  name: logstash-core-plugin-api
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -32,14 +31,14 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.99'
 - !ruby/object:Gem::Dependency
+  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -71,7 +70,6 @@ licenses:
 metadata:
   logstash_plugin: 'true'
   logstash_group: filter
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -86,9 +84,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.14.1
-signing_key:
+rubygems_version: 3.6.3
 specification_version: 4
 summary: Aggregates information from several events originating with a single task
 test_files: