logstash-filter-aggregate 2.9.2 → 2.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 15460afa4f78789d3f4ee56ae45e06e5f31b8ef4
-  data.tar.gz: f4bd241aa7207b29366d2456d6c42cb6ecc499c3
+SHA256:
+  metadata.gz: e58c8dca379dfba4361b616d161cd03b8c32211f48fd696b1acc977e4506c3d3
+  data.tar.gz: a6ec0abe65bb04db42f28c2db73095d5bf20f0f23c5c8710547a1834572975dc
 SHA512:
-  metadata.gz: 10f18942af2c7cd6f343502478d6d6d2ffc530700b9462e377770a0adbd6bb38b71c2a7debeeaba832aa0aafa9eba85acd041c35a468f2876ad61d0a51566f40
-  data.tar.gz: d6c92aa3b6bf04bfdd1be6e6b3fafbeb8b31416e20db4ee2c795d3fa20ac193db9a8422634b90df90efed88f135e5941a2f7373707b0e20be886a68a30abd8c7
+  metadata.gz: 222a895a6d0106f19bd642a131babb8f4ec2dee84fc1b608d901a3c1865e062e252dc966971e5adb71321128c7292a4d4dd8845b6eea60878d3011ccea968406
+  data.tar.gz: e4380712a5b97a69dd7d3e53e634f2fa8caed0fc373f9fdae899d9ae73b2d5e724c07883c1ad582c8851c1f51101cd96c528639a4c0c123fe2b169d762d77de1

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 2.11.0
+  - new feature: log a warning message when the number of tasks stored in memory exceeds the configured threshold (#125)
+
+## 2.10.0
+  - new feature: add ability to generate new event during code execution (#116)
+
 ## 2.9.2
   - bugfix: remove 'default_timeout' at pipeline level (fix #112)
   - ci: update travis ci configuration

data/docs/index.asciidoc

@@ -26,7 +26,7 @@ include::{include_path}/plugin_header.asciidoc[]
 The aim of this filter is to aggregate information available among several events (typically log lines) belonging to a same task,
 and finally push aggregated information into final task event.
 
-You should be very careful to set Logstash filter workers to 1 (`-w 1` flag) for this filter to work correctly
+You should be very careful to set Logstash filter workers to 1 (`-w 1` in [command-line flag](https://www.elastic.co/guide/en/logstash/current/running-logstash-command-line.html#command-line-flags)) for this filter to work correctly
 otherwise events may be processed out of sequence and unexpected results will occur.
 
 
@@ -364,6 +364,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-timeout_tags>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-timeout_task_id_field>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-timeout_timestamp_field>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-map_count_warning_threshold>> |<<number,number>>|No
 |=======================================================================
 
 Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
@@ -402,7 +403,7 @@ The code to execute to update aggregated map, using current event.
 
 Or on the contrary, the code to execute to update event, using aggregated map.
 
-Available variables are :
+Available variables are:
 
 `event`: current Logstash event
 
@@ -411,8 +412,11 @@ Available variables are :
 `map_meta`: meta informations associated to aggregate map. It allows to set a custom `timeout` or `inactivity_timeout`. 
 It allows also to get `creation_timestamp`, `lastevent_timestamp` and `task_id`.
 
+`new_event_block`: block used to emit new Logstash events. See the second example on how to use it.
+
 When option push_map_as_event_on_timeout=true, if you set `map_meta.timeout=0` in `code` block, then aggregated map is immediately pushed as a new event.
 
+
 Example:
 [source,ruby]
     filter {
@@ -421,6 +425,26 @@ Example:
       }
     }
 
+
+To create additional events during the code execution, to be emitted immediately, you can use `new_event_block.call(event)` function, like in the following example:
+
+[source,ruby]
+    filter {
+      aggregate {
+        code => "
+            data = {:my_sql_duration => map['sql_duration']}
+            generated_event = LogStash::Event.new(data)
+            generated_event.set('my_other_field', 34)
+            new_event_block.call(generated_event)
+        "
+      }
+    }
+
+The parameter of the function `new_event_block.call` must be of type `LogStash::Event`.  
+To create such an object, the constructor of the same class can be used: `LogStash::Event.new()`.  
+`LogStash::Event.new()` can receive a parameter of type ruby http://ruby-doc.org/core-1.9.1/Hash.html[Hash] to initialize the new event fields.
+
+
 [id="plugins-{type}s-{plugin}-end_of_task"]
 ===== `end_of_task`
 
@@ -591,6 +615,29 @@ Example:
       }
     }
 
+[id="plugins-{type}s-{plugin}-map_count_warning_threshold"]
+===== `map_count_warning_threshold`
+
+  * Value type is <<number,number>>
+  * Default value is `5000`
+
+Defines the threshold for the number of tasks in memory before a warning is logged.
+When the number of maps for a task_id pattern exceeds this threshold, a warning message is logged to help identify potential memory issues caused by unterminated tasks.
+
+The warning is repeated every 20% of the threshold events (e.g., with threshold 5000, warnings appear every 1000 events while above threshold).
+
+Set to `0` to disable memory warnings.
+
+Example:
+[source,ruby]
+    filter {
+      aggregate {
+        task_id => "%{taskid}"
+        code => "map['count'] ||= 0; map['count'] += 1"
+        map_count_warning_threshold => 10000
+      }
+    }
+
 
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]

data/lib/logstash/filters/aggregate.rb

@@ -42,6 +42,8 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
 
   config :timeout_tags, :validate => :array, :required => false, :default => []
 
+  config :map_count_warning_threshold, :validate => :number, :required => false, :default => 5000
+
 
   # ################## #
   # INSTANCE VARIABLES #
@@ -62,6 +64,9 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
   # Default timeout (in seconds) when not defined in plugin configuration
   DEFAULT_TIMEOUT = 1800
 
+  # Warning frequency divisor: warn every (threshold / divisor) events when above threshold
+  WARNING_FREQUENCY_DIVISOR = 5
+
   # Store all shared aggregate attributes per pipeline id
   @@pipelines = {}
 
@@ -83,7 +88,7 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
     end
 
     # process lambda expression to call in each filter call
-    eval("@codeblock = lambda { |event, map, map_meta| #{@code} }", binding, "(aggregate filter code)")
+    eval("@codeblock = lambda { |event, map, map_meta, &new_event_block| #{@code} }", binding, "(aggregate filter code)")
 
     # process lambda expression to call in the timeout case or previous event case
     if @timeout_code
@@ -138,6 +143,9 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
       @current_pipeline.aggregate_maps[@task_id] ||= {}
       update_aggregate_maps_metric()
 
+      # calculate warning frequency (warn every threshold/divisor events, minimum 1)
+      @map_count_warning_frequency = [@map_count_warning_threshold / WARNING_FREQUENCY_DIVISOR, 1].max
+
     end
   end
 
@@ -168,7 +176,7 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
 
   # This method is invoked each time an event matches the filter
   public
-  def filter(event)
+  def filter(event, &new_event_block)
 
     # define task id
     task_id = event.sprintf(@task_id)
@@ -180,6 +188,8 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
     # protect aggregate_maps against concurrent access, using a mutex
     @current_pipeline.mutex.synchronize do
 
+      check_map_count_warning()
+
       # if timeout is based on event timestamp, check if task_id map is expired and should be removed
       if @timeout_timestamp_field
         event_to_yield = remove_expired_map_based_on_event_timestamp(task_id, event)
@@ -213,7 +223,7 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
       # execute the code to read/update map and event
       map = aggregate_maps_element.map
       begin
-        @codeblock.call(event, map, aggregate_maps_element)
+        @codeblock.call(event, map, aggregate_maps_element, &new_event_block)
         @logger.debug("Aggregate successful filter code execution", :code => @code)
         noError = true
       rescue => exception
@@ -485,6 +495,26 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
     end
   end
 
+  # checks if map count exceeds warning threshold and logs a warning if so
+  def check_map_count_warning()
+    return if @map_count_warning_threshold == 0
+
+    map_count = @current_pipeline.aggregate_maps[@task_id].length
+    if map_count >= @map_count_warning_threshold
+      @events_since_last_warning ||= 0
+      if @events_since_last_warning == 0
+        @logger.warn("Aggregate filter memory warning: task_id pattern '#{@task_id}' has #{map_count} maps in memory (threshold: #{@map_count_warning_threshold}).",
+                     :task_id_pattern => @task_id,
+                     :map_count => map_count,
+                     :threshold => @map_count_warning_threshold)
+      end
+      @events_since_last_warning = (@events_since_last_warning + 1) % @map_count_warning_frequency
+    else
+      @events_since_last_warning = 0
+    end
+
+  end
+
 end # class LogStash::Filters::Aggregate
 
 # Element of "aggregate_maps"

data/logstash-filter-aggregate.gemspec

@@ -1,8 +1,8 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-filter-aggregate'
-  s.version         = '2.9.2'
-  s.licenses = ['Apache License (2.0)']
-  s.summary = "Aggregates information from several events originating with a single task"
+  s.version = '2.11.0'
+  s.licenses = ['Apache-2.0']
+  s.summary = 'Aggregates information from several events originating with a single task'
   s.description = 'This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program'
   s.authors = ['Elastic', 'Fabien Baligand']
   s.email = 'info@elastic.co'

data/spec/filters/aggregate_spec.rb

@@ -420,4 +420,50 @@ describe LogStash::Filters::Aggregate do
     end
   end
 
-end
+  context "Custom event generation code is used" do
+    describe "when a new event is manually generated" do
+      it "should push a new event immediately" do
+        agg_filter = setup_filter({ "task_id" => "%{task_id}", "code" => "map['sql_duration'] = 2; new_event_block.call(LogStash::Event.new({:my_sql_duration => map['sql_duration']}))", "timeout" => 120 })
+        agg_filter.filter(event({"task_id" => "1"})) do |yield_event|
+          expect(yield_event).not_to be_nil
+          expect(yield_event.get("my_sql_duration")).to eq(2)
+        end
+      end
+    end
+
+  end
+
+  context "map_count_warning_threshold option" do
+    describe "when threshold is set to 0 (disabled)" do
+      it "should not log any warning" do
+        filter = setup_filter({ "task_id" => "%{warn_id}", "code" => "", "map_count_warning_threshold" => 0 })
+        expect(filter.logger).not_to receive(:warn)
+        3.times { |i| filter.filter(event({"warn_id" => "task_#{i}"})) }
+      end
+    end
+
+    describe "when map count reaches threshold" do
+      it "should log a warning" do
+        filter = setup_filter({ "task_id" => "%{warn_id}", "code" => "", "map_count_warning_threshold" => 3 })
+        expect(filter.logger).to receive(:warn).with(
+          /Aggregate filter memory warning.*has 3 maps in memory/,
+          hash_including(:task_id_pattern => "%{warn_id}", :map_count => 3, :threshold => 3)
+        ).once
+        4.times { |i| filter.filter(event({"warn_id" => "task_#{i}"})) }
+      end
+
+      it "should log warning every 20% of map_count_warning_threshold occurrences" do
+        filter = setup_filter({ "task_id" => "%{warn_id}", "code" => "", "map_count_warning_threshold" => 5 })
+        expect(filter.logger).to receive(:warn).twice
+        7.times { |i| filter.filter(event({"warn_id" => "task_#{i}"})) }
+      end
+    end
+
+    describe "when using default threshold" do
+      it "should have default threshold of 5000" do
+        filter = setup_filter({ "task_id" => "%{warn_id}", "code" => "" })
+        expect(filter.map_count_warning_threshold).to eq(5000)
+      end
+    end
+  end
+end

metadata

@@ -1,17 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-aggregate
 version: !ruby/object:Gem::Version
-  version: 2.9.2
+  version: 2.11.0
 platform: ruby
 authors:
 - Elastic
 - Fabien Baligand
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-25 00:00:00.000000000 Z
+date: 2026-02-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -20,9 +20,8 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
-  name: logstash-core-plugin-api
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -32,20 +31,22 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.99'
 - !ruby/object:Gem::Dependency
+  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
@@ -65,11 +66,10 @@ files:
 - spec/filters/aggregate_spec_helper.rb
 homepage: https://github.com/logstash-plugins/logstash-filter-aggregate
 licenses:
-- Apache License (2.0)
+- Apache-2.0
 metadata:
   logstash_plugin: 'true'
   logstash_group: filter
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -84,9 +84,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.4.8
-signing_key:
+rubygems_version: 3.6.3
 specification_version: 4
 summary: Aggregates information from several events originating with a single task
 test_files: