logstash-filter-aggregate 2.7.2 → 2.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: da3a7caa73ead37f402a19b71ebd6793fb438f17
-  data.tar.gz: 3e230ab339bb070ad94bf9312866619c65ead910
+  metadata.gz: 8cbb27a58c0339ae9f7908d18250a5f91c456f6f
+  data.tar.gz: e86b38aa410918fb62cf7050be5fd8b9f44d6ed9
 SHA512:
-  metadata.gz: bde2b92c6b4f956294b2a8d430e103b4f012022739ddd66a12df17acd23748ba8d68204a1fc99b0d60093e69ce36c1af9fe59e9de6cb4fafd2e7644e43805532
-  data.tar.gz: dee95f8441d5459efa28fb07d9e82aeebae150747d30eb1ea1a65bef7a5045e430b0a3e3dbc67d6060988a65fa951d1cf2ac9d16c5fc8580c4704096fdaec2fc
+  metadata.gz: d14e1e14cdf0342db92e06ceac124f4c0d384183e4074f7da55b87e27d117d4feddc63844b3a7b06b196d704154a44005bae18e8b3f135d8008f877ef6bafe76
+  data.tar.gz: 090d1e070fab264a8f4b1c7150e6d4161e357ab146cc0303480cc6ee0aebdcba80659f9a7dba02ffadf0939f47140a7038f83aa571b9053e2cd9d1fb58a2b91f

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 2.8.0
+  - new feature: add 'timeout_timestamp_field' option.  
+    When set, this option lets to compute timeout based on event timestamp field (and not system time). It's particularly useful when processing old logs.
+
 ## 2.7.2
   - bugfix: fix synchronisation issue at Logstash shutdown (#75)
 
@@ -24,7 +28,8 @@
 - docs: bump patch level for doc build
 
 ## 2.6.0
-- new feature: 'inactivity_timeout'. Events for a given `task_id` will be aggregated for as long as they keep arriving within the defined `inactivity_timeout` option - the inactivity timeout is reset each time a new event happens. On the contrary, `timeout` is never reset and happens after `timeout` seconds since aggregation map creation.
+- new feature: add 'inactivity_timeout' option.  
+  Events for a given `task_id` will be aggregated for as long as they keep arriving within the defined `inactivity_timeout` option - the inactivity timeout is reset each time a new event happens. On the contrary, `timeout` is never reset and happens after `timeout` seconds since aggregation map creation.
 
 ## 2.5.2
 - bugfix: fix 'aggregate_maps_path' load (issue #62). Re-start of Logstash died when no data were provided in 'aggregate_maps_path' file for some aggregate task_id patterns

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012-2015 Elasticsearch <http://www.elasticsearch.org>
+Copyright (c) 2012-2018 Elasticsearch <http://www.elasticsearch.org>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/docs/index.asciidoc

@@ -328,7 +328,7 @@ filter {
 * an aggregate map is tied to one task_id value which is tied to one task_id pattern. So if you have 2 filters with different task_id patterns, even if you have same task_id value, they won't share the same aggregate map.
 * in one filter configuration, it is recommanded to define a timeout option to protect the feature against unterminated tasks. It tells the filter to delete expired maps
 * if no timeout is defined, by default, all maps older than 1800 seconds are automatically deleted
-* all timeout options have to be defined in only one aggregate filter per task_id pattern (per pipeline). Timeout options are : timeout, inactivity_timeout, timeout_code, push_map_as_event_on_timeout, push_previous_map_as_event, timeout_task_id_field, timeout_tags
+* all timeout options have to be defined in only one aggregate filter per task_id pattern (per pipeline). Timeout options are : timeout, inactivity_timeout, timeout_code, push_map_as_event_on_timeout, push_previous_map_as_event, timeout_timestamp_field, timeout_task_id_field, timeout_tags
 * if `code` execution raises an exception, the error is logged and event is tagged '_aggregateexception'
 
 
@@ -362,6 +362,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-timeout_code>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-timeout_tags>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-timeout_task_id_field>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-timeout_timestamp_field>> |<<string,string>>|No
 |=======================================================================
 
 Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
@@ -532,7 +533,7 @@ Example:
 [source,ruby]
     filter {
       aggregate {
-        timeout_tags => ["aggregate_timeout']
+        timeout_tags => ["aggregate_timeout"]
       }
     }
 
@@ -542,13 +543,43 @@ Example:
   * Value type is <<string,string>>
   * There is no default value for this setting.
 
-This option indicates the timeout generated event's field for the "task_id" value.
-The task id will then be set into the timeout event. This can help correlate which tasks have been timed out.
-
-For example, with option `timeout_task_id_field => "my_id"` ,when timeout task id is `"12345"`, the generated timeout event will contain `'my_id' => '12345'`.
+This option indicates the timeout generated event's field where the current "task_id" value will be set.
+This can help to correlate which tasks have been timed out.
 
 By default, if this option is not set, task id value won't be set into timeout generated event.
 
+Example:
+[source,ruby]
+    filter {
+      aggregate {
+        timeout_task_id_field => "task_id"
+      }
+    }
+
+[id="plugins-{type}s-{plugin}-timeout_timestamp_field"]
+===== `timeout_timestamp_field`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+By default, timeout is computed using system time, where Logstash is running.
+
+When this option is set, timeout is computed using event timestamp field indicated in this option. 
+It means that when a first event arrives on aggregate filter and induces a map creation, map creation time will be equal to this event timestamp. 
+Then, each time a new event arrives on aggregate filter, event timestamp is compared to map creation time to check if timeout happened.
+
+This option is particularly useful when processing old logs with option `push_map_as_event_on_timeout => true`. 
+It lets to generate aggregated events based on timeout on old logs, where system time is inappropriate.
+
+Warning : so that this option works fine, it must be set on first aggregate filter.
+
+Example:
+[source,ruby]
+    filter {
+      aggregate {
+        timeout_timestamp_field => "@timestamp"
+      }
+    }
 
 
 [id="plugins-{type}s-{plugin}-common-options"]

data/lib/logstash/filters/aggregate.rb

@@ -36,6 +36,8 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
 
   config :push_previous_map_as_event, :validate => :boolean, :required => false, :default => false
 
+  config :timeout_timestamp_field, :validate => :string, :required => false
+
   config :timeout_task_id_field, :validate => :string, :required => false
 
   config :timeout_tags, :validate => :array, :required => false, :default => []
@@ -44,7 +46,7 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
   # ################## #
   # INSTANCE VARIABLES #
   # ################## #
-  
+
 
   # pointer to current pipeline context
   attr_accessor :current_pipeline
@@ -57,7 +59,7 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
 
   # Default timeout (in seconds) when not defined in plugin configuration
   DEFAULT_TIMEOUT = 1800
-  
+
   # Store all shared aggregate attributes per pipeline id
   @@pipelines = {}
 
@@ -77,7 +79,7 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
     if !@task_id.match(/%\{.+\}/)
       raise LogStash::ConfigurationError, "Aggregate plugin: task_id pattern '#{@task_id}' must contain a dynamic expression like '%{field}'"
     end
-    
+
     # process lambda expression to call in each filter call
     eval("@codeblock = lambda { |event, map| #{@code} }", binding, "(aggregate filter code)")
 
@@ -87,7 +89,7 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
     end
 
     # init pipeline context
-    @@pipelines[pipeline_id] ||= LogStash::Filters::Aggregate::Pipeline.new();
+    @@pipelines[pipeline_id] ||= LogStash::Filters::Aggregate::Pipeline.new()
     @current_pipeline = @@pipelines[pipeline_id]
 
     @current_pipeline.mutex.synchronize do
@@ -103,23 +105,23 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
       end
 
       # timeout management : define default_timeout
-      if !@timeout.nil? && (@current_pipeline.default_timeout.nil? || @timeout < @current_pipeline.default_timeout)
+      if @timeout && (@current_pipeline.default_timeout.nil? || @timeout < @current_pipeline.default_timeout)
         @current_pipeline.default_timeout = @timeout
         @logger.debug("Aggregate default timeout: #{@timeout} seconds")
       end
 
       # inactivity timeout management: make sure it is lower than timeout
-      if !@inactivity_timeout.nil? && ((!@timeout.nil? && @inactivity_timeout > @timeout) || (!@current_pipeline.default_timeout.nil? && @inactivity_timeout > @current_pipeline.default_timeout))
+      if @inactivity_timeout && ((@timeout && @inactivity_timeout > @timeout) || (@current_pipeline.default_timeout && @inactivity_timeout > @current_pipeline.default_timeout))
         raise LogStash::ConfigurationError, "Aggregate plugin: For task_id pattern #{@task_id}, inactivity_timeout must be lower than timeout"
       end
 
       # reinit pipeline_close_instance (if necessary)
-      if !@current_pipeline.aggregate_maps_path_set && !@current_pipeline.pipeline_close_instance.nil?
+      if !@current_pipeline.aggregate_maps_path_set && @current_pipeline.pipeline_close_instance
         @current_pipeline.pipeline_close_instance = nil
       end
 
       # check if aggregate_maps_path option has already been set on another instance else set @current_pipeline.aggregate_maps_path_set
-      if !@aggregate_maps_path.nil?
+      if @aggregate_maps_path
         if @current_pipeline.aggregate_maps_path_set
           @current_pipeline.aggregate_maps_path_set = false
           raise LogStash::ConfigurationError, "Aggregate plugin: Option 'aggregate_maps_path' must be set on only one aggregate filter"
@@ -130,7 +132,7 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
       end
 
       # load aggregate maps from file (if option defined)
-      if !@aggregate_maps_path.nil? && File.exist?(@aggregate_maps_path)
+      if @aggregate_maps_path && File.exist?(@aggregate_maps_path)
         File.open(@aggregate_maps_path, "r") { |from_file| @current_pipeline.aggregate_maps.merge!(Marshal.load(from_file)) }
         File.delete(@aggregate_maps_path)
         @logger.info("Aggregate maps loaded from : #{@aggregate_maps_path}")
@@ -138,8 +140,7 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
 
       # init aggregate_maps
       @current_pipeline.aggregate_maps[@task_id] ||= {}
-      
-      
+
     end
   end
 
@@ -156,7 +157,7 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
       # store aggregate maps to file (if option defined)
       @current_pipeline.mutex.synchronize do
         @current_pipeline.aggregate_maps.delete_if { |key, value| value.empty? }
-        if !@aggregate_maps_path.nil? && !@current_pipeline.aggregate_maps.empty?
+        if @aggregate_maps_path && !@current_pipeline.aggregate_maps.empty?
           File.open(@aggregate_maps_path, "w"){ |to_file| Marshal.dump(@current_pipeline.aggregate_maps, to_file) }
           @logger.info("Aggregate maps stored to : #{@aggregate_maps_path}")
         end
@@ -182,26 +183,36 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
     # protect aggregate_maps against concurrent access, using a mutex
     @current_pipeline.mutex.synchronize do
 
+      # if timeout is based on event timestamp, check if task_id map is expired and should be removed
+      if @timeout_timestamp_field
+        event_to_yield = remove_expired_map_based_on_event_timestamp(task_id, event)
+      end
+
       # retrieve the current aggregate map
       aggregate_maps_element = @current_pipeline.aggregate_maps[@task_id][task_id]
 
-
-      # create aggregate map, if it doesn't exist
+      # case where aggregate map isn't already created
       if aggregate_maps_element.nil?
         return if @map_action == "update"
+
         # create new event from previous map, if @push_previous_map_as_event is enabled
         if @push_previous_map_as_event && !@current_pipeline.aggregate_maps[@task_id].empty?
           event_to_yield = extract_previous_map_as_event()
         end
-        aggregate_maps_element = LogStash::Filters::Aggregate::Element.new(Time.now);
+
+        # create aggregate map
+        creation_timestamp = reference_timestamp(event)
+        aggregate_maps_element = LogStash::Filters::Aggregate::Element.new(creation_timestamp)
         @current_pipeline.aggregate_maps[@task_id][task_id] = aggregate_maps_element
       else
         return if @map_action == "create"
       end
-      map = aggregate_maps_element.map
+
       # update last event timestamp
-      aggregate_maps_element.lastevent_timestamp = Time.now
+      aggregate_maps_element.lastevent_timestamp = reference_timestamp(event)
+
       # execute the code to read/update map and event
+      map = aggregate_maps_element.map
       begin
         @codeblock.call(event, map)
         @logger.debug("Aggregate successful filter code execution", :code => @code)
@@ -224,8 +235,7 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
     filter_matched(event) if noError
 
     # yield previous map as new event if set
-    yield event_to_yield unless event_to_yield.nil?
-
+    yield event_to_yield if event_to_yield
   end
 
   # Create a new event from the aggregation_map and the corresponding task_id
@@ -279,23 +289,11 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
 
     @logger.debug("Aggregate flush call with #{options}")
 
-    # Protection against no timeout defined by Logstash conf : define a default eviction instance with timeout = DEFAULT_TIMEOUT seconds
-    if @current_pipeline.default_timeout.nil?
-      @current_pipeline.default_timeout = DEFAULT_TIMEOUT
-    end
-    if !@current_pipeline.flush_instance_map.has_key?(@task_id)
-      @current_pipeline.flush_instance_map[@task_id] = self
-      @timeout = @current_pipeline.default_timeout
-    elsif @current_pipeline.flush_instance_map[@task_id].timeout.nil?
-      @current_pipeline.flush_instance_map[@task_id].timeout = @current_pipeline.default_timeout
-    end
-
-    if @current_pipeline.flush_instance_map[@task_id].inactivity_timeout.nil?
-      @current_pipeline.flush_instance_map[@task_id].inactivity_timeout = @current_pipeline.flush_instance_map[@task_id].timeout
-    end
-
-    # Launch timeout management only every interval of (@inactivity_timeout / 2) seconds or at Logstash shutdown
-    if @current_pipeline.flush_instance_map[@task_id] == self && !@current_pipeline.aggregate_maps[@task_id].nil? && (!@current_pipeline.last_flush_timestamp_map.has_key?(@task_id) || Time.now > @current_pipeline.last_flush_timestamp_map[@task_id] + @inactivity_timeout / 2 || options[:final])
+    # init flush/timeout properties for current pipeline
+    init_pipeline_timeout_management()
+    
+    # launch timeout management only every interval of (@inactivity_timeout / 2) seconds or at Logstash shutdown
+    if @current_pipeline.flush_instance_map[@task_id] == self && @current_pipeline.aggregate_maps[@task_id] && (!@current_pipeline.last_flush_timestamp_map.has_key?(@task_id) || Time.now > @current_pipeline.last_flush_timestamp_map[@task_id] + @inactivity_timeout / 2 || options[:final])
       events_to_flush = remove_expired_maps()
 
       # at Logstash shutdown, if push_previous_map_as_event is enabled, it's important to force flush (particularly for jdbc input plugin)
@@ -318,9 +316,32 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
     else
       return []
     end
-
   end
+  
+  # init flush/timeout properties for current pipeline
+  def init_pipeline_timeout_management()
+    
+    # Define default timeout (if not defined by user)
+    if @current_pipeline.default_timeout.nil?
+      @current_pipeline.default_timeout = DEFAULT_TIMEOUT
+    end
+    
+    # Define default flush instance that manages timeout (if not defined by user)
+    if !@current_pipeline.flush_instance_map.has_key?(@task_id)
+      @current_pipeline.flush_instance_map[@task_id] = self
+    end
 
+    # Define timeout and inactivity_timeout (if not defined by user)
+    if @current_pipeline.flush_instance_map[@task_id] == self
+      if @timeout.nil?
+        @timeout = @current_pipeline.default_timeout
+      end
+      if @inactivity_timeout.nil?
+        @inactivity_timeout = @timeout
+      end
+    end
+
+  end
 
   # Remove the expired Aggregate maps from @current_pipeline.aggregate_maps if they are older than timeout or if no new event has been received since inactivity_timeout.
   # If @push_previous_map_as_event option is set, or @push_map_as_event_on_timeout is set, expired maps are returned as new events to be flushed to Logstash pipeline.
@@ -334,7 +355,7 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
       @logger.debug("Aggregate remove_expired_maps call with '#{@task_id}' pattern and #{@current_pipeline.aggregate_maps[@task_id].length} maps")
 
       @current_pipeline.aggregate_maps[@task_id].delete_if do |key, element|
-        if element.creation_timestamp < min_timestamp || element.lastevent_timestamp < min_inactivity_timestamp
+        if element.creation_timestamp + element.difference_from_creation_to_now < min_timestamp || element.lastevent_timestamp + element.difference_from_creation_to_now < min_inactivity_timestamp
           if @push_previous_map_as_event || @push_map_as_event_on_timeout
             events_to_flush << create_timeout_event(element.map, key)
           end
@@ -347,6 +368,33 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
     return events_to_flush
   end
 
+  # Remove the expired Aggregate map associated to task_id if it is older than timeout or if no new event has been received since inactivity_timeout (relative to current event timestamp).
+  # If @push_previous_map_as_event option is set, or @push_map_as_event_on_timeout is set, expired map is returned as new event to be flushed to Logstash pipeline.
+  def remove_expired_map_based_on_event_timestamp(task_id, event)
+
+    @logger.debug("Aggregate remove_expired_map_based_on_event_timestamp call with task_id : '#{@task_id}'")
+
+    # get aggregate map element
+    element = @current_pipeline.aggregate_maps[@task_id][task_id]
+    return nil if element.nil?
+
+    init_pipeline_timeout_management()
+
+    event_to_flush = nil
+    event_timestamp = reference_timestamp(event)
+    min_timestamp = event_timestamp - @timeout
+    min_inactivity_timestamp = event_timestamp - @inactivity_timeout
+
+    if element.creation_timestamp < min_timestamp || element.lastevent_timestamp < min_inactivity_timestamp
+      if @push_previous_map_as_event || @push_map_as_event_on_timeout
+        event_to_flush = create_timeout_event(element.map, task_id)
+      end
+      @current_pipeline.aggregate_maps[@task_id].delete(task_id)
+    end
+
+    return event_to_flush
+  end
+
   # return if this filter instance has any timeout option enabled in logstash configuration
   def has_timeout_options?()
     return (
@@ -355,6 +403,7 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
       timeout_code ||
       push_map_as_event_on_timeout ||
       push_previous_map_as_event ||
+      timeout_timestamp_field ||
       timeout_task_id_field ||
       !timeout_tags.empty?
     )
@@ -368,11 +417,12 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
       "timeout_code",
       "push_map_as_event_on_timeout",
       "push_previous_map_as_event",
+      "timeout_timestamp_field",
       "timeout_task_id_field",
       "timeout_tags"
     ].join(", ")
   end
-  
+
   # return current pipeline id
   def pipeline_id()
     if @execution_context
@@ -382,45 +432,52 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
     end
   end
 
+  # compute and return "reference" timestamp to compute timeout :
+  # by default "system current time" or event timestamp if timeout_timestamp_field option is defined
+  def reference_timestamp(event)
+    return (@timeout_timestamp_field) ? event.get(@timeout_timestamp_field).time : Time.now
+  end
+
 end # class LogStash::Filters::Aggregate
 
 # Element of "aggregate_maps"
 class LogStash::Filters::Aggregate::Element
 
-  attr_accessor :creation_timestamp, :lastevent_timestamp, :map
+  attr_accessor :creation_timestamp, :lastevent_timestamp, :difference_from_creation_to_now, :map
 
   def initialize(creation_timestamp)
     @creation_timestamp = creation_timestamp
     @lastevent_timestamp = creation_timestamp
+    @difference_from_creation_to_now = (Time.now - creation_timestamp).to_i
     @map = {}
   end
 end
 
 # shared aggregate attributes for each pipeline
 class LogStash::Filters::Aggregate::Pipeline
-  
+
   attr_accessor :aggregate_maps, :mutex, :default_timeout, :flush_instance_map, :last_flush_timestamp_map, :aggregate_maps_path_set, :pipeline_close_instance
 
   def initialize()
     # Stores all aggregate maps, per task_id pattern, then per task_id value
     @aggregate_maps = {}
-  
+
     # Mutex used to synchronize access to 'aggregate_maps'
     @mutex = Mutex.new
-  
+
     # Default timeout for task_id patterns where timeout is not defined in Logstash filter configuration
     @default_timeout = nil
-  
+
     # For each "task_id" pattern, defines which Aggregate instance will process flush() call, processing expired Aggregate elements (older than timeout)
     # For each entry, key is "task_id pattern" and value is "aggregate instance"
     @flush_instance_map = {}
-  
+
     # last time where timeout management in flush() method was launched, per "task_id" pattern
     @last_flush_timestamp_map = {}
-  
+
     # flag indicating if aggregate_maps_path option has been already set on one aggregate instance
     @aggregate_maps_path_set = false
-  
+
     # defines which Aggregate instance will close Aggregate variables associated to current pipeline
     @pipeline_close_instance = nil
   end

data/logstash-filter-aggregate.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-filter-aggregate'
-  s.version         = '2.7.2'
+  s.version         = '2.8.0'
   s.licenses = ['Apache License (2.0)']
   s.summary = "Aggregates information from several events originating with a single task"
   s.description = 'This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program'

data/spec/filters/aggregate_spec.rb

@@ -163,6 +163,7 @@ describe LogStash::Filters::Aggregate do
     describe "no timeout defined in none filter" do
       it "defines a default timeout on a default filter" do
         reset_timeout_management()
+        @end_filter.timeout = nil
         expect(taskid_eviction_instance).to be_nil
         @end_filter.flush()
         expect(taskid_eviction_instance).to eq(@end_filter)
@@ -332,7 +333,10 @@ describe LogStash::Filters::Aggregate do
       it "should push previous map as new event" do
         push_filter = setup_filter({ "task_id" => "%{ppm_id}", "code" => "map['ppm_id'] = event.get('ppm_id')", "push_previous_map_as_event" => true, "timeout" => 5, "timeout_task_id_field" => "timeout_task_id_field" })
         push_filter.filter(event({"ppm_id" => "1"})) { |yield_event| fail "task 1 shouldn't have yield event" }
-        push_filter.filter(event({"ppm_id" => "2"})) { |yield_event| expect(yield_event.get("ppm_id")).to eq("1") ; expect(yield_event.get("timeout_task_id_field")).to eq("1") }
+        push_filter.filter(event({"ppm_id" => "2"})) do |yield_event| 
+          expect(yield_event.get("ppm_id")).to eq("1")
+          expect(yield_event.get("timeout_task_id_field")).to eq("1")
+        end
         expect(aggregate_maps["%{ppm_id}"].size).to eq(1)
       end
     end
@@ -367,5 +371,22 @@ describe LogStash::Filters::Aggregate do
     end
   end
 
+  context "timeout_timestamp_field option is defined, " do
+    describe "when 3 old events arrive, " do
+      it "should push a new aggregated event using timeout based on events timestamp" do
+        agg_filter = setup_filter({ "task_id" => "%{ppm_id}", "code" => "map['sql_duration'] ||= 0; map['sql_duration'] += event.get('duration')", "timeout_timestamp_field" => "@timestamp", "push_map_as_event_on_timeout" => true, "timeout" => 120 })
+        agg_filter.filter(event({"ppm_id" => "1", "duration" => 2, "@timestamp" => timestamp("2018-01-31T00:00:00Z")})) { |yield_event| fail "it shouldn't have yield event" }
+        agg_filter.filter(event({"ppm_id" => "1", "duration" => 3, "@timestamp" => timestamp("2018-01-31T00:00:01Z")})) { |yield_event| fail "it shouldn't have yield event" }
+        events_to_flush = agg_filter.flush()
+        expect(events_to_flush).to be_empty
+        agg_filter.filter(event({"ppm_id" => "1", "duration" => 4, "@timestamp" => timestamp("2018-01-31T00:05:00Z")})) do |yield_event| 
+          expect(yield_event).not_to be_nil
+          expect(yield_event.get("sql_duration")).to eq(5)
+        end
+        expect(aggregate_maps["%{ppm_id}"].size).to eq(1)
+        expect(aggregate_maps["%{ppm_id}"]["1"].map["sql_duration"]).to eq(4)
+      end
+    end
+  end
 
 end

data/spec/filters/aggregate_spec_helper.rb

@@ -5,6 +5,10 @@ def event(data = {})
 	LogStash::Event.new(data)
 end
 
+def timestamp(iso8601)
+  LogStash::Timestamp.new(iso8601)
+end
+
 def start_event(data = {})
 	data["logger"] = "TASK_START"
 	event(data)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-aggregate
 version: !ruby/object:Gem::Version
-  version: 2.7.2
+  version: 2.8.0
 platform: ruby
 authors:
 - Elastic
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-11-16 00:00:00.000000000 Z
+date: 2018-03-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement