logstash-filter-aggregate 2.2.0 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e023e6c80ed96fa874477b00888a78bf45ee56a7
-  data.tar.gz: 1375bebbcda30c0052f8f2aa5188dda548d53d6c
+  metadata.gz: 89d2a20ef87157c9155e42001819efcd8b65c908
+  data.tar.gz: ad218ff47a177f75185534405db70ffa2c7fd7ca
 SHA512:
-  metadata.gz: f611bd231a13d8933662dbe7df7075a551e89772f0007d4d694636f307e576c828f091f37c7a3f917b717bec9dbb0acacff944cff14a4fc27bee92318d394e5a
-  data.tar.gz: bccd38bcb4f37a1f8f4a06ab01baa92883d6d4e3848ff380fa318583830a3600fe464ecd93f299993a42908ea70f0fd596300b6331d469eadfef6b47b165b4db
+  metadata.gz: cfd494edb94755712121a5a72325eeac1252d0b78e8ddcdffa69b861bc08b682b4cb08b6759e76be8b71066ddbe93d89f786b69891d7c5736ed4bc15ec0e7232
+  data.tar.gz: 1910c47decb6a4fc25cb03315c323a261071a69f7182d62f74f8b393ae02d58102fb9491802f34cb0a2a4dfee3c663b05eaa048e18e23b6346fa8e410a59a249

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 2.3.0
+ - new feature: Add new option "push_map_as_event_on_timeout" so that when a task timeout happens the aggregation map can be yielded as a new event
+ - new feature: Add new option "timeout_code" which takes the timeout event populated with the aggregation map and executes code on it. This works for "push_map_as_event_on_timeout" as well as "push_previous_map_as_event"
+ - new feature: Add new option "timeout_task_id_field" which is used to map the task_id on timeout events.
+
 ## 2.2.0
  - new feature: add new option "push_previous_map_as_event" so that each time aggregate plugin detects a new task id, it pushes previous aggregate map as a new logstash event
 

data/CONTRIBUTORS

@@ -6,6 +6,7 @@ Maintainers:
 
 Contributors:
 * Fabien Baligand (fbaligand)
+* Artur Kronenberg (pandaadb)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/README.md

@@ -62,7 +62,7 @@ otherwise events may be processed out of sequence and unexpected results will oc
 
 the field `sql_duration` is added and contains the sum of all sql queries durations.
 
-## Example #2
+## Example #2 : no start event
 
 * If you have the same logs than example #1, but without a start log : 
 ```
@@ -100,11 +100,63 @@ the field `sql_duration` is added and contains the sum of all sql queries durati
 * the key point is the "||=" ruby operator.  
 it allows to initialize 'sql_duration' map entry to 0 only if this map entry is not already initialized
 
-## Example #3
+## Example #3 : no end event
 
-Third use case : you have no specific start event and no specific end event.  
-A typical case is aggregating results from jdbc input plugin.  
-* Given that you have this SQL query : `SELECT country_name, town_name FROM town`  
+Third use case: You have no specific end event. 
+
+A typical case is aggregating or tracking user behaviour. We can track a user by its ID through the events, however once the user stops interacting, the events stop coming in. There is no specific event indicating the end of the user's interaction.
+
+In this case, we can enable the option 'push_map_as_event_on_timeout' to enable pushing the aggregation map as a new event when a timeout occurs.  
+In addition, we can enable 'timeout_code' to execute code on the populated timeout event.
+We can also add 'timeout_task_id_field' so we can correlate the task_id, which in this case would be the user's ID. 
+
+* Given these logs:
+
+```
+    INFO - 12345 - Clicked One
+    INFO - 12345 - Clicked Two
+    INFO - 12345 - Clicked Three
+```
+
+* You can aggregate the amount of clicks the user did like this:
+
+``` ruby
+    filter {
+        grok {
+                 match => [ "message", "%{LOGLEVEL:loglevel} - %{NOTSPACE:user_id} - %{GREEDYDATA:msg_text}" ]
+        }
+
+        aggregate {
+            task_id => "%{user_id}"
+            code => "map['clicks'] ||= 0; map['clicks'] += 1;"
+            push_map_as_event_on_timeout => true
+            timeout_task_id_field => "user_id"
+            timeout => 600 # 10 minutes timeout
+            timeout_code => "event.tag('_aggregatetimeout')"
+        }
+    }
+```
+
+* After ten minutes, this will yield an event like:
+
+``` json
+    {
+        "user_id" : "12345",
+        "clicks" : 3,
+        "tags" : [
+            "_aggregatetimeout"
+        ]
+    }
+```
+
+
+## Example #4 : no end event and tasks come one after the other
+
+Fourth use case : like example #3, you have no specific end event, but also, tasks come one after the other.  
+That is to say : tasks are not interlaced. All task1 events come, then all task2 events come, ...  
+In that case, you don't want to wait task timeout to flush aggregation map.  
+* A typical case is aggregating results from jdbc input plugin.
+* Given that you have this SQL query : `SELECT country_name, town_name FROM town ORDER BY country_name`  
 * Using jdbc input plugin, you get these 3 events from :
 ``` json
   { "country_name": "France", "town_name": "Paris" }
@@ -119,26 +171,26 @@ A typical case is aggregating results from jdbc input plugin.
 * You can do that using `push_previous_map_as_event` aggregate plugin option :
 ``` ruby
      filter {
-		 aggregate {
-		     task_id => "%{country_name}"
-		     code => "
-		     	map['tags'] ||= ['aggregated']
-		     	map['town_name'] ||= []
-		     	event.to_hash.each do |key,value|
-		     		map[key] = value unless map.has_key?(key)
-		     		map[key] << value if map[key].is_a?(Array)
-		     	end
-		     "
-		     push_previous_map_as_event => true
-		     timeout => 5
-		 }
-
-		 if "aggregated" not in [tags] {
-		 	drop {}
-		 }
-	 }
+         aggregate {
+             task_id => "%{country_name}"
+             code => "
+                map['tags'] ||= ['aggregated']
+                map['town_name'] ||= []
+                event.to_hash.each do |key,value|
+                    map[key] = value unless map.has_key?(key)
+                    map[key] << value if map[key].is_a?(Array)
+                end
+             "
+             push_previous_map_as_event => true
+             timeout => 5
+         }
+
+         if "aggregated" not in [tags] {
+            drop {}
+         }
+     }
 ```
-* The key point is that, each time aggregate plugin detects a new `country_name`, it pushes previous aggregate map as a new logstash event (with 'aggregated' tag), and then creates a new empty map for the next country
+* The key point is that each time aggregate plugin detects a new `country_name`, it pushes previous aggregate map as a new logstash event (with 'aggregated' tag), and then creates a new empty map for the next country
 * When 5s timeout comes, the last aggregate map is pushed as a new event
 * Finally, initial events (which are not aggregated) are dropped because useless
 
@@ -202,6 +254,23 @@ and then creates a new empty map for the next task.
 _WARNING:_ this option works fine only if tasks come one after the other. It means : all task1 events, then all task2 events, etc...  
 Default value: `false`  
 
+- **push_map_as_event_on_timeout**  
+When this option is enabled, each time a task timeout is detected, it pushes task aggregation map as a new logstash event.  
+This enables to detect and process task timeouts in logstash, but also to manage tasks that have no explicit end event.
+
+- **timeout_code**  
+The code to execute to complete timeout generated event, when 'push_map_as_event_on_timeout' or 'push_previous_map_as_event' is set to true.  
+The code block will have access to the newly generated timeout event that is pre-populated with the aggregation map.  
+If 'timeout_task_id_field' is set, the event is also populated with the task_id value  
+Example value: `"event.tag('_aggregatetimeout')"`
+
+- **timeout_task_id_field**  
+This option indicates the timeout generated event's field for the "task_id" value.  
+The task id will then be set into the timeout event. This can help correlate which tasks have been timed out.  
+This field has no default value and will not be set on the event if not configured.  
+Example:  
+If the task_id is "12345" and this field is set to "my_id", the generated event will have:  
+`event[ "my_id" ] = "12345"`
 
 ## Changelog
 

data/lib/logstash/filters/aggregate.rb

@@ -69,7 +69,7 @@ require "thread"
 # 
 # the field `sql_duration` is added and contains the sum of all sql queries durations.
 # 
-# ==== Example #2
+# ==== Example #2 : no start event
 # 
 # * If you have the same logs than example #1, but without a start log :
 # [source,ruby]
@@ -109,9 +109,63 @@ require "thread"
 # * the key point is the "||=" ruby operator. It allows to initialize 'sql_duration' map entry to 0 only if this map entry is not already initialized
 #
 #
-# ==== Example #3
+# ==== Example #3 : no end event
+#
+# Third use case: You have no specific end event. 
+#
+# A typical case is aggregating or tracking user behaviour. We can track a user by its ID through the events, however once the user stops interacting, the events stop coming in. There is no specific event indicating the end of the user's interaction.
+#
+# In this case, we can enable the option 'push_map_as_event_on_timeout' to enable pushing the aggregation map as a new event when a timeout occurs.  
+# In addition, we can enable 'timeout_code' to execute code on the populated timeout event.
+# We can also add 'timeout_task_id_field' so we can correlate the task_id, which in this case would be the user's ID. 
+#
+# * Given these logs: 
+#
+# [source,ruby]
+# ----------------------------------
+# INFO - 12345 - Clicked One
+# INFO - 12345 - Clicked Two
+# INFO - 12345 - Clicked Three
+# ----------------------------------
+#
+# * You can aggregate the amount of clicks the user did like this:
+# 
+# [source,ruby]
+# ----------------------------------
+# filter {
+#   grok {
+#     match => [ "message", "%{LOGLEVEL:loglevel} - %{NOTSPACE:user_id} - %{GREEDYDATA:msg_text}" ]
+#   }
+#
+#   aggregate {
+#     task_id => "%{user_id}"
+#     code => "map['clicks'] ||= 0; map['clicks'] += 1;"
+#     push_map_as_event_on_timeout => true
+#     timeout_task_id_field => "user_id"
+#     timeout => 600 # 10 minutes timeout
+#     timeout_code => "event.tag('_aggregatetimeout')"
+#   }
+# }
+# ----------------------------------
+#
+# * After ten minutes, this will yield an event like:
+#
+# [source,json]
+# ----------------------------------
+# {
+#   "user_id" : "12345",
+#   "clicks" : 3,
+#     "tags" : [
+#        "_aggregatetimeout"
+#     ]
+# }
+# ----------------------------------
+#
+# ==== Example #4 : no end event and tasks come one after the other
 # 
-# Third use case : you have no specific start event and no specific end event.  
+# Fourth use case : like example #3, you have no specific end event, but also, tasks come one after the other.  
+# That is to say : tasks are not interlaced. All task1 events come, then all task2 events come, ...  
+# In that case, you don't want to wait task timeout to flush aggregation map.  
 # * A typical case is aggregating results from jdbc input plugin.  
 # * Given that you have this SQL query : `SELECT country_name, town_name FROM town`  
 # * Using jdbc input plugin, you get these 3 events from :
@@ -150,7 +204,7 @@ require "thread"
 #      }
 #    }
 # ----------------------------------
-# * The key point is that, each time aggregate plugin detects a new `country_name`, it pushes previous aggregate map as a new logstash event (with 'aggregated' tag), and then creates a new empty map for the next country
+# * The key point is that each time aggregate plugin detects a new `country_name`, it pushes previous aggregate map as a new logstash event (with 'aggregated' tag), and then creates a new empty map for the next country
 # * When 5s timeout comes, the last aggregate map is pushed as a new event
 # * Finally, initial events (which are not aggregated) are dropped because useless
 # 
@@ -195,6 +249,30 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
   # Example value : `"map['sql_duration'] += event['duration']"`
   config :code, :validate => :string, :required => true
 
+
+
+  # The code to execute to complete timeout generated event, when 'push_map_as_event_on_timeout' or 'push_previous_map_as_event' is set to true. 
+  # The code block will have access to the newly generated timeout event that is pre-populated with the aggregation map. 
+  #
+  # If 'timeout_task_id_field' is set, the event is also populated with the task_id value 
+  #
+  # Example value: `"event.tag('_aggregatetimeout')"`
+  config :timeout_code, :validate => :string, :required => false
+
+
+  # This option indicates the timeout generated event's field for the "task_id" value. 
+  # The task id will then be set into the timeout event. This can help correlate which tasks have been timed out.  
+  #
+  # This field has no default value and will not be set on the event if not configured.
+  #
+  # Example:
+  #
+  # If the task_id is "12345" and this field is set to "my_id", the generated event will have:
+  # event[ "my_id" ] = "12345"
+  #
+  config :timeout_task_id_field, :validate => :string, :required => false
+
+
   # Tell the filter what to do with aggregate map.
   #
   # `create`: create the map, and execute the code only if map wasn't created before
@@ -225,10 +303,13 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
   
   # When this option is enabled, each time aggregate plugin detects a new task id, it pushes previous aggregate map as a new logstash event, 
   # and then creates a new empty map for the next task.
-  # 
+  #
   # WARNING: this option works fine only if tasks come one after the other. It means : all task1 events, then all task2 events, etc...
   config :push_previous_map_as_event, :validate => :boolean, :required => false, :default => false
   
+  # When this option is enabled, each time a task timeout is detected, it pushes task aggregation map as a new logstash event.  
+  # This enables to detect and process task timeouts in logstash, but also to manage tasks that have no explicit end event.
+  config :push_map_as_event_on_timeout, :validate => :boolean, :required => false, :default => false
   
   # Default timeout (in seconds) when not defined in plugin configuration
   DEFAULT_TIMEOUT = 1800
@@ -256,6 +337,11 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
     # process lambda expression to call in each filter call
     eval("@codeblock = lambda { |event, map| #{@code} }", binding, "(aggregate filter code)")
 
+    # process lambda expression to call in the timeout case or previous event case
+    if @timeout_code
+      eval("@timeout_codeblock = lambda { |event| #{@timeout_code} }", binding, "(aggregate filter timeout code)")
+    end
+
     @@mutex.synchronize do
       # define eviction_instance
       if (!@timeout.nil? && (@@eviction_instance.nil? || @timeout < @@eviction_instance.timeout))
@@ -317,13 +403,14 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
       # retrieve the current aggregate map
       aggregate_maps_element = @@aggregate_maps[task_id]
       
+
       # create aggregate map, if it doesn't exist
       if (aggregate_maps_element.nil?)
         return if @map_action == "update"
         # create new event from previous map, if @push_previous_map_as_event is enabled
         if (@push_previous_map_as_event and !@@aggregate_maps.empty?)
           previous_map = @@aggregate_maps.shift[1].map
-          event_to_yield = LogStash::Event.new(previous_map)
+          event_to_yield = create_timeout_event(previous_map, task_id)
         end
         aggregate_maps_element = LogStash::Filters::Aggregate::Element.new(Time.now);
         @@aggregate_maps[task_id] = aggregate_maps_element
@@ -354,6 +441,31 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
 
   end
 
+  # Create a new event from the aggregation_map and the corresponding task_id
+  # This will create the event and
+  #  if @timeout_task_id_field is set, it will set the task_id on the timeout event
+  #  if @timeout_code is set, it will execute the timeout code on the created timeout event
+  # returns the newly created event
+  def create_timeout_event(aggregation_map, task_id)
+    event_to_yield = LogStash::Event.new(aggregation_map)        
+
+    if @timeout_task_id_field
+      event_to_yield[@timeout_task_id_field] = task_id
+    end
+
+    # Call code block if available
+    if @timeout_code
+      begin
+        @timeout_codeblock.call(event_to_yield)
+      rescue => exception
+        @logger.error("Aggregate exception occurred. Error: #{exception} ; TimeoutCode: #{@timeout_code} ; TimeoutEventData: #{event_to_yield.instance_variable_get('@data')}")
+        event_to_yield.tag("_aggregateexception")
+      end
+    end
+            
+    return event_to_yield
+  end 
+
   # Necessary to indicate logstash to periodically call 'flush' method
   def periodic_flush
     true
@@ -371,23 +483,24 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
     if (@@eviction_instance == self && (@@last_eviction_timestamp.nil? || Time.now > @@last_eviction_timestamp + @timeout / 2))
       events_to_flush = remove_expired_maps()
       @@last_eviction_timestamp = Time.now
+      return events_to_flush
     end
-    
-    return events_to_flush
+
   end
 
   
   # Remove the expired Aggregate maps from @@aggregate_maps if they are older than timeout.
-  # If @push_previous_map_as_event option is set, expired maps are returned as new events to be flushed to Logstash pipeline.
+  # If @push_previous_map_as_event option is set, or @push_map_as_event_on_timeout is set, expired maps are returned as new events to be flushed to Logstash pipeline.
   def remove_expired_maps()
     events_to_flush = []
     min_timestamp = Time.now - @timeout
     
     @@mutex.synchronize do
+
       @@aggregate_maps.delete_if do |key, element| 
         if (element.creation_timestamp < min_timestamp)
-          if (@push_previous_map_as_event)
-            events_to_flush << LogStash::Event.new(element.map)
+          if (@push_previous_map_as_event) || (@push_map_as_event_on_timeout)
+            events_to_flush << create_timeout_event(element.map, key)
           end
           next true
         end
@@ -409,4 +522,4 @@ class LogStash::Filters::Aggregate::Element
     @creation_timestamp = creation_timestamp
     @map = {}
   end
-end
+end

data/logstash-filter-aggregate.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-filter-aggregate'
-  s.version = '2.2.0'
+  s.version = '2.3.0'
   s.licenses = ['Apache License (2.0)']
   s.summary = "The aim of this filter is to aggregate information available among several events (typically log lines) belonging to a same task, and finally push aggregated information into final task event."
   s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/aggregate_spec.rb

@@ -10,7 +10,7 @@ describe LogStash::Filters::Aggregate do
     aggregate_maps.clear()
     @start_filter = setup_filter({ "map_action" => "create", "code" => "map['sql_duration'] = 0" })
     @update_filter = setup_filter({ "map_action" => "update", "code" => "map['sql_duration'] += event['duration']" })
-    @end_filter = setup_filter({ "map_action" => "update", "code" => "event.to_hash.merge!(map)", "end_of_task" => true, "timeout" => 5 })
+    @end_filter = setup_filter({"timeout_task_id_field" => "my_id", "push_map_as_event_on_timeout" => true, "map_action" => "update", "code" => "event.to_hash.merge!(map)", "end_of_task" => true, "timeout" => 5, "timeout_code" => "event['test'] = 'testValue'" })
   end
 
   context "Start event" do
@@ -170,13 +170,18 @@ describe LogStash::Filters::Aggregate do
 
     describe "timeout defined on the filter" do
       it "event is not removed if not expired" do
-        @end_filter.flush()
+        entries = @end_filter.flush()
         expect(aggregate_maps.size).to eq(1)
+        expect(entries).to be_empty
       end
-      it "event is removed if expired" do
+      it "removes event if expired and creates a new timeout event" do
         sleep(2)
-        @end_filter.flush()
+        entries = @end_filter.flush()
         expect(aggregate_maps).to be_empty
+        expect(entries.size).to eq(1)
+        expect(entries[0]['my_id']).to eq("id_123") # task id 
+        expect(entries[0]["sql_duration"]).to eq(0) # Aggregation map
+        expect(entries[0]['test']).to eq("testValue") # Timeout code
       end
     end
 
@@ -231,13 +236,14 @@ describe LogStash::Filters::Aggregate do
 
     describe "when timeout happens, " do
       it "flush method should return last map as new event" do
-        push_filter = setup_filter({ "code" => "map['taskid'] = event['taskid']", "push_previous_map_as_event" => true, "timeout" => 1 })
+        push_filter = setup_filter({ "code" => "map['taskid'] = event['taskid']", "push_previous_map_as_event" => true, "timeout" => 1, "timeout_code" => "event['test'] = 'testValue'" })
         push_filter.filter(event({"taskid" => "1"}))
         sleep(2)
         events_to_flush = push_filter.flush()
         expect(events_to_flush).not_to be_nil
         expect(events_to_flush.size).to eq(1)
         expect(events_to_flush[0]["taskid"]).to eq("1")
+        expect(events_to_flush[0]['test']).to eq("testValue")
         expect(aggregate_maps.size).to eq(0)
       end
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-aggregate
 version: !ruby/object:Gem::Version
-  version: 2.2.0
+  version: 2.3.0
 platform: ruby
 authors:
 - Elastic
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-07-09 00:00:00.000000000 Z
+date: 2016-08-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement