logstash-filter-aggregate 2.3.0 → 2.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 89d2a20ef87157c9155e42001819efcd8b65c908
-  data.tar.gz: ad218ff47a177f75185534405db70ffa2c7fd7ca
+  metadata.gz: eaa83c85ffda851fdae41182c4ec2d010483dbbe
+  data.tar.gz: bfbb53a989e67654304b4e4c9cf89f2b91411c6a
 SHA512:
-  metadata.gz: cfd494edb94755712121a5a72325eeac1252d0b78e8ddcdffa69b861bc08b682b4cb08b6759e76be8b71066ddbe93d89f786b69891d7c5736ed4bc15ec0e7232
-  data.tar.gz: 1910c47decb6a4fc25cb03315c323a261071a69f7182d62f74f8b393ae02d58102fb9491802f34cb0a2a4dfee3c663b05eaa048e18e23b6346fa8e410a59a249
+  metadata.gz: 0adfe2d3916570f84230a5823ad6b038f661a5837bb78600e66c3a58d4b066630cf31cf974f939a8071f2ab2549698756a24cb768e6b15df983a2546f0dc6490
+  data.tar.gz: 7720f2d38e7757145294e05903376874dcd88a600bcd9174c5da1ac2ef0ca3ba4bd2545c01a107317cf037f99d8be8005cd3830aba62af32d85c6e8e5f57f1c7

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 2.3.1
+ - new feature: Add new option "timeout_tags" so that you can add tags to generated timeout events
+ 
 ## 2.3.0
  - new feature: Add new option "push_map_as_event_on_timeout" so that when a task timeout happens the aggregation map can be yielded as a new event
  - new feature: Add new option "timeout_code" which takes the timeout event populated with the aggregation map and executes code on it. This works for "push_map_as_event_on_timeout" as well as "push_previous_map_as_event"

data/README.md

@@ -132,7 +132,8 @@ We can also add 'timeout_task_id_field' so we can correlate the task_id, which i
             push_map_as_event_on_timeout => true
             timeout_task_id_field => "user_id"
             timeout => 600 # 10 minutes timeout
-            timeout_code => "event.tag('_aggregatetimeout')"
+            timeout_tags => ['_aggregatetimeout']
+            timeout_code => "event['several_clicks'] = (event['clicks'] > 1)"
         }
     }
 ```
@@ -141,9 +142,10 @@ We can also add 'timeout_task_id_field' so we can correlate the task_id, which i
 
 ``` json
     {
-        "user_id" : "12345",
-        "clicks" : 3,
-        "tags" : [
+        "user_id": "12345",
+        "clicks": 3,
+        "several_clicks": true,
+        "tags": [
             "_aggregatetimeout"
         ]
     }
@@ -174,7 +176,6 @@ In that case, you don't want to wait task timeout to flush aggregation map.
          aggregate {
              task_id => "%{country_name}"
              code => "
-                map['tags'] ||= ['aggregated']
                 map['town_name'] ||= []
                 event.to_hash.each do |key,value|
                     map[key] = value unless map.has_key?(key)
@@ -183,6 +184,7 @@ In that case, you don't want to wait task timeout to flush aggregation map.
              "
              push_previous_map_as_event => true
              timeout => 5
+             timeout_tags => ['aggregated']
          }
 
          if "aggregated" not in [tags] {
@@ -262,7 +264,7 @@ This enables to detect and process task timeouts in logstash, but also to manage
 The code to execute to complete timeout generated event, when 'push_map_as_event_on_timeout' or 'push_previous_map_as_event' is set to true.  
 The code block will have access to the newly generated timeout event that is pre-populated with the aggregation map.  
 If 'timeout_task_id_field' is set, the event is also populated with the task_id value  
-Example value: `"event.tag('_aggregatetimeout')"`
+Example value: `"event['state'] = 'timeout'"`
 
 - **timeout_task_id_field**  
 This option indicates the timeout generated event's field for the "task_id" value.  
@@ -272,6 +274,10 @@ Example:
 If the task_id is "12345" and this field is set to "my_id", the generated event will have:  
 `event[ "my_id" ] = "12345"`
 
+- **timeout_tags**  
+Defines tags to add when a timeout event is generated and yield.  
+Default value: `[]`  
+
 ## Changelog
 
 Read [CHANGELOG.md](CHANGELOG.md).

data/lib/logstash/filters/aggregate.rb

@@ -3,6 +3,7 @@
 require "logstash/filters/base"
 require "logstash/namespace"
 require "thread"
+require "logstash/util/decorators"
 
 # 
 # The aim of this filter is to aggregate information available among several events (typically log lines) belonging to a same task,
@@ -143,7 +144,8 @@ require "thread"
 #     push_map_as_event_on_timeout => true
 #     timeout_task_id_field => "user_id"
 #     timeout => 600 # 10 minutes timeout
-#     timeout_code => "event.tag('_aggregatetimeout')"
+#     timeout_tags => ['_aggregatetimeout']
+#     timeout_code => "event['several_clicks'] = (event['clicks'] > 1)"
 #   }
 # }
 # ----------------------------------
@@ -153,9 +155,10 @@ require "thread"
 # [source,json]
 # ----------------------------------
 # {
-#   "user_id" : "12345",
-#   "clicks" : 3,
-#     "tags" : [
+#   "user_id": "12345",
+#   "clicks": 3,
+#   "several_clicks": true,
+#     "tags": [
 #        "_aggregatetimeout"
 #     ]
 # }
@@ -188,7 +191,6 @@ require "thread"
 #      aggregate {
 #          task_id => "%{country_name}"
 #          code => "
-#           map['tags'] ||= ['aggregated']
 #           map['town_name'] ||= []
 #           event.to_hash.each do |key,value|
 #             map[key] = value unless map.has_key?(key)
@@ -197,6 +199,7 @@ require "thread"
 #          "
 #          push_previous_map_as_event => true
 #          timeout => 5
+#          timeout_tags => ['aggregated']
 #      }
 # 
 #      if "aggregated" not in [tags] {
@@ -256,7 +259,7 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
   #
   # If 'timeout_task_id_field' is set, the event is also populated with the task_id value 
   #
-  # Example value: `"event.tag('_aggregatetimeout')"`
+  # Example value: `"event['state'] = 'timeout'"`
   config :timeout_code, :validate => :string, :required => false
 
 
@@ -310,10 +313,15 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
   # When this option is enabled, each time a task timeout is detected, it pushes task aggregation map as a new logstash event.  
   # This enables to detect and process task timeouts in logstash, but also to manage tasks that have no explicit end event.
   config :push_map_as_event_on_timeout, :validate => :boolean, :required => false, :default => false
+
+  # Defines tags to add when a timeout event is generated and yield
+  config :timeout_tags, :validate => :array, :required => false, :default => []
   
+
   # Default timeout (in seconds) when not defined in plugin configuration
   DEFAULT_TIMEOUT = 1800
 
+  
   # This is the state of the filter.
   # For each entry, key is "task_id" and value is a map freely updatable by 'code' config
   @@aggregate_maps = {}
@@ -452,6 +460,8 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
     if @timeout_task_id_field
       event_to_yield[@timeout_task_id_field] = task_id
     end
+    
+    LogStash::Util::Decorators.add_tags(@timeout_tags,event_to_yield,"filters/#{self.class.name}")
 
     # Call code block if available
     if @timeout_code

data/logstash-filter-aggregate.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-filter-aggregate'
-  s.version = '2.3.0'
+  s.version = '2.3.1'
   s.licenses = ['Apache License (2.0)']
   s.summary = "The aim of this filter is to aggregate information available among several events (typically log lines) belonging to a same task, and finally push aggregated information into final task event."
   s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/aggregate_spec.rb

@@ -10,7 +10,7 @@ describe LogStash::Filters::Aggregate do
     aggregate_maps.clear()
     @start_filter = setup_filter({ "map_action" => "create", "code" => "map['sql_duration'] = 0" })
     @update_filter = setup_filter({ "map_action" => "update", "code" => "map['sql_duration'] += event['duration']" })
-    @end_filter = setup_filter({"timeout_task_id_field" => "my_id", "push_map_as_event_on_timeout" => true, "map_action" => "update", "code" => "event.to_hash.merge!(map)", "end_of_task" => true, "timeout" => 5, "timeout_code" => "event['test'] = 'testValue'" })
+    @end_filter = setup_filter({"timeout_task_id_field" => "my_id", "push_map_as_event_on_timeout" => true, "map_action" => "update", "code" => "event.to_hash.merge!(map)", "end_of_task" => true, "timeout" => 5, "timeout_code" => "event['test'] = 'testValue'", "timeout_tags" => ["tag1", "tag2"] })
   end
 
   context "Start event" do
@@ -182,6 +182,7 @@ describe LogStash::Filters::Aggregate do
         expect(entries[0]['my_id']).to eq("id_123") # task id 
         expect(entries[0]["sql_duration"]).to eq(0) # Aggregation map
         expect(entries[0]['test']).to eq("testValue") # Timeout code
+        expect(entries[0]['tags']).to eq(["tag1", "tag2"]) # Timeout tags
       end
     end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-aggregate
 version: !ruby/object:Gem::Version
-  version: 2.3.0
+  version: 2.3.1
 platform: ruby
 authors:
 - Elastic
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-08-03 00:00:00.000000000 Z
+date: 2016-10-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement