logstash-filter-aggregate 2.9.0 → 2.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 468412047e7db18515ba09a4a112753019e7bcb1
-  data.tar.gz: db82a1a0b681c38c600d72a8bc0a9d8215013437
+  metadata.gz: bd090d41c0cd4b8fad41ca7ae9328d0ade9f5f45
+  data.tar.gz: 613247681d664249d2977d817d6c29479c6ba580
 SHA512:
-  metadata.gz: 4b5ac8ba6307a2e41d582655b34ecd4d2c2da808ac0f49dc332107676631eb32155e19dd3c870783e06babd39f55a7f2e5346d3c1d565d3a4de5342a0969452e
-  data.tar.gz: fd827e4944b5f30dd3b53561a3914668911861584f125302136179d4d27d4cfb66f957438994b6f42c188eb778264da5f5c46452ff216ddba6d66d9566536512
+  metadata.gz: c29a97c6dadfb87134ab46d027b93a7e16210d5eb39bf6262be8ab30fc3a3f9c3d4cea50c55bc778b9f431326e15b187c61bd8f214d6117b912d3515c5daadc0
+  data.tar.gz: 31cae051db32bd52a1bd712d12037184e910abec9f37a01e34e68f9a4a985364ea0b9b03f050e934c1f80e5e8a35e528263968d73153716687fa38620b928917

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 2.9.1
+  - bugfix: fix inactivity timeout feature when processing old logs (PR [#103](https://github.com/logstash-plugins/logstash-filter-aggregate/pull/103), thanks @jdratlif for his contribution!)
+  - docs: fix several typos in documentation
+  - docs: enhance example 4 documentation
+  - ci: enhance plugin continuous integration
+
 ## 2.9.0
   - new feature: add ability to dynamically define a custom `timeout` or `inactivity_timeout` in `code` block (fix issues [#91](https://github.com/logstash-plugins/logstash-filter-aggregate/issues/91) and [#92](https://github.com/logstash-plugins/logstash-filter-aggregate/issues/92))
   - new feature: add meta informations available in `code` block through `map_meta` variable

data/docs/index.asciidoc

@@ -228,7 +228,7 @@ In that case, you don't want to wait task timeout to flush aggregation map.
      aggregate {
        task_id => "%{country_name}"
        code => "
-         map['country_name'] = event.get('country_name')
+         map['country_name'] ||= event.get('country_name')
          map['towns'] ||= []
          map['towns'] << {'town_name' => event.get('town_name')}
          event.cancel()
@@ -240,8 +240,9 @@ In that case, you don't want to wait task timeout to flush aggregation map.
 ----------------------------------
 
 * The key point is that each time aggregate plugin detects a new `country_name`, it pushes previous aggregate map as a new Logstash event, and then creates a new empty map for the next country
-* When 5s timeout comes, the last aggregate map is pushed as a new event
-* Finally, initial events (which are not aggregated) are dropped because useless (thanks to `event.cancel()`)
+* When 3s timeout comes, the last aggregate map is pushed as a new event
+* Initial events (which are not aggregated) are dropped because useless (thanks to `event.cancel()`)
+* Last point: if a field is not fulfilled for every event (say "town_postcode" field), the `||=` operator will let you to push into aggregate map, the first "not null" value. Example: `map['town_postcode'] ||= event.get('town_postcode')`
 
 
 [id="plugins-{type}s-{plugin}-example5"]
@@ -249,7 +250,7 @@ In that case, you don't want to wait task timeout to flush aggregation map.
 
 Fifth use case: like example #3, there is no end event.
 
-Events keep comming for an indefinite time and you want to push the aggregation map as soon as possible after the last user interaction without waiting for the `timeout`.
+Events keep coming for an indefinite time and you want to push the aggregation map as soon as possible after the last user interaction without waiting for the `timeout`.
 
 This allows to have the aggregated events pushed closer to real time.
 
@@ -260,7 +261,7 @@ We can track a user by its ID through the events, however once the user stops in
 
 There is no specific event indicating the end of the user's interaction.
 
-The user ineraction will be considered as ended when no events for the specified user (task_id) arrive after the specified inactivity_timeout`.
+The user interaction will be considered as ended when no events for the specified user (task_id) arrive after the specified inactivity_timeout`.
 
 If the user continues interacting for longer than `timeout` seconds (since first event), the aggregation map will still be deleted and pushed as a new event when timeout occurs.
 
@@ -295,7 +296,7 @@ filter {
    code => "map['clicks'] ||= 0; map['clicks'] += 1;"
    push_map_as_event_on_timeout => true
    timeout_task_id_field => "user_id"
-   timeout => 3600 # 1 hour timeout, user activity will be considered finished one hour after the first event, even if events keep comming
+   timeout => 3600 # 1 hour timeout, user activity will be considered finished one hour after the first event, even if events keep coming
    inactivity_timeout => 300 # 5 minutes timeout, user activity will be considered finished if no new events arrive 5 minutes after the last event
    timeout_tags => ['_aggregatetimeout']
    timeout_code => "event.set('several_clicks', event.get('clicks') > 1)"
@@ -326,7 +327,7 @@ filter {
 * in the final event, you can execute a last code (for instance, add map data to final event)
 * after the final event, the map attached to task is deleted (thanks to `end_of_task => true`)
 * an aggregate map is tied to one task_id value which is tied to one task_id pattern. So if you have 2 filters with different task_id patterns, even if you have same task_id value, they won't share the same aggregate map.
-* in one filter configuration, it is recommanded to define a timeout option to protect the feature against unterminated tasks. It tells the filter to delete expired maps
+* in one filter configuration, it is recommended to define a timeout option to protect the feature against unterminated tasks. It tells the filter to delete expired maps
 * if no timeout is defined, by default, all maps older than 1800 seconds are automatically deleted
 * all timeout options have to be defined in only one aggregate filter per task_id pattern (per pipeline). Timeout options are : timeout, inactivity_timeout, timeout_code, push_map_as_event_on_timeout, push_previous_map_as_event, timeout_timestamp_field, timeout_task_id_field, timeout_tags
 * if `code` execution raises an exception, the error is logged and event is tagged '_aggregateexception'

data/lib/logstash/filters/aggregate.rb

@@ -214,6 +214,7 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
 
       # update last event timestamp
       aggregate_maps_element.lastevent_timestamp = reference_timestamp(event)
+      aggregate_maps_element.difference_from_lastevent_to_now = (Time.now - aggregate_maps_element.lastevent_timestamp).to_i
 
       # execute the code to read/update map and event
       map = aggregate_maps_element.map
@@ -395,7 +396,7 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
       @current_pipeline.aggregate_maps[@task_id].delete_if do |key, element|
         min_timestamp = element.timeout ? Time.now - element.timeout : default_min_timestamp
         min_inactivity_timestamp = element.inactivity_timeout ? Time.now - element.inactivity_timeout : default_min_inactivity_timestamp
-        if element.creation_timestamp + element.difference_from_creation_to_now < min_timestamp || element.lastevent_timestamp + element.difference_from_creation_to_now < min_inactivity_timestamp
+        if element.creation_timestamp + element.difference_from_creation_to_now < min_timestamp || element.lastevent_timestamp + element.difference_from_lastevent_to_now < min_inactivity_timestamp
           if @push_previous_map_as_event || @push_map_as_event_on_timeout
             events_to_flush << create_timeout_event(element.map, key)
           end
@@ -500,12 +501,13 @@ end # class LogStash::Filters::Aggregate
 # Element of "aggregate_maps"
 class LogStash::Filters::Aggregate::Element
 
-  attr_accessor :creation_timestamp, :lastevent_timestamp, :difference_from_creation_to_now, :timeout, :inactivity_timeout, :task_id, :map
+  attr_accessor :creation_timestamp, :lastevent_timestamp, :difference_from_creation_to_now, :difference_from_lastevent_to_now, :timeout, :inactivity_timeout, :task_id, :map
 
   def initialize(creation_timestamp, task_id)
     @creation_timestamp = creation_timestamp
-    @lastevent_timestamp = creation_timestamp
+    @lastevent_timestamp = creation_timestamp    
     @difference_from_creation_to_now = (Time.now - creation_timestamp).to_i
+    @difference_from_lastevent_to_now = @difference_from_creation_to_now
     @timeout = nil
     @inactivity_timeout = nil
     @task_id = task_id

data/logstash-filter-aggregate.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-filter-aggregate'
-  s.version         = '2.9.0'
+  s.version         = '2.9.1'
   s.licenses = ['Apache License (2.0)']
   s.summary = "Aggregates information from several events originating with a single task"
   s.description = 'This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-aggregate
 version: !ruby/object:Gem::Version
-  version: 2.9.0
+  version: 2.9.1
 platform: ruby
 authors:
 - Elastic
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-11-03 00:00:00.000000000 Z
+date: 2019-09-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement