logstash-filter-aggregate 2.5.0 → 2.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b8769a07ab85fdc52359b1e08838f8f308f96e61
-  data.tar.gz: 0e2c0b0c078b789c87c57678683ac2ce72919fa5
+  metadata.gz: a9d474bc096fd4b164adb655607abdc0d52b7f15
+  data.tar.gz: 9e7b57528e3201bea76ed198e080be776b0ed3af
 SHA512:
-  metadata.gz: 7baa1843fac215316d44e8d029aa840b9fe295f7bb953e2bb51873665da473d70f54551b23960491256a542687732d20e482a3efd98fbd8124012c4cea5c3e84
-  data.tar.gz: b073f65ebad9629bdda9893f7e08fe630c8f48643bb4575001ed2430e46fbf6f13a218b09e2df9585788d519a5acf2fcb5b36c0e7e67ed8c45245c61dbdab523
+  metadata.gz: 6d08a02cdf7a32904e74f235f2f3888170cd535dce21aeb5767c6ddfa302920beb2a6c4216aac1b485ccb502a0f0180d687b5276e7e18ec691e4c8ae2c5895e2
+  data.tar.gz: 5d509ab8cf7d26fbce5cf4f53d8255ff7c203492c9fe96aab363afa73dedcda278e2497a7be91e7f8f7ce58f08bb42cfd93f08afd94628c4d54b6cd671dfeee9

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 2.5.1
+- enhancement: when final flush occurs (just before Logstash shutdown), add `_aggregatefinalflush` tag on generated timeout events 
+- bugfix: when final flush occurs (just before Logstash shutdown), push last aggregate map as event (if push_previous_map_as_event=true)
+- bugfix: fix 'timeout_task_id_field' feature when push_previous_map_as_event=true
+- bugfix: fix aggregate_maps_path feature (bug since v2.4.0)
+- internal: add debug logging
+- internal: refactor flush management static variables
+
 ## 2.5.0
  - new feature: add compatibility with Logstash 5
  - breaking: need Logstash 2.4 or later  

data/README.md

@@ -179,7 +179,7 @@ In that case, you don't want to wait task timeout to flush aggregation map.
                 map['town_name'] ||= []
                 event.to_hash.each do |key,value|
                     map[key] = value unless map.has_key?(key)
-                    map[key] << value if map[key].is_a?(Array)
+                    map[key] << value if map[key].is_a?(Array) and !value.is_a?(Array)
                 end
              "
              push_previous_map_as_event => true

data/lib/logstash/filters/aggregate.rb

@@ -194,7 +194,7 @@ require "logstash/util/decorators"
 #           map['town_name'] ||= []
 #           event.to_hash.each do |key,value|
 #             map[key] = value unless map.has_key?(key)
-#             map[key] << value if map[key].is_a?(Array)
+#             map[key] << value if map[key].is_a?(Array) and !value.is_a?(Array)
 #           end
 #          "
 #          push_previous_map_as_event => true
@@ -313,12 +313,14 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
 
   # Defines tags to add when a timeout event is generated and yield
   config :timeout_tags, :validate => :array, :required => false, :default => []
-  
 
+      
+  # STATIC VARIABLES
+
+        
   # Default timeout (in seconds) when not defined in plugin configuration
   DEFAULT_TIMEOUT = 1800
 
-  
   # This is the state of the filter.
   # For each entry, key is "task_id" and value is a map freely updatable by 'code' config
   @@aggregate_maps = {}
@@ -329,20 +331,26 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
   # Default timeout for task_id patterns where timeout is not defined in logstash filter configuration
   @@default_timeout = nil
 
-  # For each "task_id" pattern, defines which Aggregate instance will evict all expired Aggregate elements (older than timeout)
+  # For each "task_id" pattern, defines which Aggregate instance will process flush() call, processing expired Aggregate elements (older than timeout)
   # For each entry, key is "task_id pattern" and value is "aggregate instance"
-  @@eviction_instance_map = {}
+  @@flush_instance_map = {}
 
-  # last time where eviction was launched, per "task_id" pattern
-  @@last_eviction_timestamp_map = {}
+  # last time where timeout management in flush() method was launched, per "task_id" pattern
+  @@last_flush_timestamp_map = {}
 
   # flag indicating if aggregate_maps_path option has been already set on one aggregate instance
   @@aggregate_maps_path_set = false
 
+  # defines which Aggregate instance will close Aggregate static variables
+  @@static_close_instance = nil
+
   
   # Initialize plugin
   public
   def register
+    
+    @logger.debug("Aggregate register call", :code => @code)
+
     # process lambda expression to call in each filter call
     eval("@codeblock = lambda { |event, map| #{@code} }", binding, "(aggregate filter code)")
 
@@ -355,11 +363,11 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
       
       # timeout management : define eviction_instance for current task_id pattern
       if has_timeout_options?
-        if @@eviction_instance_map.has_key?(@task_id)
+        if @@flush_instance_map.has_key?(@task_id)
           # all timeout options have to be defined in only one aggregate filter per task_id pattern
           raise LogStash::ConfigurationError, "Aggregate plugin: For task_id pattern #{@task_id}, there are more than one filter which defines timeout options. All timeout options have to be defined in only one aggregate filter per task_id pattern. Timeout options are : #{display_timeout_options}"
         end
-        @@eviction_instance_map[@task_id] = self
+        @@flush_instance_map[@task_id] = self
         @logger.debug("Aggregate timeout for '#{@task_id}' pattern: #{@timeout} seconds")
       end
 
@@ -369,13 +377,19 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
         @logger.debug("Aggregate default timeout: #{@timeout} seconds")
       end
 
-      # check if aggregate_maps_path option has already been set on another instance
+      # reinit static_close_instance (if necessary)
+      if !@@aggregate_maps_path_set && !@@static_close_instance.nil?
+        @@static_close_instance = nil
+      end
+
+      # check if aggregate_maps_path option has already been set on another instance else set @@aggregate_maps_path_set
       if !@aggregate_maps_path.nil?
         if @@aggregate_maps_path_set
           @@aggregate_maps_path_set = false
           raise LogStash::ConfigurationError, "Aggregate plugin: Option 'aggregate_maps_path' must be set on only one aggregate filter"
         else
           @@aggregate_maps_path_set = true
+          @@static_close_instance = self
         end
       end
       
@@ -394,21 +408,29 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
   # Called when logstash stops
   public
   def close
-
-    # store aggregate maps to file (if option defined)
-    @@mutex.synchronize do
-      @@aggregate_maps.delete_if { |key, value| value.empty? }
-      if !@aggregate_maps_path.nil? && !@@aggregate_maps.empty?
-        File.open(@aggregate_maps_path, "w"){ |to_file| Marshal.dump(@@aggregate_maps, to_file) }
-        @logger.info("Aggregate maps stored to : #{@aggregate_maps_path}")
+    
+    @logger.debug("Aggregate close call", :code => @code)
+
+    # define static close instance if none is already defined    
+    @@static_close_instance = self if @@static_close_instance.nil?
+
+    if @@static_close_instance == self
+      # store aggregate maps to file (if option defined)
+      @@mutex.synchronize do
+        @@aggregate_maps.delete_if { |key, value| value.empty? }
+        if !@aggregate_maps_path.nil? && !@@aggregate_maps.empty?
+          File.open(@aggregate_maps_path, "w"){ |to_file| Marshal.dump(@@aggregate_maps, to_file) }
+          @logger.info("Aggregate maps stored to : #{@aggregate_maps_path}")
+        end
+        @@aggregate_maps.clear()
       end
-      @@aggregate_maps.clear()
+  
+      # reinit static variables for logstash reload
+      @@default_timeout = nil
+      @@flush_instance_map = {}
+      @@last_flush_timestamp_map = {}
+      @@aggregate_maps_path_set = false
     end
-
-    # Protection against logstash reload
-    @@aggregate_maps_path_set = false if @@aggregate_maps_path_set
-    @@default_timeout = nil unless @@default_timeout.nil?
-    @@eviction_instance_map = {} unless @@eviction_instance_map.empty?
     
   end
   
@@ -435,8 +457,7 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
         return if @map_action == "update"
         # create new event from previous map, if @push_previous_map_as_event is enabled
         if @push_previous_map_as_event && !@@aggregate_maps[@task_id].empty?
-          previous_map = @@aggregate_maps[@task_id].shift[1].map
-          event_to_yield = create_timeout_event(previous_map, task_id)
+          event_to_yield = extract_previous_map_as_event()
         end
         aggregate_maps_element = LogStash::Filters::Aggregate::Element.new(Time.now);
         @@aggregate_maps[@task_id][task_id] = aggregate_maps_element
@@ -448,6 +469,7 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
       # execute the code to read/update map and event
       begin
         @codeblock.call(event, map)
+        @logger.debug("Aggregate successful filter code execution", :code => @code)
         noError = true
       rescue => exception
         @logger.error("Aggregate exception occurred", 
@@ -477,6 +499,9 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
   #  if @timeout_code is set, it will execute the timeout code on the created timeout event
   # returns the newly created event
   def create_timeout_event(aggregation_map, task_id)
+
+    @logger.debug("Aggregate create_timeout_event call with task_id '#{task_id}'")
+    
     event_to_yield = LogStash::Event.new(aggregation_map)        
 
     if @timeout_task_id_field
@@ -501,6 +526,14 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
     return event_to_yield
   end 
 
+  # Extract the previous map in aggregate maps, and return it as a new Logstash event
+  def extract_previous_map_as_event
+    previous_entry = @@aggregate_maps[@task_id].shift()
+    previous_task_id = previous_entry[0]
+    previous_map = previous_entry[1].map
+    return create_timeout_event(previous_map, previous_task_id)
+  end
+
   # Necessary to indicate logstash to periodically call 'flush' method
   def periodic_flush
     true
@@ -508,22 +541,41 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
   
   # This method is invoked by LogStash every 5 seconds.
   def flush(options = {})
+    
+    @logger.debug("Aggregate flush call with #{options}")
+      
     # Protection against no timeout defined by logstash conf : define a default eviction instance with timeout = DEFAULT_TIMEOUT seconds
     if @@default_timeout.nil?
       @@default_timeout = DEFAULT_TIMEOUT
     end
-    if !@@eviction_instance_map.has_key?(@task_id)
-      @@eviction_instance_map[@task_id] = self
+    if !@@flush_instance_map.has_key?(@task_id)
+      @@flush_instance_map[@task_id] = self
       @timeout = @@default_timeout
-    elsif @@eviction_instance_map[@task_id].timeout.nil?
-      @@eviction_instance_map[@task_id].timeout = @@default_timeout
+    elsif @@flush_instance_map[@task_id].timeout.nil?
+      @@flush_instance_map[@task_id].timeout = @@default_timeout
     end
     
-    # Launch eviction only every interval of (@timeout / 2) seconds
-    if @@eviction_instance_map[@task_id] == self && (!@@last_eviction_timestamp_map.has_key?(@task_id) || Time.now > @@last_eviction_timestamp_map[@task_id] + @timeout / 2)
+    # Launch timeout management only every interval of (@timeout / 2) seconds or at Logstash shutdown
+    if @@flush_instance_map[@task_id] == self && (!@@last_flush_timestamp_map.has_key?(@task_id) || Time.now > @@last_flush_timestamp_map[@task_id] + @timeout / 2 || options[:final])
       events_to_flush = remove_expired_maps()
-      @@last_eviction_timestamp_map[@task_id] = Time.now
+
+      # at Logstash shutdown, if push_previous_map_as_event is enabled, it's important to force flush (particularly for jdbc input plugin)
+      if options[:final] && @push_previous_map_as_event && !@@aggregate_maps[@task_id].empty?
+        events_to_flush << extract_previous_map_as_event()
+      end
+
+      # tag flushed events, indicating "final flush" special event
+      if options[:final]
+        events_to_flush.each { |event_to_flush| event_to_flush.tag("_aggregatefinalflush") }
+      end
+      
+      # update last flush timestamp
+      @@last_flush_timestamp_map[@task_id] = Time.now
+        
+      # return events to flush into Logstash pipeline
       return events_to_flush
+    else
+      return []
     end
 
   end
@@ -537,6 +589,8 @@ class LogStash::Filters::Aggregate < LogStash::Filters::Base
     
     @@mutex.synchronize do
 
+      @logger.debug("Aggregate remove_expired_maps call with '#{@task_id}' pattern and #{@@aggregate_maps[@task_id].length} maps")
+
       @@aggregate_maps[@task_id].delete_if do |key, element| 
         if element.creation_timestamp < min_timestamp
           if @push_previous_map_as_event || @push_map_as_event_on_timeout

data/logstash-filter-aggregate.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-filter-aggregate'
-  s.version = '2.5.0'
+  s.version = '2.5.1'
   s.licenses = ['Apache License (2.0)']
   s.summary = 'The aim of this filter is to aggregate information available among several events (typically log lines) belonging to a same task, and finally push aggregated information into final task event.'
   s.description = 'This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program'

data/spec/filters/aggregate_spec.rb

@@ -6,8 +6,7 @@ require_relative "aggregate_spec_helper"
 describe LogStash::Filters::Aggregate do
 
   before(:each) do
-    reset_timeout_management()
-    aggregate_maps.clear()
+    reset_static_variables()
     @start_filter = setup_filter({ "map_action" => "create", "code" => "map['sql_duration'] = 0" })
     @update_filter = setup_filter({ "map_action" => "update", "code" => "map['sql_duration'] += event.get('duration')" })
     @end_filter = setup_filter({"timeout_task_id_field" => "my_id", "push_map_as_event_on_timeout" => true, "map_action" => "update", "code" => "event.set('sql_duration', map['sql_duration'])", "end_of_task" => true, "timeout" => 5, "timeout_code" => "event.set('test', 'testValue')", "timeout_tags" => ["tag1", "tag2"] })
@@ -212,6 +211,9 @@ describe LogStash::Filters::Aggregate do
         filter = store_filter.filter(start_event)
         expect(aggregate_maps["%{taskid}"].size).to eq(1)
         
+        @end_filter.close()
+        expect(aggregate_maps).not_to be_empty
+        
         store_filter.close()
         expect(File.exist?(store_file)).to be true
         expect(aggregate_maps).to be_empty
@@ -232,6 +234,21 @@ describe LogStash::Filters::Aggregate do
     end
   end
   
+  context "Logstash reload occurs, " do
+    describe "close method is called, " do
+      it "reinitializes static variables" do
+        @end_filter.close()
+        expect(aggregate_maps).to be_empty
+        expect(taskid_eviction_instance).to be_nil
+        expect(static_close_instance).not_to be_nil
+        expect(aggregate_maps_path_set).to be false
+        
+        @end_filter.register()
+        expect(static_close_instance).to be_nil
+      end
+    end
+  end
+
   context "push_previous_map_as_event option is defined, " do 
     describe "when push_previous_map_as_event option is activated on another filter with same task_id pattern" do
       it "should throw a LogStash::ConfigurationError" do
@@ -243,9 +260,9 @@ describe LogStash::Filters::Aggregate do
     
     describe "when a new task id is detected, " do
       it "should push previous map as new event" do
-        push_filter = setup_filter({ "task_id" => "%{ppm_id}", "code" => "map['ppm_id'] = event.get('ppm_id')", "push_previous_map_as_event" => true, "timeout" => 5 })
+        push_filter = setup_filter({ "task_id" => "%{ppm_id}", "code" => "map['ppm_id'] = event.get('ppm_id')", "push_previous_map_as_event" => true, "timeout" => 5, "timeout_task_id_field" => "timeout_task_id_field" })
         push_filter.filter(event({"ppm_id" => "1"})) { |yield_event| fail "task 1 shouldn't have yield event" }
-        push_filter.filter(event({"ppm_id" => "2"})) { |yield_event| expect(yield_event.get("ppm_id")).to eq("1") }
+        push_filter.filter(event({"ppm_id" => "2"})) { |yield_event| expect(yield_event.get("ppm_id")).to eq("1") ; expect(yield_event.get("timeout_task_id_field")).to eq("1") }
         expect(aggregate_maps["%{ppm_id}"].size).to eq(1)
       end
     end
@@ -263,6 +280,21 @@ describe LogStash::Filters::Aggregate do
         expect(aggregate_maps["%{ppm_id}"].size).to eq(0)
       end
     end
+
+    describe "when Logstash shutdown happens, " do
+      it "flush method should return last map as new event even if timeout has not occured" do
+        push_filter = setup_filter({ "task_id" => "%{ppm_id}", "code" => "", "push_previous_map_as_event" => true, "timeout" => 4 })
+        push_filter.filter(event({"ppm_id" => "1"}))
+        events_to_flush = push_filter.flush({:final=>false})
+        expect(events_to_flush).to be_empty
+        expect(aggregate_maps["%{ppm_id}"].size).to eq(1)
+        events_to_flush = push_filter.flush({:final=>true})
+        expect(events_to_flush).not_to be_nil
+        expect(events_to_flush.size).to eq(1)
+        expect(events_to_flush[0].get("tags")).to eq(["_aggregatefinalflush"])
+        expect(aggregate_maps["%{ppm_id}"].size).to eq(0)
+      end
+    end
   end
   
 

data/spec/filters/aggregate_spec_helper.rb

@@ -38,10 +38,26 @@ def aggregate_maps()
 end
 
 def taskid_eviction_instance()
-	LogStash::Filters::Aggregate.class_variable_get(:@@eviction_instance_map)["%{taskid}"]
+	LogStash::Filters::Aggregate.class_variable_get(:@@flush_instance_map)["%{taskid}"]
+end
+
+def static_close_instance()
+  LogStash::Filters::Aggregate.class_variable_get(:@@static_close_instance)
+end
+
+def aggregate_maps_path_set()
+  LogStash::Filters::Aggregate.class_variable_get(:@@aggregate_maps_path_set)
 end
 
 def reset_timeout_management()
 	LogStash::Filters::Aggregate.class_variable_set(:@@default_timeout, nil)
-  LogStash::Filters::Aggregate.class_variable_get(:@@eviction_instance_map).clear()
+  LogStash::Filters::Aggregate.class_variable_get(:@@flush_instance_map).clear()
+  LogStash::Filters::Aggregate.class_variable_get(:@@last_flush_timestamp_map).clear()
+end
+
+def reset_static_variables()
+  reset_timeout_management()
+  aggregate_maps().clear()
+  LogStash::Filters::Aggregate.class_variable_set(:@@static_close_instance, nil)
+  LogStash::Filters::Aggregate.class_variable_set(:@@aggregate_maps_path_set, false)
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-aggregate
 version: !ruby/object:Gem::Version
-  version: 2.5.0
+  version: 2.5.1
 platform: ruby
 authors:
 - Elastic
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-10-30 00:00:00.000000000 Z
+date: 2017-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement