logstash-filter-aggregate 2.0.5 → 2.1.0

data/logstash-filter-aggregate.gemspec

@@ -1,24 +1,24 @@
-Gem::Specification.new do |s|
-  s.name = 'logstash-filter-aggregate'
-  s.version         = '2.0.5'
-  s.licenses = ['Apache License (2.0)']
-  s.summary = "The aim of this filter is to aggregate information available among several events (typically log lines) belonging to a same task, and finally push aggregated information into final task event."
-  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
-  s.authors = ["Elastic", "Fabien Baligand"]
-  s.email = 'info@elastic.co'
-  s.homepage = "https://github.com/logstash-plugins/logstash-filter-aggregate"
-  s.require_paths = ["lib"]
-
-  # Files
-  s.files = Dir['lib/**/*','spec/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE']
-
-  # Tests
-  s.test_files = s.files.grep(%r{^(test|spec|features)/})
-
-  # Special flag to let us know this is actually a logstash plugin
-  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
-
-  # Gem dependencies
-  s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
-  s.add_development_dependency 'logstash-devutils', '~> 0'
-end
+Gem::Specification.new do |s|
+  s.name = 'logstash-filter-aggregate'
+  s.version         = '2.1.0'
+  s.licenses = ['Apache License (2.0)']
+  s.summary = "The aim of this filter is to aggregate information available among several events (typically log lines) belonging to a same task, and finally push aggregated information into final task event."
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
+  s.authors = ["Elastic", "Fabien Baligand"]
+  s.email = 'info@elastic.co'
+  s.homepage = "https://github.com/logstash-plugins/logstash-filter-aggregate"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE']
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
+  s.add_development_dependency 'logstash-devutils', '~> 0'
+end

data/spec/filters/aggregate_spec.rb

@@ -1,185 +1,210 @@
-# encoding: utf-8
-require "logstash/devutils/rspec/spec_helper"
-require "logstash/filters/aggregate"
-require_relative "aggregate_spec_helper"
-
-describe LogStash::Filters::Aggregate do
-
-  before(:each) do
-    set_eviction_instance(nil)
-    aggregate_maps.clear()
-    @start_filter = setup_filter({ "map_action" => "create", "code" => "map['sql_duration'] = 0" })
-    @update_filter = setup_filter({ "map_action" => "update", "code" => "map['sql_duration'] += event['duration']" })
-    @end_filter = setup_filter({ "map_action" => "update", "code" => "event.to_hash.merge!(map)", "end_of_task" => true, "timeout" => 5 })
-  end
-
-  context "Start event" do
-    describe "and receiving an event without task_id" do
-      it "does not record it" do
-        @start_filter.filter(event())
-        expect(aggregate_maps).to be_empty
-      end
-    end
-    describe "and receiving an event with task_id" do
-      it "records it" do
-        event = start_event("taskid" => "id123")
-        @start_filter.filter(event)
-
-        expect(aggregate_maps.size).to eq(1)
-        expect(aggregate_maps["id123"]).not_to be_nil
-        expect(aggregate_maps["id123"].creation_timestamp).to be >= event["@timestamp"]
-        expect(aggregate_maps["id123"].map["sql_duration"]).to eq(0)
-      end
-    end
-
-    describe "and receiving two 'start events' for the same task_id" do
-      it "keeps the first one and does nothing with the second one" do
-
-        first_start_event = start_event("taskid" => "id124")
-        @start_filter.filter(first_start_event)
-        
-        first_update_event = update_event("taskid" => "id124", "duration" => 2)
-        @update_filter.filter(first_update_event)
-        
-        sleep(1)
-        second_start_event = start_event("taskid" => "id124")
-        @start_filter.filter(second_start_event)
-
-        expect(aggregate_maps.size).to eq(1)
-        expect(aggregate_maps["id124"].creation_timestamp).to be < second_start_event["@timestamp"]
-        expect(aggregate_maps["id124"].map["sql_duration"]).to eq(first_update_event["duration"])
-      end
-    end
-  end
-
-  context "End event" do
-    describe "receiving an event without a previous 'start event'" do
-      describe "but without a previous 'start event'" do
-        it "does nothing with the event" do
-          end_event = end_event("taskid" => "id124")
-          @end_filter.filter(end_event)
-
-          expect(aggregate_maps).to be_empty
-          expect(end_event["sql_duration"]).to be_nil
-        end
-      end
-    end
-  end
-
-  context "Start/end events interaction" do
-    describe "receiving a 'start event'" do
-      before(:each) do
-        @task_id_value = "id_123"
-        @start_event = start_event({"taskid" => @task_id_value})
-        @start_filter.filter(@start_event)
-        expect(aggregate_maps.size).to eq(1)
-      end
-
-      describe "and receiving an end event" do
-        describe "and without an id" do
-          it "does nothing" do
-            end_event = end_event()
-            @end_filter.filter(end_event)
-            expect(aggregate_maps.size).to eq(1)
-            expect(end_event["sql_duration"]).to be_nil
-          end
-        end
-
-        describe "and an id different from the one of the 'start event'" do
-          it "does nothing" do
-            different_id_value = @task_id_value + "_different"
-            @end_filter.filter(end_event("taskid" => different_id_value))
-
-            expect(aggregate_maps.size).to eq(1)
-            expect(aggregate_maps[@task_id_value]).not_to be_nil
-          end
-        end
-
-        describe "and the same id of the 'start event'" do
-          it "add 'sql_duration' field to the end event and deletes the aggregate map associated to taskid" do
-            expect(aggregate_maps.size).to eq(1)
-
-            @update_filter.filter(update_event("taskid" => @task_id_value, "duration" => 2))
-
-            end_event = end_event("taskid" => @task_id_value)
-            @end_filter.filter(end_event)
-
-            expect(aggregate_maps).to be_empty
-            expect(end_event["sql_duration"]).to eq(2)
-          end
-
-        end
-      end
-    end
-  end
-
-  context "Event with integer task id" do
-    it "works as well as with a string task id" do
-      start_event = start_event("taskid" => 124)
-      @start_filter.filter(start_event)
-      expect(aggregate_maps.size).to eq(1)
-    end
-  end
-
-  context "Event which causes an exception when code call" do
-    it "intercepts exception, logs the error and tags the event with '_aggregateexception'" do
-      @start_filter = setup_filter({ "code" => "fail 'Test'" })
-      start_event = start_event("taskid" => "id124")
-      @start_filter.filter(start_event)
-
-      expect(start_event["tags"]).to eq(["_aggregateexception"])
-    end
-  end
-
-  context "flush call" do
-    before(:each) do
-      @end_filter.timeout = 1
-      expect(@end_filter.timeout).to eq(1)
-      @task_id_value = "id_123"
-      @start_event = start_event({"taskid" => @task_id_value})
-      @start_filter.filter(@start_event)
-      expect(aggregate_maps.size).to eq(1)
-    end
-
-    describe "no timeout defined in none filter" do
-      it "defines a default timeout on a default filter" do
-        set_eviction_instance(nil)
-        expect(eviction_instance).to be_nil
-        @end_filter.flush()
-        expect(eviction_instance).to eq(@end_filter)
-        expect(@end_filter.timeout).to eq(LogStash::Filters::Aggregate::DEFAULT_TIMEOUT)
-      end
-    end
-
-    describe "timeout is defined on another filter" do
-      it "eviction_instance is not updated" do
-        expect(eviction_instance).not_to be_nil
-        @start_filter.flush()
-        expect(eviction_instance).not_to eq(@start_filter)
-        expect(eviction_instance).to eq(@end_filter)
-      end
-    end
-
-    describe "no timeout defined on the filter" do
-      it "event is not removed" do
-        sleep(2)
-        @start_filter.flush()
-        expect(aggregate_maps.size).to eq(1)
-      end
-    end
-
-    describe "timeout defined on the filter" do
-      it "event is not removed if not expired" do
-        @end_filter.flush()
-        expect(aggregate_maps.size).to eq(1)
-      end
-      it "event is removed if expired" do
-        sleep(2)
-        @end_filter.flush()
-        expect(aggregate_maps).to be_empty
-      end
-    end
-
-  end
-
-end
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/filters/aggregate"
+require_relative "aggregate_spec_helper"
+
+describe LogStash::Filters::Aggregate do
+
+  before(:each) do
+    set_eviction_instance(nil)
+    aggregate_maps.clear()
+    @start_filter = setup_filter({ "map_action" => "create", "code" => "map['sql_duration'] = 0" })
+    @update_filter = setup_filter({ "map_action" => "update", "code" => "map['sql_duration'] += event['duration']" })
+    @end_filter = setup_filter({ "map_action" => "update", "code" => "event.to_hash.merge!(map)", "end_of_task" => true, "timeout" => 5 })
+  end
+
+  context "Start event" do
+    describe "and receiving an event without task_id" do
+      it "does not record it" do
+        @start_filter.filter(event())
+        expect(aggregate_maps).to be_empty
+      end
+    end
+    describe "and receiving an event with task_id" do
+      it "records it" do
+        event = start_event("taskid" => "id123")
+        @start_filter.filter(event)
+
+        expect(aggregate_maps.size).to eq(1)
+        expect(aggregate_maps["id123"]).not_to be_nil
+        expect(aggregate_maps["id123"].creation_timestamp).to be >= event["@timestamp"]
+        expect(aggregate_maps["id123"].map["sql_duration"]).to eq(0)
+      end
+    end
+
+    describe "and receiving two 'start events' for the same task_id" do
+      it "keeps the first one and does nothing with the second one" do
+
+        first_start_event = start_event("taskid" => "id124")
+        @start_filter.filter(first_start_event)
+        
+        first_update_event = update_event("taskid" => "id124", "duration" => 2)
+        @update_filter.filter(first_update_event)
+        
+        sleep(1)
+        second_start_event = start_event("taskid" => "id124")
+        @start_filter.filter(second_start_event)
+
+        expect(aggregate_maps.size).to eq(1)
+        expect(aggregate_maps["id124"].creation_timestamp).to be < second_start_event["@timestamp"]
+        expect(aggregate_maps["id124"].map["sql_duration"]).to eq(first_update_event["duration"])
+      end
+    end
+  end
+
+  context "End event" do
+    describe "receiving an event without a previous 'start event'" do
+      describe "but without a previous 'start event'" do
+        it "does nothing with the event" do
+          end_event = end_event("taskid" => "id124")
+          @end_filter.filter(end_event)
+
+          expect(aggregate_maps).to be_empty
+          expect(end_event["sql_duration"]).to be_nil
+        end
+      end
+    end
+  end
+
+  context "Start/end events interaction" do
+    describe "receiving a 'start event'" do
+      before(:each) do
+        @task_id_value = "id_123"
+        @start_event = start_event({"taskid" => @task_id_value})
+        @start_filter.filter(@start_event)
+        expect(aggregate_maps.size).to eq(1)
+      end
+
+      describe "and receiving an end event" do
+        describe "and without an id" do
+          it "does nothing" do
+            end_event = end_event()
+            @end_filter.filter(end_event)
+            expect(aggregate_maps.size).to eq(1)
+            expect(end_event["sql_duration"]).to be_nil
+          end
+        end
+
+        describe "and an id different from the one of the 'start event'" do
+          it "does nothing" do
+            different_id_value = @task_id_value + "_different"
+            @end_filter.filter(end_event("taskid" => different_id_value))
+
+            expect(aggregate_maps.size).to eq(1)
+            expect(aggregate_maps[@task_id_value]).not_to be_nil
+          end
+        end
+
+        describe "and the same id of the 'start event'" do
+          it "add 'sql_duration' field to the end event and deletes the aggregate map associated to taskid" do
+            expect(aggregate_maps.size).to eq(1)
+
+            @update_filter.filter(update_event("taskid" => @task_id_value, "duration" => 2))
+
+            end_event = end_event("taskid" => @task_id_value)
+            @end_filter.filter(end_event)
+
+            expect(aggregate_maps).to be_empty
+            expect(end_event["sql_duration"]).to eq(2)
+          end
+
+        end
+      end
+    end
+  end
+
+  context "Event with integer task id" do
+    it "works as well as with a string task id" do
+      start_event = start_event("taskid" => 124)
+      @start_filter.filter(start_event)
+      expect(aggregate_maps.size).to eq(1)
+    end
+  end
+
+  context "Event which causes an exception when code call" do
+    it "intercepts exception, logs the error and tags the event with '_aggregateexception'" do
+      @start_filter = setup_filter({ "code" => "fail 'Test'" })
+      start_event = start_event("taskid" => "id124")
+      @start_filter.filter(start_event)
+
+      expect(start_event["tags"]).to eq(["_aggregateexception"])
+    end
+  end
+
+  context "flush call" do
+    before(:each) do
+      @end_filter.timeout = 1
+      expect(@end_filter.timeout).to eq(1)
+      @task_id_value = "id_123"
+      @start_event = start_event({"taskid" => @task_id_value})
+      @start_filter.filter(@start_event)
+      expect(aggregate_maps.size).to eq(1)
+    end
+
+    describe "no timeout defined in none filter" do
+      it "defines a default timeout on a default filter" do
+        set_eviction_instance(nil)
+        expect(eviction_instance).to be_nil
+        @end_filter.flush()
+        expect(eviction_instance).to eq(@end_filter)
+        expect(@end_filter.timeout).to eq(LogStash::Filters::Aggregate::DEFAULT_TIMEOUT)
+      end
+    end
+
+    describe "timeout is defined on another filter" do
+      it "eviction_instance is not updated" do
+        expect(eviction_instance).not_to be_nil
+        @start_filter.flush()
+        expect(eviction_instance).not_to eq(@start_filter)
+        expect(eviction_instance).to eq(@end_filter)
+      end
+    end
+
+    describe "no timeout defined on the filter" do
+      it "event is not removed" do
+        sleep(2)
+        @start_filter.flush()
+        expect(aggregate_maps.size).to eq(1)
+      end
+    end
+
+    describe "timeout defined on the filter" do
+      it "event is not removed if not expired" do
+        @end_filter.flush()
+        expect(aggregate_maps.size).to eq(1)
+      end
+      it "event is removed if expired" do
+        sleep(2)
+        @end_filter.flush()
+        expect(aggregate_maps).to be_empty
+      end
+    end
+
+  end
+
+  context "aggregate_maps_path option is defined, " do
+    describe "close event append then register event append, " do
+      it "stores aggregate maps to configured file and then loads aggregate maps from file" do
+        
+        store_file = "aggregate_maps"
+        expect(File.exist?(store_file)).to be false
+          
+        store_filter = setup_filter({ "code" => "map['sql_duration'] = 0", "aggregate_maps_path" => store_file })
+        expect(aggregate_maps).to be_empty
+
+        start_event = start_event("taskid" => 124)
+        filter = store_filter.filter(start_event)
+        expect(aggregate_maps.size).to eq(1)
+        
+        store_filter.close()
+        expect(File.exist?(store_file)).to be true
+        expect(aggregate_maps).to be_empty
+
+        store_filter = setup_filter({ "code" => "map['sql_duration'] = 0", "aggregate_maps_path" => store_file })
+        expect(File.exist?(store_file)).to be false
+        expect(aggregate_maps.size).to eq(1)
+        
+      end
+    end
+  end
+end

data/spec/filters/aggregate_spec_helper.rb

@@ -1,49 +1,49 @@
-# encoding: utf-8
-require "logstash/filters/aggregate"
-
-def event(data = {})
-	data["message"] ||= "Log message"
-	data["@timestamp"] ||= Time.now
-	LogStash::Event.new(data)
-end
-
-def start_event(data = {})
-	data["logger"] = "TASK_START"
-	event(data)
-end
-
-def update_event(data = {})
-	data["logger"] = "SQL"
-	event(data)
-end
-
-def end_event(data = {})
-	data["logger"] = "TASK_END"
-	event(data)
-end
-
-def setup_filter(config = {})
-	config["task_id"] ||= "%{taskid}"
-	filter = LogStash::Filters::Aggregate.new(config)
-	filter.register()
-	return filter
-end
-
-def filter(event)
-	@start_filter.filter(event)
-	@update_filter.filter(event)
-	@end_filter.filter(event)
-end
-
-def aggregate_maps()
-	LogStash::Filters::Aggregate.class_variable_get(:@@aggregate_maps)
-end
-
-def eviction_instance()
-	LogStash::Filters::Aggregate.class_variable_get(:@@eviction_instance)
-end
-
-def set_eviction_instance(new_value)
-	LogStash::Filters::Aggregate.class_variable_set(:@@eviction_instance, new_value)
-end
-
+# encoding: utf-8
+require "logstash/filters/aggregate"
+
+def event(data = {})
+	data["message"] ||= "Log message"
+	data["@timestamp"] ||= Time.now
+	LogStash::Event.new(data)
+end
+
+def start_event(data = {})
+	data["logger"] = "TASK_START"
+	event(data)
+end
+
+def update_event(data = {})
+	data["logger"] = "SQL"
+	event(data)
+end
+
+def end_event(data = {})
+	data["logger"] = "TASK_END"
+	event(data)
+end
+
+def setup_filter(config = {})
+	config["task_id"] ||= "%{taskid}"
+	filter = LogStash::Filters::Aggregate.new(config)
+	filter.register()
+	return filter
+end
+
+def filter(event)
+	@start_filter.filter(event)
+	@update_filter.filter(event)
+	@end_filter.filter(event)
+end
+
+def aggregate_maps()
+	LogStash::Filters::Aggregate.class_variable_get(:@@aggregate_maps)
+end
+
+def eviction_instance()
+	LogStash::Filters::Aggregate.class_variable_get(:@@eviction_instance)
+end
+
+def set_eviction_instance(new_value)
+	LogStash::Filters::Aggregate.class_variable_set(:@@eviction_instance, new_value)
+end
+