logstash-filter-foreach 0.2.0 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: '060786427644bdb8d829fd5f6f44adca1b1f7a48a3b0cea726cc1b3a92c71f1d'
-  data.tar.gz: 78ebd5c7a394f5ffa1ebbe7a3bc84506f4f1d25cca0f1dca1b1b1eea3937bbb6
+SHA1:
+  metadata.gz: c4dd177028408de6e56bf069a42bba2bacd89364
+  data.tar.gz: ce69da20c37d7121b650cb4258e1f329887abda4
 SHA512:
-  metadata.gz: 82d2d3c2b1f61085f19e082e2131ef42181608eb711c92e456814d5665def3ee01948f00ff167ce5cd8953c19e97b6e85dfa5810116009e7967f4a76d8c167b2
-  data.tar.gz: 7c3c7960e299d4b3ece6b22bc80520b7eee44cb949ff7ef3d04ea965da26e7fdf10f6a20ffffc53b4d9cf3a29452f3d7907afe1230bcfbb48a09c5af53990f65
+  metadata.gz: 27aee6615ab0eeda4f41de716975e7fe8c9dfbfc507ad455bb1f043f68cacb174265ce77c6450fba117470ead6d1fab0c0baf216a542c3fd6708d1357590b6c9
+  data.tar.gz: cf2e7fc701fdac53d74025ebcd6fd60fd3ca48d81a5e27951d074a29ce0857b7dd621c7fd8e9f9910e0e2ec7b3643374efdf5b4afd8994be9a4100d13ccd3807

data/lib/logstash/filters/foreach.rb

@@ -119,7 +119,7 @@ class LogStash::Filters::Foreach < LogStash::Filters::Base
 
             @logger.trace("Foreach plugin: elsif @@event_data.has_key?(task_id)");
 
-            @logger.warn("Foreach plugin: task_id whould be unique. Duplicate value found: '#{task_id}'. Passing through")
+            @logger.warn("Foreach plugin: task_id should be unique. Duplicate value found: '#{task_id}'. Passing through")
             event.tag(FAILURE_TAG)
             passthrough = true
 
@@ -163,7 +163,7 @@ class LogStash::Filters::Foreach < LogStash::Filters::Base
 
             @logger.trace("Foreach plugin: if !@@event_data.has_key?(task_id)");
 
-            @logger.warn("Foreach plugin: found `end` event fot task_id = '#{task_id}' without `start` event. Passing through")
+            @logger.warn("Foreach plugin: found `end` event for task_id = '#{task_id}' without `start` event. Passing through")
             event.tag(FAILURE_TAG)
             passthrough = true
 
@@ -186,18 +186,13 @@ class LogStash::Filters::Foreach < LogStash::Filters::Base
 
               event_data.lastevent_timestamp = Time.now()
 
-              configuration.join_fields.each do |join_field|
-                event_data.join_fields[join_field] += [*event.get(join_field)]
-              end
+              event_data.add_join_fields_values(event)
               event_data.counter -= 1
 
               if event_data.counter == 0
 
                 @logger.trace("Foreach plugin: if event_data.counter == 0");
 
-                configuration.join_fields.each do |join_field|
-                  event_data.initial_event.set(join_field, event_data.join_fields[join_field])
-                end
                 ret_event = event_data.event()
                 filter_matched(ret_event)
                 yield ret_event
@@ -233,10 +228,7 @@ class LogStash::Filters::Foreach < LogStash::Filters::Base
           if obj.lastevent_timestamp < Time.now() - obj.configuration.timeout
             if obj.counter < obj.sub_events_count
               @logger.warn("Foreach plugin: Flushing partly processed event with task_id = '#{obj.initial_event.sprintf(@task_id)}' after timeout = '#{obj.configuration.timeout.to_s}'")
-              obj.configuration.join_fields.each do |join_field|
-                obj.initial_event.set(join_field, obj.join_fields[join_field])
-              end
-              events_to_flush << obj.initial_event
+              events_to_flush << obj.event()
             else
               @logger.warn("Foreach plugin: Removing unprocessed event with task_id = '#{obj.initial_event.sprintf(@task_id)}' after timeout = '#{obj.configuration.timeout.to_s}'")
             end
@@ -291,8 +283,19 @@ class LogStash::Filters::Foreach::Element
     end
   end
 
+  def add_join_fields_values(event)
+    @configuration.join_fields.each do |join_field|
+      @join_fields[join_field] += [event.get(join_field)].flatten
+    end
+  end
+
   def event()
     e = @initial_event.clone
+    @configuration.join_fields.each do |join_field|
+      if @join_fields[join_field].length > 0
+        e.set(join_field, @join_fields[join_field])
+      end
+    end
     e.set('@metadata', @initial_metadata)
     return e
   end

data/logstash-filter-foreach.gemspec

@@ -1,12 +1,12 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-filter-foreach'
-  s.version       = '0.2.0'
+  s.version       = '0.2.3'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'Process filters for every array item'
   s.description   = 'Plugin splits event for every item in array, then you could process other filters for every item and then join event back'
   s.homepage      = 'https://github.com/IIIEII/logstash-filter-foreach'
-  s.authors       = ['IIIEII']
-  s.email         = 'al.iiieii@gmail.com'
+  s.authors       = ['IIIEII', 'Boris Gorbylev']
+  s.email         = ['al.iiieii@gmail.com', 'ekho@ekho.name']
   s.require_paths = ['lib']
 
   # Files

data/spec/filters/foreach_spec.rb

@@ -492,5 +492,89 @@ describe LogStash::Filters::Foreach do
         insist { subject.get("[@metadata][unchanged]") } == "unchanged_value"
       end
     end
+
+    describe "should not set empty arrays" do
+      let(:config) do
+        <<-CONFIG
+        filter {
+          foreach {
+            task_id => "%{task_id}"
+            array_field => "array"
+            join_fields => ["join", "join2"]
+          }
+
+          mutate {
+            add_field => { "join" => "%{array}_changed" }
+          }
+
+          foreach {
+            task_id => "%{task_id}"
+            end => true
+          }
+        }
+        CONFIG
+      end
+
+      sample("task_id" => 1, "array" => ["big", "bird", "sesame street"], "unchanged" => "unchanged_value") do
+        insist { subject.is_a?(LogStash::Event) } == true
+        insist { subject.get("array").is_a?(Array) } == true
+        insist { subject.get("array") } == ["big", "bird", "sesame street"]
+        insist { subject.get("join").is_a?(Array) } == true
+        insist { subject.get("join") } == ["big_changed", "bird_changed", "sesame street_changed"]
+        insist { subject.get("join2").nil? } == true
+      end
+    end
+
+    describe "should work with nested loops" do
+      let(:config) do
+        <<-CONFIG
+        filter {
+          foreach {
+            task_id => "%{task_id}"
+            array_field => "array"
+            join_fields => ["join", "join2"]
+          }
+
+          mutate {
+            add_field => { "join" => "%{[array][str]}_changed" }
+          }
+
+          foreach {
+            task_id => "%{task_id}_%{[array][str]}"
+            array_field => "[array][nested]"
+            join_fields => ["join2"]
+          }
+          
+          mutate {
+            add_field => { "join2" => [ "%{[array][nested]}_changed", "%{[array][nested]}_changed2" ] }
+          }
+
+          foreach {
+            task_id => "%{task_id}_%{[array][str]}"
+            end => true
+          }
+
+          foreach {
+            task_id => "%{task_id}"
+            end => true
+          }
+        }
+        CONFIG
+      end
+
+      sample("task_id" => 1, "array" => [{"str" => "big", "nested" => ["nested_big1", "nested_big2"]}, {"str" => "bird", "nested" => ["nested_bird1", "nested_bird2"]}, {"str" => "sesame street", "nested" => ["nested_sesame street1", "nested_sesame street2"]}], "unchanged" => "unchanged_value") do
+        insist { subject.is_a?(LogStash::Event) } == true
+        insist { subject.get("join").is_a?(Array) } == true
+        insist { subject.get("join") } == ["big_changed", "bird_changed", "sesame street_changed"]
+        insist { subject.get("join2").is_a?(Array) } == true
+        insist { subject.get("join2") } == [
+            "nested_big1_changed", "nested_big1_changed2", "nested_big2_changed", "nested_big2_changed2",
+            "nested_bird1_changed", "nested_bird1_changed2", "nested_bird2_changed", "nested_bird2_changed2",
+            "nested_sesame street1_changed", "nested_sesame street1_changed2", "nested_sesame street2_changed", "nested_sesame street2_changed2"
+        ]
+        insist { subject.get("unchanged").is_a?(String) } == true
+        insist { subject.get("unchanged") } == "unchanged_value"
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-foreach
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.3
 platform: ruby
 authors:
 - IIIEII
+- Boris Gorbylev
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-08-17 00:00:00.000000000 Z
+date: 2018-02-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -74,7 +75,9 @@ dependencies:
         version: 1.3.1
 description: Plugin splits event for every item in array, then you could process other
   filters for every item and then join event back
-email: al.iiieii@gmail.com
+email:
+- al.iiieii@gmail.com
+- ekho@ekho.name
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -111,7 +114,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.11
+rubygems_version: 2.6.14
 signing_key:
 specification_version: 4
 summary: Process filters for every array item