logstash-filter-elasticsearch 3.3.0 → 3.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 13f03ed46b5acfa84d22de961c7be68dfba316ccf9b4c3930405aa83454008ec
-  data.tar.gz: 32c91902583d770b2f98ec0a65d6b5f24006e11509978d90e19cb14c10828852
+  metadata.gz: f0392952d180a690b095f3639f5b7e6b02ab3bb37df033c7d1cbc588a563060c
+  data.tar.gz: a5673bdde9c5f497b1ab6e258d8b90bd193ca0a63d20f73c09f2a06e1fdafe60
 SHA512:
-  metadata.gz: 10989b0b2dc5369c11b82b7d88b6dd4e899ace3dcb2b010051daf6db1fb672ce9762a1faa87d153652b52c659864a163b22512428a1f8a8f5df5c7ae6141a7ff
-  data.tar.gz: 5c4a7e1cf126047839893544073942f0ca48d438d19519b7c7a7c63f299de108dde3c0fdf107b5cc2d9e38f51ec09907af1e77ce7e3004adff4b4bfe26bf2ad6
+  metadata.gz: e18f6af27ef9effc88d17fc96e7146c67efb2ea575a7a851c4e9e7edb2d860d91202da7ad49bb2bed0a2581fc0ac68f8fff522afa08b7eac282aafe809eefe95
+  data.tar.gz: 78cfbdd8d6204b645921bf5efe3e293968a65117f957487df021aa581d48000fa90927cf1662ce7e39b180fb44d250e8db76d6172654afdf1cc51992b4b6eae2

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 3.3.1
+  - Fix: The filter now only calls `filter_matched` on events that actually matched.
+    This fixes issues where all events would have success-related actions happened
+    when no match had actually happened (`add_tag`, `add_field`, `remove_tag`,
+    `remove_field`)
+
 ## 3.3.0
   - Enhancement : if elasticsearch response contains any shard failure, then `tag_on_failure` tags are added to Logstash event
   - Enhancement : add support for nested fields

data/docs/index.asciidoc

@@ -40,50 +40,56 @@ filter to find the matching "start" event based on some operation identifier.
 Then it copies the `@timestamp` field from the "start" event into a new field on
 the "end" event.  Finally, using a combination of the "date" filter and the
 "ruby" filter, we calculate the time duration in hours between the two events.
+
 [source,ruby]
 --------------------------------------------------
-      if [type] == "end" {
-         elasticsearch {
-            hosts => ["es-server"]
-            query => "type:start AND operation:%{[opid]}"
-            fields => { "@timestamp" => "started" }
-         }
-
-         date {
-            match => ["[started]", "ISO8601"]
-            target => "[started]"
-         }
-
-         ruby {
-            code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600)"
-         }
-      }
-
- The example below reproduces the above example but utilises the query_template.  This query_template represents a full
- Elasticsearch query DSL and supports the standard Logstash field substitution syntax.  The example below issues
- the same query as the first example but uses the template shown.
-
-  if [type] == "end" {
-         elasticsearch {
-            hosts => ["es-server"]
-            query_template => "template.json"
-            fields => { "@timestamp" => "started" }
-         }
+if [type] == "end" {
+   elasticsearch {
+      hosts => ["es-server"]
+      query => "type:start AND operation:%{[opid]}"
+      fields => { "@timestamp" => "started" }
+   }
+
+   date {
+      match => ["[started]", "ISO8601"]
+      target => "[started]"
+   }
+
+   ruby {
+      code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600)"
+   }
+}
+--------------------------------------------------
 
-         date {
-            match => ["[started]", "ISO8601"]
-            target => "[started]"
-         }
+The example below reproduces the above example but utilises the query_template.
+This query_template represents a full Elasticsearch query DSL and supports the
+standard Logstash field substitution syntax.  The example below issues
+the same query as the first example but uses the template shown.
 
-         ruby {
-            code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600)"
-         }
-  }
+[source,ruby]
+--------------------------------------------------
+if [type] == "end" {
+      elasticsearch {
+         hosts => ["es-server"]
+         query_template => "template.json"
+         fields => { "@timestamp" => "started" }
+      }
 
+      date {
+         match => ["[started]", "ISO8601"]
+         target => "[started]"
+      }
 
+      ruby {
+         code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600)"
+      }
+}
+--------------------------------------------------
 
-  template.json:
+template.json:
 
+[source,json]
+--------------------------------------------------
  {
     "query": {
       "query_string": {
@@ -92,12 +98,31 @@ the "end" event.  Finally, using a combination of the "date" filter and the
     },
    "_source": ["@timestamp"]
  }
+--------------------------------------------------
 
-As illustrated above, through the use of 'opid', fields from the Logstash events can be referenced within the template.
+As illustrated above, through the use of 'opid', fields from the Logstash
+events can be referenced within the template.
 The template will be populated per event prior to being used to query Elasticsearch.
 
+Note that when you use `query_template`, the Logstash attributes `result_size`
+and `sort` will be ignored. They should be specified directly in the JSON
+template. Example:
+
+[source,json]
+--------------------------------------------------
+{
+  "size": 1,
+  "sort" : [ { "@timestamp" : "desc" } ],
+  "query": {
+    "query_string": {
+      "query": "type:start AND operation:%{[opid]}"
+    }
+  },
+  "_source": ["@timestamp"]
+}
 --------------------------------------------------
 
+
 [id="plugins-{type}s-{plugin}-options"]
 ==== Elasticsearch Filter Configuration Options
 

data/lib/logstash/filters/elasticsearch.rb

@@ -74,6 +74,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   end # def register
 
   def filter(event)
+    matched = false
     begin
       params = {:index => event.sprintf(@index) }
 
@@ -94,6 +95,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
 
       resultsHits = results["hits"]["hits"]
       if !resultsHits.nil? && !resultsHits.empty?
+        matched = true
         @fields.each do |old_key, new_key|
           old_key_path = extract_path(old_key)
           set = resultsHits.map do |doc|
@@ -112,6 +114,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
 
       resultsAggs = results["aggregations"]
       if !resultsAggs.nil? && !resultsAggs.empty?
+        matched = true
         @aggregation_fields.each do |agg_name, ls_field|
           event.set(ls_field, resultsAggs[agg_name])
         end
@@ -120,8 +123,9 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
     rescue => e
       @logger.warn("Failed to query elasticsearch for previous event", :index => @index, :query => query, :event => event, :error => e)
       @tag_on_failure.each{|tag| event.tag(tag)}
+    else
+      filter_matched(event) if matched
     end
-    filter_matched(event)
   end # def filter
 
   private

data/logstash-filter-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-elasticsearch'
-  s.version         = '3.3.0'
+  s.version         = '3.3.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Copies fields from previous log events in Elasticsearch to current events "
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/elasticsearch_spec.rb

@@ -124,13 +124,77 @@ describe LogStash::Filters::Elasticsearch do
       end
     end
 
+    # Tagging test for positive results
+    context "Tagging should occur if query returns results" do
+      let(:config) do
+        {
+          "index" => "foo*",
+          "hosts" => ["localhost:9200"],
+          "query" => "response: 404",
+          "add_tag" => ["tagged"]
+        }
+      end
+
+      let(:response) do
+        LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_x_10.json")))
+      end
+
+      it "should tag the current event if results returned" do
+        plugin.filter(event)
+        expect(event.to_hash["tags"]).to include("tagged")
+      end
+    end
+
+    context "an aggregation search with size 0 that matches" do
+      let(:config) do
+        {
+          "index" => "foo*",
+          "hosts" => ["localhost:9200"],
+          "query" => "response: 404",
+          "add_tag" => ["tagged"],
+          "result_size" => 0,
+          "aggregation_fields" => { "bytes_avg" => "bytes_avg_ls_field" }
+        }
+      end
+
+      let(:response) do
+        LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_size0_agg.json")))
+      end
+
+      it "should tag the current event" do
+        plugin.filter(event)
+        expect(event.get("tags")).to include("tagged")
+        expect(event.get("bytes_avg_ls_field")["value"]).to eq(294)
+      end
+    end
+
+    # Tagging test for negative results
+    context "Tagging should not occur if query has no results" do
+      let(:config) do
+        {
+          "index" => "foo*",
+          "hosts" => ["localhost:9200"],
+          "query" => "response: 404",
+          "add_tag" => ["tagged"]
+        }
+      end
+
+      let(:response) do
+        LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_error.json")))
+      end
+
+      it "should not tag the current event" do
+        plugin.filter(event)
+        expect(event.to_hash["tags"]).to_not include("tagged")
+      end
+    end
     context "testing a simple query template" do
       let(:config) do
         {
-            "hosts" => ["localhost:9200"],
-            "query_template" => File.join(File.dirname(__FILE__), "fixtures", "query_template.json"),
-            "fields" => { "response" => "code" },
-            "result_size" => 1
+          "hosts" => ["localhost:9200"],
+          "query_template" => File.join(File.dirname(__FILE__), "fixtures", "query_template.json"),
+          "fields" => { "response" => "code" },
+          "result_size" => 1
         }
       end
 

data/spec/filters/fixtures/request_size0_agg.json

@@ -0,0 +1,19 @@
+{
+	"took": 49,
+	"timed_out": false,
+	"_shards": {
+		"total": 155,
+		"successful": 155,
+		"failed": 0
+	},
+	"hits": {
+		"total": 13476,
+		"max_score": 1,
+		"hits": []
+	},
+	"aggregations": {
+		"bytes_avg": {
+			"value": 294
+		}
+	}
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.3.0
+  version: 3.3.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-01-15 00:00:00.000000000 Z
+date: 2018-05-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -85,6 +85,7 @@ files:
 - spec/filters/elasticsearch_spec.rb
 - spec/filters/fixtures/query_template.json
 - spec/filters/fixtures/request_error.json
+- spec/filters/fixtures/request_size0_agg.json
 - spec/filters/fixtures/request_x_1.json
 - spec/filters/fixtures/request_x_10.json
 - spec/filters/integration/elasticsearch_spec.rb
@@ -118,6 +119,7 @@ test_files:
 - spec/filters/elasticsearch_spec.rb
 - spec/filters/fixtures/query_template.json
 - spec/filters/fixtures/request_error.json
+- spec/filters/fixtures/request_size0_agg.json
 - spec/filters/fixtures/request_x_1.json
 - spec/filters/fixtures/request_x_10.json
 - spec/filters/integration/elasticsearch_spec.rb