logstash-filter-elasticsearch 3.2.1 → 3.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7e05102f70c7e104e992f6452fe120e3a7c36908eff0e0927b096b5ee420b040
-  data.tar.gz: 0b7fa6d082030618ab5c1743bd9825432d538605d30a1da494e200747f7a510f
+  metadata.gz: 13f03ed46b5acfa84d22de961c7be68dfba316ccf9b4c3930405aa83454008ec
+  data.tar.gz: 32c91902583d770b2f98ec0a65d6b5f24006e11509978d90e19cb14c10828852
 SHA512:
-  metadata.gz: '08698e711d7f451102be1962aab48998dcb9cb4eca1560b8a314d6ddacb113fac494d811bd5dcf40d48f7b091c87cb3e29b48f4db3b745090f1dd8e140b8f93f'
-  data.tar.gz: 502324704c4752889da2ba85a7dcad9e8241b73766f3495b4f62754fef6d4c1c49cc3319eae9f62cc146b9b6bc95cf0a904ccea8f7c7547875e0754494fa2c5e
+  metadata.gz: 10989b0b2dc5369c11b82b7d88b6dd4e899ace3dcb2b010051daf6db1fb672ce9762a1faa87d153652b52c659864a163b22512428a1f8a8f5df5c7ae6141a7ff
+  data.tar.gz: 5c4a7e1cf126047839893544073942f0ca48d438d19519b7c7a7c63f299de108dde3c0fdf107b5cc2d9e38f51ec09907af1e77ce7e3004adff4b4bfe26bf2ad6

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 3.3.0
+  - Enhancement : if elasticsearch response contains any shard failure, then `tag_on_failure` tags are added to Logstash event
+  - Enhancement : add support for nested fields
+  - Enhancement : add 'docinfo_fields' option
+  - Enhancement : add 'aggregation_fields' option
+
 ## 3.2.1
   - Update gemspec summary
 

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012–2016 Elasticsearch <http://www.elastic.co>
+Copyright (c) 2012-2018 Elasticsearch <http://www.elastic.co>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/docs/index.asciidoc

@@ -55,7 +55,7 @@ the "end" event.  Finally, using a combination of the "date" filter and the
          }
 
          ruby {
-            code => "event['duration_hrs'] = (event['@timestamp'] - event['started']) / 3600 rescue nil"
+            code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600)"
          }
       }
 
@@ -67,6 +67,7 @@ the "end" event.  Finally, using a combination of the "date" filter and the
          elasticsearch {
             hosts => ["es-server"]
             query_template => "template.json"
+            fields => { "@timestamp" => "started" }
          }
 
          date {
@@ -75,7 +76,7 @@ the "end" event.  Finally, using a combination of the "date" filter and the
          }
 
          ruby {
-            code => "event['duration_hrs'] = (event['@timestamp'] - event['started']) / 3600 rescue nil"
+            code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600)"
          }
   }
 
@@ -89,7 +90,7 @@ the "end" event.  Finally, using a combination of the "date" filter and the
        "query": "type:start AND operation:%{[opid]}"
       }
     },
-   "_source": ["@timestamp", "started"]
+   "_source": ["@timestamp"]
  }
 
 As illustrated above, through the use of 'opid', fields from the Logstash events can be referenced within the template.
@@ -105,7 +106,9 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 [cols="<,<,<",options="header",]
 |=======================================================================
 |Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-aggregation_fields>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-docinfo_fields>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-enable_sort>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-fields>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-hosts>> |<<array,array>>|No
@@ -125,6 +128,24 @@ filter plugins.
 
 &nbsp;
 
+[id="plugins-{type}s-{plugin}-aggregation_fields"]
+===== `aggregation_fields` 
+
+  * Value type is <<hash,hash>>
+  * Default value is `{}`
+
+Hash of aggregation names to copy from elasticsearch response into Logstash event fields
+
+Example:
+[source,ruby]
+    filter {
+      elasticsearch {
+        aggregation_fields => {
+          "my_agg_name" => "my_ls_field"
+        }
+      }
+    }
+
 [id="plugins-{type}s-{plugin}-ca_file"]
 ===== `ca_file` 
 
@@ -133,6 +154,25 @@ filter plugins.
 
 SSL Certificate Authority file
 
+[id="plugins-{type}s-{plugin}-docinfo_fields"]
+===== `docinfo_fields` 
+
+  * Value type is <<hash,hash>>
+  * Default value is `{}`
+
+Hash of docinfo fields to copy from old event (found via elasticsearch) into new event
+
+Example:
+[source,ruby]
+    filter {
+      elasticsearch {
+        docinfo_fields => {
+          "_id" => "document_id"
+          "_index" => "document_index"
+        }
+      }
+    }
+
 [id="plugins-{type}s-{plugin}-enable_sort"]
 ===== `enable_sort` 
 

data/lib/logstash/filters/elasticsearch.rb

@@ -5,82 +5,6 @@ require_relative "elasticsearch/client"
 require "logstash/json"
 java_import "java.util.concurrent.ConcurrentHashMap"
 
-# .Compatibility Note
-# [NOTE]
-# ================================================================================
-# Starting with Elasticsearch 5.3, there's an {ref}modules-http.html[HTTP setting]
-# called `http.content_type.required`. If this option is set to `true`, and you
-# are using Logstash 2.4 through 5.2, you need to update the Elasticsearch filter
-# plugin to version 3.1.1 or higher.
-# 
-# ================================================================================
-# 
-# Search Elasticsearch for a previous log event and copy some fields from it
-# into the current event.  Below are two complete examples of how this filter might
-# be used.
-#
-# The first example uses the legacy 'query' parameter where the user is limited to an Elasticsearch query_string.
-# Whenever logstash receives an "end" event, it uses this elasticsearch
-# filter to find the matching "start" event based on some operation identifier.
-# Then it copies the `@timestamp` field from the "start" event into a new field on
-# the "end" event.  Finally, using a combination of the "date" filter and the
-# "ruby" filter, we calculate the time duration in hours between the two events.
-# [source,ruby]
-# --------------------------------------------------
-#       if [type] == "end" {
-#          elasticsearch {
-#             hosts => ["es-server"]
-#             query => "type:start AND operation:%{[opid]}"
-#             fields => { "@timestamp" => "started" }
-#          }
-#
-#          date {
-#             match => ["[started]", "ISO8601"]
-#             target => "[started]"
-#          }
-#
-#          ruby {
-#             code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600) rescue nil"
-#          }
-#       }
-#
-#  The example below reproduces the above example but utilises the query_template.  This query_template represents a full
-#  Elasticsearch query DSL and supports the standard Logstash field substitution syntax.  The example below issues
-#  the same query as the first example but uses the template shown.
-#
-#   if [type] == "end" {
-#          elasticsearch {
-#             hosts => ["es-server"]
-#             query_template => "template.json"
-#          }
-#
-#          date {
-#             match => ["started", "ISO8601"]
-#             target => "started"
-#          }
-#
-#          ruby {
-#             code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600) rescue nil"
-#          }
-#   }
-#
-#
-#
-#   template.json:
-#
-#  {
-#     "query": {
-#       "query_string": {
-#        "query": "type:start AND operation:%{[opid]}"
-#       }
-#     },
-#    "_source": ["@timestamp", "started"]
-#  }
-#
-# As illustrated above, through the use of 'opid', fields from the Logstash events can be referenced within the template.
-# The template will be populated per event prior to being used to query Elasticsearch.
-#
-# --------------------------------------------------
 
 class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   config_name "elasticsearch"
@@ -106,6 +30,12 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   # Array of fields to copy from old event (found via elasticsearch) into new event
   config :fields, :validate => :array, :default => {}
 
+  # Hash of docinfo fields to copy from old event (found via elasticsearch) into new event
+  config :docinfo_fields, :validate => :hash, :default => {}
+
+  # Hash of aggregation names to copy from elasticsearch response into Logstash event fields
+  config :aggregation_fields, :validate => :hash, :default => {}
+
   # Basic Auth - username
   config :user, :validate => :string
 
@@ -145,7 +75,6 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
 
   def filter(event)
     begin
-
       params = {:index => event.sprintf(@index) }
 
       if @query_dsl
@@ -161,15 +90,33 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
       @logger.debug("Querying elasticsearch for lookup", :params => params)
 
       results = get_client.search(params)
-      @fields.each do |old_key, new_key|
-        if !results['hits']['hits'].empty?
-          set = []
-          results["hits"]["hits"].to_a.each do |doc|
-            set << doc["_source"][old_key]
+      raise "Elasticsearch query error: #{results["_shards"]["failures"]}" if results["_shards"].include? "failures"
+
+      resultsHits = results["hits"]["hits"]
+      if !resultsHits.nil? && !resultsHits.empty?
+        @fields.each do |old_key, new_key|
+          old_key_path = extract_path(old_key)
+          set = resultsHits.map do |doc|
+            extract_value(doc["_source"], old_key_path)
+          end
+          event.set(new_key, set.count > 1 ? set : set.first)
+        end
+        @docinfo_fields.each do |old_key, new_key|
+          old_key_path = extract_path(old_key)
+          set = resultsHits.map do |doc|
+            extract_value(doc, old_key_path)
           end
           event.set(new_key, set.count > 1 ? set : set.first)
         end
       end
+
+      resultsAggs = results["aggregations"]
+      if !resultsAggs.nil? && !resultsAggs.empty?
+        @aggregation_fields.each do |agg_name, ls_field|
+          event.set(ls_field, resultsAggs[agg_name])
+        end
+      end
+
     rescue => e
       @logger.warn("Failed to query elasticsearch for previous event", :index => @index, :query => query, :event => event, :error => e)
       @tag_on_failure.each{|tag| event.tag(tag)}
@@ -194,4 +141,22 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   def get_client
     @clients_pool.computeIfAbsent(Thread.current, lambda { |x| new_client })
   end
+
+  # get an array of path elements from a path reference
+  def extract_path(path_reference)
+    return [path_reference] unless path_reference.start_with?('[') && path_reference.end_with?(']')
+
+    path_reference[1...-1].split('][')
+  end
+
+  # given a Hash and an array of path fragments, returns the value at the path
+  # @param source [Hash{String=>Object}]
+  # @param path [Array{String}]
+  # @return [Object]
+  def extract_value(source, path)
+    path.reduce(source) do |memo, old_key_fragment|
+      break unless memo.include?(old_key_fragment)
+      memo[old_key_fragment]
+    end
+  end
 end #class LogStash::Filters::Elasticsearch

data/logstash-filter-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-elasticsearch'
-  s.version         = '3.2.1'
+  s.version         = '3.3.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Copies fields from previous log events in Elasticsearch to current events "
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/elasticsearch_spec.rb

@@ -20,7 +20,9 @@ describe LogStash::Filters::Elasticsearch do
       {
         "hosts" => ["localhost:9200"],
         "query" => "response: 404",
-        "fields" => [ ["response", "code"] ],
+        "fields" => { "response" => "code" },
+        "docinfo_fields" => { "_index" => "es_index" },
+        "aggregation_fields" => { "bytes_avg" => "bytes_avg_ls_field" }
       }
     end
     let(:plugin) { described_class.new(config) }
@@ -60,6 +62,8 @@ describe LogStash::Filters::Elasticsearch do
     it "should enhance the current event with new data" do
       plugin.filter(event)
       expect(event.get("code")).to eq(404)
+      expect(event.get("es_index")).to eq("logstash-2014.08.26")
+      expect(event.get("bytes_avg_ls_field")["value"]).to eq(294)
     end
 
     it "should receive all necessary params to perform the search" do
@@ -74,7 +78,7 @@ describe LogStash::Filters::Elasticsearch do
           "index" => "foo*",
           "hosts" => ["localhost:9200"],
           "query" => "response: 404",
-          "fields" => [ ["response", "code"] ],
+          "fields" => { "response" => "code" }
         }
       end
 
@@ -90,7 +94,7 @@ describe LogStash::Filters::Elasticsearch do
         {
           "hosts" => ["localhost:9200"],
           "query" => "response: 404",
-          "fields" => [ ["response", "code"] ],
+          "fields" => { "response" => "code" },
           "result_size" => 10
         }
       end
@@ -125,7 +129,7 @@ describe LogStash::Filters::Elasticsearch do
         {
             "hosts" => ["localhost:9200"],
             "query_template" => File.join(File.dirname(__FILE__), "fixtures", "query_template.json"),
-            "fields" => [ ["response", "code"] ],
+            "fields" => { "response" => "code" },
             "result_size" => 1
         }
       end
@@ -154,7 +158,7 @@ describe LogStash::Filters::Elasticsearch do
             "index" => "foo_%{subst_field}*",
             "hosts" => ["localhost:9200"],
             "query" => "response: 404",
-            "fields" => [["response", "code"]]
+            "fields" => { "response" => "code" }
         }
       end
 
@@ -162,6 +166,39 @@ describe LogStash::Filters::Elasticsearch do
         expect(client).to receive(:search).with({:q => "response: 404", :size => 1, :index => "foo_subst_value*", :sort => "@timestamp:desc"})
         plugin.filter(event)
       end
+    end
+
+    context "if query result errored but no exception is thrown" do
+      let(:response) do
+        LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_error.json")))
+      end
+
+      before(:each) do
+        allow(LogStash::Filters::ElasticsearchClient).to receive(:new).and_return(client)
+        allow(client).to receive(:search).and_return(response)
+        plugin.register
+      end
+
+      it "tag the event as something happened, but still deliver it" do
+        expect(plugin.logger).to receive(:warn)
+        plugin.filter(event)
+        expect(event.to_hash["tags"]).to include("_elasticsearch_lookup_failure")
+      end
+    end
+
+    context "if query is on nested field" do
+      let(:config) do
+        {
+            "hosts" => ["localhost:9200"],
+            "query" => "response: 404",
+            "fields" => [ ["[geoip][ip]", "ip_address"] ]
+        }
+      end
+
+      it "should enhance the current event with new data" do
+        plugin.filter(event)
+        expect(event.get("ip_address")).to eq("66.249.73.185")
+      end
 
     end
 

data/spec/filters/fixtures/request_error.json

@@ -0,0 +1,25 @@
+{
+	"took": 16,
+	"timed_out": false,
+	"_shards": {
+		"total": 155,
+		"successful": 100,
+		"failed": 55,
+		"failures": [
+			{
+				"shard": 0,
+				"index": "logstash-2014.08.26",
+				"node": "YI1MT0H-Q469pFgAVTXI2g",
+				"reason": {
+					"type": "search_parse_exception",
+					"reason": "No mapping found for [@timestamp] in order to sort on"
+				}
+			}
+		]
+	},
+	"hits": {
+		"total": 0,
+		"max_score": null,
+		"hits": []
+	}
+}

data/spec/filters/fixtures/request_x_1.json

@@ -58,5 +58,10 @@
 				"timestamp": "26/Aug/2014:21:22:13 +0000"
 			}
     }]
+	},
+	"aggregations": {
+		"bytes_avg": {
+			"value": 294
+		}
 	}
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.2.1
+  version: 3.3.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-11-07 00:00:00.000000000 Z
+date: 2018-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -84,6 +84,7 @@ files:
 - logstash-filter-elasticsearch.gemspec
 - spec/filters/elasticsearch_spec.rb
 - spec/filters/fixtures/query_template.json
+- spec/filters/fixtures/request_error.json
 - spec/filters/fixtures/request_x_1.json
 - spec/filters/fixtures/request_x_10.json
 - spec/filters/integration/elasticsearch_spec.rb
@@ -109,13 +110,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: Copies fields from previous log events in Elasticsearch to current events
 test_files:
 - spec/filters/elasticsearch_spec.rb
 - spec/filters/fixtures/query_template.json
+- spec/filters/fixtures/request_error.json
 - spec/filters/fixtures/request_x_1.json
 - spec/filters/fixtures/request_x_10.json
 - spec/filters/integration/elasticsearch_spec.rb