logstash-filter-elasticsearch 3.1.1 → 3.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fca76ef931de7a0a36ebe8a10c0797fbea7d5a40
-  data.tar.gz: 6d54f667037cc1376c2991b29e81ae08bbd86e56
+  metadata.gz: 2d8d89ca168ed0ebf323797e15f3fccdc42f25e8
+  data.tar.gz: 088c46969a6855d284b4a343ed81379d5600bc33
 SHA512:
-  metadata.gz: 208ffd629273a862f53b702fe4f7ad19c979e79e2d7519c02e8324472497bf529e8fa988e21100d63a2ed4aa5b0d5328e397318a8f088ff343eeb49e278744bb
-  data.tar.gz: e4b94567c7d54ac8c4f3a26ed9ce0730261f1827f1beb08492362a0f570244d858f90b1483cd0601b1f2aade5eed6cd8354bb2c333ec15f8e1a140b0aeace352
+  metadata.gz: 23c8a2066af25344e1e4f4226bb07583175a1b5566ebb342d97704dbd77f21fcc1a688edb23d49d2a1c9df7a946d04ecbe5c930f3463759cd1d4a4a829ecf4f3
+  data.tar.gz: 901f9323aed42e79b60a9fd10798372babe2ec439fbb2808456172b1dc22fae198310ce887eb5f0cb22e449f687adb41e3d54709f5352b38e50c4ba5c61fa039

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 3.1.3
+  - Change the queries loglevel from info to debug.
+
+## 3.1.2
+  - Docs: Add requirement to use version 3.1.1 or higher to support sending Content-Type headers.
+  
 ## 3.1.1
   - Upgrade es-ruby client to support correct content-type
 

data/CONTRIBUTORS

@@ -8,6 +8,7 @@ Contributors:
 * Pier-Hugues Pellerin (ph)
 * Richard Pijnenburg (electrical)
 * Suyog Rao (suyograo)
+* Adrian Solom (addrians)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/docs/index.asciidoc

@@ -0,0 +1,236 @@
+:plugin: elasticsearch
+:type: filter
+
+///////////////////////////////////////////
+START - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+:version: %VERSION%
+:release_date: %RELEASE_DATE%
+:changelog_url: %CHANGELOG_URL%
+:include_path: ../../../logstash/docs/include
+///////////////////////////////////////////
+END - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+
+[id="plugins-{type}-{plugin}"]
+
+=== Elasticsearch
+
+include::{include_path}/plugin_header.asciidoc[]
+
+==== Description
+
+.Compatibility Note
+[NOTE]
+================================================================================
+Starting with Elasticsearch 5.3, there's an {ref}modules-http.html[HTTP setting]
+called `http.content_type.required`. If this option is set to `true`, and you
+are using Logstash 2.4 through 5.2, you need to update the Elasticsearch filter
+plugin to version 3.1.1 or higher.
+
+================================================================================
+
+Search Elasticsearch for a previous log event and copy some fields from it
+into the current event.  Below are two complete examples of how this filter might
+be used.
+
+The first example uses the legacy 'query' parameter where the user is limited to an Elasticsearch query_string.
+Whenever logstash receives an "end" event, it uses this elasticsearch
+filter to find the matching "start" event based on some operation identifier.
+Then it copies the `@timestamp` field from the "start" event into a new field on
+the "end" event.  Finally, using a combination of the "date" filter and the
+"ruby" filter, we calculate the time duration in hours between the two events.
+[source,ruby]
+--------------------------------------------------
+      if [type] == "end" {
+         elasticsearch {
+            hosts => ["es-server"]
+            query => "type:start AND operation:%{[opid]}"
+            fields => { "@timestamp" => "started" }
+         }
+
+         date {
+            match => ["[started]", "ISO8601"]
+            target => "[started]"
+         }
+
+         ruby {
+            code => "event['duration_hrs'] = (event['@timestamp'] - event['started']) / 3600 rescue nil"
+         }
+      }
+
+ The example below reproduces the above example but utilises the query_template.  This query_template represents a full
+ Elasticsearch query DSL and supports the standard Logstash field substitution syntax.  The example below issues
+ the same query as the first example but uses the template shown.
+
+  if [type] == "end" {
+         elasticsearch {
+            hosts => ["es-server"]
+            query_template => "template.json"
+         }
+
+         date {
+            match => ["[started]", "ISO8601"]
+            target => "[started]"
+         }
+
+         ruby {
+            code => "event['duration_hrs'] = (event['@timestamp'] - event['started']) / 3600 rescue nil"
+         }
+  }
+
+
+
+  template.json:
+
+ {
+    "query": {
+      "query_string": {
+       "query": "type:start AND operation:%{[opid]}"
+      }
+    },
+   "_source": ["@timestamp", "started"]
+ }
+
+As illustrated above, through the use of 'opid', fields from the Logstash events can be referenced within the template.
+The template will be populated per event prior to being used to query Elasticsearch.
+
+--------------------------------------------------
+
+[id="plugins-{type}s-{plugin}-options"]
+==== Elasticsearch Filter Configuration Options
+
+This plugin supports the following configuration options plus the <<plugins-{type}s-common-options>> described later.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-enable_sort>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-fields>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-hosts>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-index>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-query>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-query_template>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-result_size>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-sort>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-tag_on_failure>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-user>> |<<string,string>>|No
+|=======================================================================
+
+Also see <<plugins-{type}s-common-options>> for a list of options supported by all
+filter plugins.
+
+&nbsp;
+
+[id="plugins-{type}s-{plugin}-ca_file"]
+===== `ca_file` 
+
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
+
+SSL Certificate Authority file
+
+[id="plugins-{type}s-{plugin}-enable_sort"]
+===== `enable_sort` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `true`
+
+Whether results should be sorted or not
+
+[id="plugins-{type}s-{plugin}-fields"]
+===== `fields` 
+
+  * Value type is <<array,array>>
+  * Default value is `{}`
+
+Array of fields to copy from old event (found via elasticsearch) into new event
+
+[id="plugins-{type}s-{plugin}-hosts"]
+===== `hosts` 
+
+  * Value type is <<array,array>>
+  * Default value is `["localhost:9200"]`
+
+List of elasticsearch hosts to use for querying.
+
+[id="plugins-{type}s-{plugin}-index"]
+===== `index` 
+
+  * Value type is <<string,string>>
+  * Default value is `""`
+
+Comma-delimited list of index names to search; use `_all` or empty string to perform the operation on all indices
+
+[id="plugins-{type}s-{plugin}-password"]
+===== `password` 
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+Basic Auth - password
+
+[id="plugins-{type}s-{plugin}-query"]
+===== `query` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Elasticsearch query string. Read the Elasticsearch query string documentation.
+for more info at: https://www.elastic.co/guide/en/elasticsearch/reference/master/query-dsl-query-string-query.html#query-string-syntax
+
+[id="plugins-{type}s-{plugin}-query_template"]
+===== `query_template` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+File path to elasticsearch query in DSL format. Read the Elasticsearch query documentation
+for more info at: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html
+
+[id="plugins-{type}s-{plugin}-result_size"]
+===== `result_size` 
+
+  * Value type is <<number,number>>
+  * Default value is `1`
+
+How many results to return
+
+[id="plugins-{type}s-{plugin}-sort"]
+===== `sort` 
+
+  * Value type is <<string,string>>
+  * Default value is `"@timestamp:desc"`
+
+Comma-delimited list of `<field>:<direction>` pairs that define the sort order
+
+[id="plugins-{type}s-{plugin}-ssl"]
+===== `ssl` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+SSL
+
+[id="plugins-{type}s-{plugin}-tag_on_failure"]
+===== `tag_on_failure` 
+
+  * Value type is <<array,array>>
+  * Default value is `["_elasticsearch_lookup_failure"]`
+
+Tags the event on failure to look up geo information. This can be used in later analysis.
+
+[id="plugins-{type}s-{plugin}-user"]
+===== `user` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Basic Auth - username
+
+
+
+include::{include_path}/{type}.asciidoc[]

data/lib/logstash/filters/elasticsearch.rb

@@ -4,6 +4,16 @@ require "logstash/namespace"
 require_relative "elasticsearch/client"
 require "logstash/json"
 
+# .Compatibility Note
+# [NOTE]
+# ================================================================================
+# Starting with Elasticsearch 5.3, there's an {ref}modules-http.html[HTTP setting]
+# called `http.content_type.required`. If this option is set to `true`, and you
+# are using Logstash 2.4 through 5.2, you need to update the Elasticsearch filter
+# plugin to version 3.1.1 or higher.
+# 
+# ================================================================================
+# 
 # Search Elasticsearch for a previous log event and copy some fields from it
 # into the current event.  Below are two complete examples of how this filter might
 # be used.
@@ -150,7 +160,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
         params[:sort] =  @sort if @enable_sort
       end
 
-      @logger.info("Querying elasticsearch for lookup", :params => params)
+      @logger.debug("Querying elasticsearch for lookup", :params => params)
 
       results = @client.search(params)
       @fields.each do |old_key, new_key|

data/logstash-filter-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-elasticsearch'
-  s.version         = '3.1.1'
+  s.version         = '3.1.3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Search elasticsearch for a previous log event and copy some fields from it into the current event"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.1.1
+  version: 3.1.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-02-07 00:00:00.000000000 Z
+date: 2017-05-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -76,6 +76,7 @@ files:
 - LICENSE
 - NOTICE.TXT
 - README.md
+- docs/index.asciidoc
 - lib/logstash/filters/elasticsearch.rb
 - lib/logstash/filters/elasticsearch/client.rb
 - logstash-filter-elasticsearch.gemspec