logstash-filter-elasticsearch 3.3.1 → 3.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f0392952d180a690b095f3639f5b7e6b02ab3bb37df033c7d1cbc588a563060c
-  data.tar.gz: a5673bdde9c5f497b1ab6e258d8b90bd193ca0a63d20f73c09f2a06e1fdafe60
+  metadata.gz: 92a2e2d6660ea8cf72381f7e7bd6832525e580c083b1f0d7188cbea58e32f2ee
+  data.tar.gz: 9cf2ac564ac0347e9f0642c0efaef8b4ac59f8725d9a9eef250805e8cde03656
 SHA512:
-  metadata.gz: e18f6af27ef9effc88d17fc96e7146c67efb2ea575a7a851c4e9e7edb2d860d91202da7ad49bb2bed0a2581fc0ac68f8fff522afa08b7eac282aafe809eefe95
-  data.tar.gz: 78cfbdd8d6204b645921bf5efe3e293968a65117f957487df021aa581d48000fa90927cf1662ce7e39b180fb44d250e8db76d6172654afdf1cc51992b4b6eae2
+  metadata.gz: 913c2c225cfa517bf983ca3ee9c85263f0008e305b27dae4619afb8ffaa535e36f2618449b5da2de498dae4bfeb7e3f8dd2d654b3e0499d60f2383e99e92865b
+  data.tar.gz: b47a71caee22bf6b95908ec48ea9ce8a17b73c619fe48453a3ed4fe1470906bb149d4c852a1b8aba22e1b003ab0ad8ec33fcf558dfff56ea464cd9b69e4dc04e

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 3.4.0
+  - Adds `[@metadata][total_hits]` with total hits returned from the query ([#106](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/106))
+  - Improves error logging to fully inspect caught exceptions ([#105](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/105))
+
 ## 3.3.1
   - Fix: The filter now only calls `filter_matched` on events that actually matched.
     This fixes issues where all events would have success-related actions happened

data/docs/index.asciidoc

@@ -89,25 +89,6 @@ if [type] == "end" {
 template.json:
 
 [source,json]
---------------------------------------------------
- {
-    "query": {
-      "query_string": {
-       "query": "type:start AND operation:%{[opid]}"
-      }
-    },
-   "_source": ["@timestamp"]
- }
---------------------------------------------------
-
-As illustrated above, through the use of 'opid', fields from the Logstash
-events can be referenced within the template.
-The template will be populated per event prior to being used to query Elasticsearch.
-
-Note that when you use `query_template`, the Logstash attributes `result_size`
-and `sort` will be ignored. They should be specified directly in the JSON
-template. Example:
-
 [source,json]
 --------------------------------------------------
 {
@@ -122,6 +103,14 @@ template. Example:
 }
 --------------------------------------------------
 
+As illustrated above, through the use of 'opid', fields from the Logstash
+events can be referenced within the template.
+The template will be populated per event prior to being used to query Elasticsearch.
+
+Notice also that when you use `query_template`, the Logstash attributes `result_size`
+and `sort` will be ignored. They should be specified directly in the JSON
+template, as shown in the example above.
+
 
 [id="plugins-{type}s-{plugin}-options"]
 ==== Elasticsearch Filter Configuration Options
@@ -212,7 +201,20 @@ Whether results should be sorted or not
   * Value type is <<array,array>>
   * Default value is `{}`
 
-Array of fields to copy from old event (found via elasticsearch) into new event
+An array of fields to copy from the old event (found via elasticsearch) into the
+new event, currently being processed.
+
+In the following example, the values of `@timestamp` and `event_id` on the event
+found via elasticsearch are copied to the current event's
+`started` and `start_id` fields, respectively:
+
+[source,ruby]
+--------------------------------------------------
+fields => {
+  "@timestamp" => "started"
+  "event_id" => "start_id"
+}
+--------------------------------------------------
 
 [id="plugins-{type}s-{plugin}-hosts"]
 ===== `hosts` 

data/lib/logstash/filters/elasticsearch.rb

@@ -93,6 +93,8 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
       results = get_client.search(params)
       raise "Elasticsearch query error: #{results["_shards"]["failures"]}" if results["_shards"].include? "failures"
 
+      event.set("[@metadata][total_hits]", results['hits']['total'])
+
       resultsHits = results["hits"]["hits"]
       if !resultsHits.nil? && !resultsHits.empty?
         matched = true
@@ -121,7 +123,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
       end
 
     rescue => e
-      @logger.warn("Failed to query elasticsearch for previous event", :index => @index, :query => query, :event => event, :error => e)
+      @logger.warn("Failed to query elasticsearch for previous event", :index => @index, :query => query, :event => event, :error => e.inspect)
       @tag_on_failure.each{|tag| event.tag(tag)}
     else
       filter_matched(event) if matched

data/logstash-filter-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-elasticsearch'
-  s.version         = '3.3.1'
+  s.version         = '3.4.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Copies fields from previous log events in Elasticsearch to current events "
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.3.1
+  version: 3.4.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-05-08 00:00:00.000000000 Z
+date: 2018-09-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement