logstash-filter-elasticsearch 3.5.0 → 3.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 57bc6bc825a1866b1ecf39eb35bcab5e062dfb73f7b1e7590feb4fe51358b2c9
-  data.tar.gz: 48df2bb55103e2fb6196ec8679f1b48ce70742f43dcd9ab3c5ff257637ccdda8
+  metadata.gz: d8d5c99d58129ffad85468e4bab7c64f184a0bdc2aa8b8a11c8f52353c8e4876
+  data.tar.gz: 30b1a189484e118138d6f7607be7319e035696464e8682b59122f830063b0c8d
 SHA512:
-  metadata.gz: e93655c43a05df8d1f7cf261ff552a875cb765111903cf9bc4903e77cb423329f346f1b67d949b6cae24c85872b8ea45a7a08e7ce812d48ab93117d575236eef
-  data.tar.gz: b3dbca71043c7cbb6ef904e19e30b88dd6e974d92c0fdf77eae52d90c1d41151bdc9b0ef4c9cc93fa8c97937dc31e30c6570a62a8343d693a0b1d00bb2d4fa6a
+  metadata.gz: 6ef80c1576cf3d84f3da23bc23ecf7931812675d3566d46a074f6c8901dc182a34a67da8c17f7a6beafd33407c81d4a392c8d604b54abbaad3e542dd2260b987
+  data.tar.gz: 7b754511d4a252de292646d67929b067d4fc44d7eabad53fb300098320592292fc0afac1d200737e6203fe7bd7422a9a1ea7b43e028d1d194879adb7a39788fb

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.6.0
+  - Add support for extracting hits total from Elasticsearch 7.x responses
+
 ## 3.5.0
   - Added connection check during register to avoid failures during processing
   - Changed Elasticsearch Client transport to use Manticore

data/lib/logstash/filters/elasticsearch.rb

@@ -94,7 +94,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
       results = get_client.search(params)
       raise "Elasticsearch query error: #{results["_shards"]["failures"]}" if results["_shards"].include? "failures"
 
-      event.set("[@metadata][total_hits]", results['hits']['total'])
+      event.set("[@metadata][total_hits]", extract_total_from_hits(results['hits']))
 
       resultsHits = results["hits"]["hits"]
       if !resultsHits.nil? && !resultsHits.empty?
@@ -173,6 +173,21 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
     end
   end
 
+  # Given a "hits" object from an Elasticsearch response, return the total number of hits in
+  # the result set.
+  # @param hits [Hash{String=>Object}]
+  # @return [Integer]
+  def extract_total_from_hits(hits)
+    total = hits['total']
+
+    # Elasticsearch 7.x produces an object containing `value` and `relation` in order
+    # to enable unambiguous reporting when the total is only a lower bound; if we get
+    # an object back, return its `value`.
+    return total['value'] if total.kind_of?(Hash)
+
+    total
+  end
+
   def test_connection!
     get_client.client.ping
   end

data/logstash-filter-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-elasticsearch'
-  s.version         = '3.5.0'
+  s.version         = '3.6.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Copies fields from previous log events in Elasticsearch to current events "
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/elasticsearch_spec.rb

@@ -113,6 +113,26 @@ describe LogStash::Filters::Elasticsearch do
       end
     end
 
+    context 'when Elasticsearch 7.x gives us a totals object instead of an integer' do
+      let(:config) do
+        {
+            "hosts" => ["localhost:9200"],
+            "query" => "response: 404",
+            "fields" => { "response" => "code" },
+            "result_size" => 10
+        }
+      end
+
+      let(:response) do
+        LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "elasticsearch_7.x_hits_total_as_object.json")))
+      end
+
+      it "should enhance the current event with new data" do
+        plugin.filter(event)
+        expect(event.get("[@metadata][total_hits]")).to eq(13476)
+      end
+    end
+
     context "if something wrong happen during connection" do
 
       before(:each) do

data/spec/filters/fixtures/elasticsearch_7.x_hits_total_as_object.json

@@ -0,0 +1,70 @@
+{
+  "took": 49,
+  "timed_out": false,
+  "_shards": {
+    "total": 155,
+    "successful": 155,
+    "failed": 0
+  },
+  "hits": {
+    "total": {
+      "value": 13476,
+      "relation": "eq"
+    },
+    "max_score": 1,
+    "hits": [{
+      "_index": "logstash-2014.08.26",
+      "_type": "logs",
+      "_id": "AVVY76L_AW7v0kX8KXo4",
+      "_score": 1,
+      "_source": {
+        "request": "/doc/index.html?org/elasticsearch/action/search/SearchResponse.html",
+        "agent": "\"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\"",
+        "geoip": {
+          "timezone": "America/Los_Angeles",
+          "ip": "66.249.73.185",
+          "latitude": 37.386,
+          "continent_code": "NA",
+          "city_name": "Mountain View",
+          "country_code2": "US",
+          "country_name": "United States",
+          "dma_code": 807,
+          "country_code3": "US",
+          "region_name": "California",
+          "location": [-122.0838,
+            37.386
+          ],
+          "postal_code": "94035",
+          "longitude": -122.0838,
+          "region_code": "CA"
+        },
+        "auth": "-",
+        "ident": "-",
+        "verb": "GET",
+        "useragent": {
+          "os": "Other",
+          "major": "2",
+          "minor": "1",
+          "name": "Googlebot",
+          "os_name": "Other",
+          "device": "Spider"
+        },
+        "message": "66.249.73.185 - - [26/Aug/2014:21:22:13 +0000] \"GET /doc/index.html?org/elasticsearch/action/search/SearchResponse.html HTTP/1.1\" 404 294 \"-\" \"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\"",
+        "referrer": "\"-\"",
+        "@timestamp": "2014-08-26T21:22:13.000Z",
+        "response": 404,
+        "bytes": 294,
+        "clientip": "66.249.73.185",
+        "@version": "1",
+        "host": "skywalker",
+        "httpversion": "1.1",
+        "timestamp": "26/Aug/2014:21:22:13 +0000"
+      }
+    }]
+  },
+  "aggregations": {
+    "bytes_avg": {
+      "value": 294
+    }
+  }
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.5.0
+  version: 3.6.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-11-22 00:00:00.000000000 Z
+date: 2018-11-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -97,6 +97,7 @@ files:
 - lib/logstash/filters/elasticsearch/client.rb
 - logstash-filter-elasticsearch.gemspec
 - spec/filters/elasticsearch_spec.rb
+- spec/filters/fixtures/elasticsearch_7.x_hits_total_as_object.json
 - spec/filters/fixtures/query_template.json
 - spec/filters/fixtures/request_error.json
 - spec/filters/fixtures/request_size0_agg.json
@@ -131,6 +132,7 @@ specification_version: 4
 summary: Copies fields from previous log events in Elasticsearch to current events
 test_files:
 - spec/filters/elasticsearch_spec.rb
+- spec/filters/fixtures/elasticsearch_7.x_hits_total_as_object.json
 - spec/filters/fixtures/query_template.json
 - spec/filters/fixtures/request_error.json
 - spec/filters/fixtures/request_size0_agg.json