logstash-input-elasticsearch 4.2.0 → 4.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c800b8e0032e2d43897b1c4e0dab3cf4d6d22c14919fcdefe7b7df8ec4a64ffa
-  data.tar.gz: 063e672f41db0c0711c7d2229fd05b5dbef0d28fed9f34afe5c2c2ad57ff1dc2
+  metadata.gz: 72f28226e8297df8d4eda501ef9a99a67c11bab0574711f553797c678cd5e9a8
+  data.tar.gz: d141edd1c0664b1f77f4ddc757cda564781aeaa2ada1ebee828ac2f76268c1de
 SHA512:
-  metadata.gz: 63503ac4073666f88eca84c6af52d2598ba425b4ca7b16ec6c77bcb97def7083119943ea7f940536bd49a8cad581d2afb502634a00e336a223f2abff9842038d
-  data.tar.gz: 5d2ec1e87c76a79f590d3f93dc8e2667597b74aba4f1503282f75de34ce98e6c6cac4d49ddcc86668669eb7706dc97a312fb93f7bad3b88fa2ff98608cd9bc94
+  metadata.gz: 468f91f896d037895c7479e7ae377cee60c4b0c22b05918695f35b6e56bdfa405524fd04d03440133ac40a71fa4ed00251ad27e38792f7c30a8b736ffaa186c9
+  data.tar.gz: 5ba240ae691f622ab6bf410b7603ebbb959ee5c50cddc90748dd26a1a349fcc36057e9147a15d0501ea1e4f0bbc2861f3042c0addc070568ec805cdd6668d999

data/CHANGELOG.md

@@ -1,3 +1,18 @@
+## 4.3.3
+  - Loosen restrictions on Elasticsearch gem [#110](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/110)
+
+## 4.3.2
+  - Fixed broken link to Elasticsearch Reference  [#106](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/106)
+
+## 4.3.1
+  - Fixed deeplink to Elasticsearch Reference  [#103](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/103)
+
+## 4.3.0
+  - Added managed slice scrolling with `slices` option
+
+## 4.2.1
+  - Docs: Set the default_codec doc attribute.
+
 ## 4.2.0
   - Docs: Deprecate `document_type`
   - Add support for scheduling periodic execution of the query #81

data/docs/index.asciidoc

@@ -1,5 +1,6 @@
 :plugin: elasticsearch
 :type: input
+:default_codec: json
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!
@@ -96,6 +97,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-schedule>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-scroll>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-size>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-slices>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-user>> |<<string,string>>|No
 |=======================================================================
@@ -161,11 +163,10 @@ It will be removed in the next major version of Logstash.
   * Value type is <<array,array>>
   * Default value is `["_index", "_type", "_id"]`
 
-If document metadata storage is requested by enabling the `docinfo`
-option, this option lists the metadata fields to save in the current
-event. See
-http://www.elasticsearch.org/guide/en/elasticsearch/guide/current/_document_metadata.html[Document Metadata]
-in the Elasticsearch documentation for more information.
+If document metadata storage is requested by enabling the `docinfo` option, this
+option lists the metadata fields to save in the current event. See
+{ref}/mapping-fields.html[Meta-Fields] in the Elasticsearch documentation for
+more information.
 
 [id="plugins-{type}s-{plugin}-docinfo_target"]
 ===== `docinfo_target` 
@@ -193,7 +194,11 @@ can be either IP, HOST, IP:port, or HOST:port. The port defaults to
   * Value type is <<string,string>>
   * Default value is `"logstash-*"`
 
-The index or alias to search.
+The index or alias to search. See
+https://www.elastic.co/guide/en/elasticsearch/reference/current/multi-index.html[Multi Indices documentation]
+in the Elasticsearch documentation for more information on how to reference
+multiple indices.
+
 
 [id="plugins-{type}s-{plugin}-password"]
 ===== `password` 
@@ -245,6 +250,30 @@ round trip (i.e. between the previous scroll request, to the next).
 
 This allows you to set the maximum number of hits returned per scroll.
 
+[id="plugins-{type}s-{plugin}-slices"]
+===== `slices`
+
+  * Value type is <<number,number>>
+  * There is no default value.
+  * Sensible values range from 2 to about 8.
+
+In some cases, it is possible to improve overall throughput by consuming multiple
+distinct slices of a query simultaneously using the
+https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-body.html#sliced-scroll[Sliced Scroll API],
+especially if the pipeline is spending significant time waiting on Elasticsearch
+to provide results.
+
+If set, the `slices` parameter tells the plugin how many slices to divide the work
+into, and will produce events from the slices in parallel until all of them are done
+scrolling.
+
+NOTE: The Elasticsearch manual indicates that there can be _negative_ performance
+      implications to both the query and the Elasticsearch cluster when a scrolling
+      query uses more slices than shards in the index.
+
+If the `slices` parameter is left unset, the plugin will _not_ inject slice
+instructions into the query.
+
 [id="plugins-{type}s-{plugin}-ssl"]
 ===== `ssl` 
 
@@ -268,3 +297,5 @@ empty string authentication will be disabled.
 
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]
+
+:default_codec!:

data/lib/logstash/inputs/elasticsearch.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 require "logstash/inputs/base"
 require "logstash/namespace"
+require "logstash/json"
 require "base64"
 
 # .Compatibility Note
@@ -83,6 +84,10 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # round trip (i.e. between the previous scroll request, to the next).
   config :scroll, :validate => :string, :default => "1m"
 
+  # This parameter controls the number of parallel slices to be consumed simultaneously
+  # by this pipeline input.
+  config :slices, :validate => :number
+
   # If set, include Elasticsearch document information such as index, type, and
   # the id in the event.
   #
@@ -147,10 +152,14 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
 
     @options = {
       :index => @index,
-      :body => @query,
       :scroll => @scroll,
       :size => @size
     }
+    @base_query = LogStash::Json.load(@query)
+    if @slices
+      @base_query.include?('slice') && fail(LogStash::ConfigurationError, "Elasticsearch Input Plugin's `query` option cannot specify specific `slice` when configured to manage parallel slices with `slices` option")
+      @slices < 1 && fail(LogStash::ConfigurationError, "Elasticsearch Input Plugin's `slices` option must be greater than zero, got `#{@slices}`")
+    end
 
     transport_options = {}
 
@@ -196,16 +205,39 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   private
 
   def do_run(output_queue)
-    # get first wave of data
-    r = @client.search(@options)
+    # if configured to run a single slice, don't bother spinning up threads
+    return do_run_slice(output_queue) if @slices.nil? || @slices <= 1
+
+    logger.warn("managed slices for query is very large (#{@slices}); consider reducing") if @slices > 8
+
+    @slices.times.map do |slice_id|
+      Thread.new do
+        LogStash::Util::set_thread_name("#{@id}_slice_#{slice_id}")
+        do_run_slice(output_queue, slice_id)
+      end
+    end.map(&:join)
+  end
+
+  def do_run_slice(output_queue, slice_id=nil)
+    slice_query = @base_query
+    slice_query = slice_query.merge('slice' => { 'id' => slice_id, 'max' => @slices}) unless slice_id.nil?
+
+    slice_options = @options.merge(:body => LogStash::Json.dump(slice_query) )
+
+    logger.info("Slice starting", slice_id: slice_id, slices: @slices) unless slice_id.nil?
+    r = search_request(slice_options)
 
     r['hits']['hits'].each { |hit| push_hit(hit, output_queue) }
+    logger.debug("Slice progress", slice_id: slice_id, slices: @slices) unless slice_id.nil?
+
     has_hits = r['hits']['hits'].any?
 
-    while has_hits && !stop?
+    while has_hits && r['_scroll_id'] && !stop?
       r = process_next_scroll(output_queue, r['_scroll_id'])
+      logger.debug("Slice progress", slice_id: slice_id, slices: @slices) unless slice_id.nil?
       has_hits = r['has_hits']
     end
+    logger.info("Slice complete", slice_id: slice_id, slices: @slices) unless slice_id.nil?
   end
 
   def process_next_scroll(output_queue, scroll_id)
@@ -243,4 +275,8 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   def scroll_request scroll_id
     @client.scroll(:body => { :scroll_id => scroll_id }, :scroll => @scroll)
   end
+
+  def search_request(options)
+    @client.search(options)
+  end
 end

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '4.2.0'
+  s.version         = '4.3.3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -22,7 +22,7 @@ Gem::Specification.new do |s|
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
 
-  s.add_runtime_dependency 'elasticsearch', ['>= 5.0.3', '< 6.0.0']
+  s.add_runtime_dependency 'elasticsearch', '>= 5.0.3'
 
   s.add_runtime_dependency 'logstash-codec-json'
   s.add_runtime_dependency 'logstash-codec-plain'

data/spec/es_helper.rb

@@ -0,0 +1,40 @@
+module ESHelper
+  def self.get_host_port
+    return "http://elasticsearch:9200" if ENV["INTEGRATION"] == "true"
+    raise "This setting is only used for integration tests"
+  end
+
+  def self.get_client
+    Elasticsearch::Client.new(:hosts => [get_host_port])
+  end
+
+  def self.doc_type
+    if ESHelper.es_version_satisfies?(">=8")
+      nil
+    elsif ESHelper.es_version_satisfies?(">=7")
+      "_doc"
+    else
+      "doc"
+    end
+  end
+
+  def self.index_doc(es, params)
+    type = doc_type
+    params[:type] = doc_type unless type.nil?
+    es.index(params)
+  end
+
+  def self.es_version
+    ENV['ES_VERSION'] || ENV['ELASTIC_STACK_VERSION']
+  end
+
+  def self.es_version_satisfies?(*requirement)
+    es_version = RSpec.configuration.filter[:es_version] || ENV['ES_VERSION'] || ENV['ELASTIC_STACK_VERSION']
+    if es_version.nil?
+      puts "Info: ES_VERSION, ELASTIC_STACK_VERSION or 'es_version' tag wasn't set. Returning false to all `es_version_satisfies?` call."
+      return false
+    end
+    es_release_version = Gem::Version.new(es_version).release
+    Gem::Requirement.new(requirement).satisfied_by?(es_release_version)
+  end
+end

data/spec/inputs/elasticsearch_spec.rb

@@ -84,6 +84,239 @@ describe LogStash::Inputs::Elasticsearch do
     insist { event.get("message") } == [ "ohayo" ]
   end
 
+
+  # This spec is an adapter-spec, ensuring that we send the right sequence of messages to our Elasticsearch Client
+  # to support sliced scrolling. The underlying implementation will spawn its own threads to consume, so we must be
+  # careful to use thread-safe constructs.
+  context "with managed sliced scrolling" do
+    let(:config) do
+      {
+          'query' => "#{LogStash::Json.dump(query)}",
+          'slices' => slices,
+          'docinfo' => true, # include ids
+      }
+    end
+    let(:query) do
+      {
+        "query" => {
+          "match" => { "city_name" => "Okinawa" }
+        },
+        "fields" => ["message"]
+      }
+    end
+    let(:slices) { 2 }
+
+    context 'with `slices => 0`' do
+      let(:slices) { 0 }
+      it 'fails to register' do
+        expect { plugin.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+
+    context 'with `slices => 1`' do
+      let(:slices) { 1 }
+      it 'runs just one slice' do
+        expect(plugin).to receive(:do_run_slice).with(duck_type(:<<))
+        expect(Thread).to_not receive(:new)
+
+        plugin.register
+        plugin.run([])
+      end
+    end
+
+    context 'without slices directive' do
+      let(:config) { super.tap { |h| h.delete('slices') } }
+      it 'runs just one slice' do
+        expect(plugin).to receive(:do_run_slice).with(duck_type(:<<))
+        expect(Thread).to_not receive(:new)
+
+        plugin.register
+        plugin.run([])
+      end
+    end
+
+    2.upto(8) do |slice_count|
+      context "with `slices => #{slice_count}`" do
+        let(:slices) { slice_count }
+        it "runs #{slice_count} independent slices" do
+          expect(Thread).to receive(:new).and_call_original.exactly(slice_count).times
+          slice_count.times do |slice_id|
+            expect(plugin).to receive(:do_run_slice).with(duck_type(:<<), slice_id)
+          end
+
+          plugin.register
+          plugin.run([])
+        end
+      end
+    end
+
+    # This section of specs heavily mocks the Elasticsearch::Client, and ensures that the Elasticsearch Input Plugin
+    # behaves as expected when handling a series of sliced, scrolled requests/responses.
+    context 'adapter/integration' do
+      let(:response_template) do
+        {
+            "took" => 12,
+            "timed_out" => false,
+            "shards" => {
+                "total" => 6,
+                "successful" => 6,
+                "failed" => 0
+            }
+        }
+      end
+
+      let(:hits_template) do
+        {
+            "total" => 4,
+            "max_score" => 1.0,
+            "hits" => []
+        }
+      end
+
+      let(:hit_template) do
+        {
+            "_index" => "logstash-2018.08.23",
+            "_type" => "logs",
+            "_score" => 1.0,
+            "_source" => { "message" => ["hello, world"] }
+        }
+      end
+
+      # BEGIN SLICE 0: a sequence of THREE scrolled responses containing 2, 1, and 0 items
+      # end-of-slice is reached when slice0_response2 is empty.
+      begin
+        let(:slice0_response0) do
+          response_template.merge({
+              "_scroll_id" => slice0_scroll1,
+              "hits" => hits_template.merge("hits" => [
+                  hit_template.merge('_id' => "slice0-response0-item0"),
+                  hit_template.merge('_id' => "slice0-response0-item1")
+                  ])
+          })
+        end
+        let(:slice0_scroll1) { 'slice:0,scroll:1' }
+        let(:slice0_response1) do
+          response_template.merge({
+              "_scroll_id" => slice0_scroll2,
+              "hits" => hits_template.merge("hits" => [
+                  hit_template.merge('_id' => "slice0-response1-item0")
+              ])
+          })
+        end
+        let(:slice0_scroll2) { 'slice:0,scroll:2' }
+        let(:slice0_response2) do
+          response_template.merge(
+              "_scroll_id" => slice0_scroll3,
+              "hits" => hits_template.merge({"hits" => []})
+          )
+        end
+        let(:slice0_scroll3) { 'slice:0,scroll:3' }
+      end
+      # END SLICE 0
+
+      # BEGIN SLICE 1: a sequence of TWO scrolled responses containing 2 and 2 items.
+      # end-of-slice is reached when slice1_response1 does not contain a next scroll id
+      begin
+        let(:slice1_response0) do
+          response_template.merge({
+              "_scroll_id" => slice1_scroll1,
+              "hits" => hits_template.merge("hits" => [
+                  hit_template.merge('_id' => "slice1-response0-item0"),
+                  hit_template.merge('_id' => "slice1-response0-item1")
+              ])
+          })
+        end
+        let(:slice1_scroll1) { 'slice:1,scroll:1' }
+        let(:slice1_response1) do
+          response_template.merge({
+              "hits" => hits_template.merge("hits" => [
+                  hit_template.merge('_id' => "slice1-response1-item0"),
+                  hit_template.merge('_id' => "slice1-response1-item1")
+              ])
+          })
+        end
+      end
+      # END SLICE 1
+
+      let(:client) { Elasticsearch::Client.new }
+
+      # RSpec mocks validations are not threadsafe.
+      # Allow caller to synchronize.
+      def synchronize_method!(object, method_name)
+        original_method = object.method(method_name)
+        mutex = Mutex.new
+        allow(object).to receive(method_name).with(any_args) do |*method_args, &method_block|
+          mutex.synchronize do
+            original_method.call(*method_args,&method_block)
+          end
+        end
+      end
+
+      before(:each) do
+        expect(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
+        plugin.register
+
+        # SLICE0 is a three-page scroll in which the last page is empty
+        slice0_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 0, 'max' => 2}))
+        expect(client).to receive(:search).with(hash_including(:body => slice0_query)).and_return(slice0_response0)
+        expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll1 })).and_return(slice0_response1)
+        expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll2 })).and_return(slice0_response2)
+
+        # SLICE1 is a two-page scroll in which the last page has no next scroll id
+        slice1_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 1, 'max' => 2}))
+        expect(client).to receive(:search).with(hash_including(:body => slice1_query)).and_return(slice1_response0)
+        expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice1_scroll1 })).and_return(slice1_response1)
+
+        synchronize_method!(plugin, :scroll_request)
+        synchronize_method!(plugin, :search_request)
+      end
+
+      let(:emitted_events) do
+        queue = Queue.new # since we are running slices in threads, we need a thread-safe queue.
+        plugin.run(queue)
+        events = []
+        events << queue.pop until queue.empty?
+        events
+      end
+
+      let(:emitted_event_ids) do
+        emitted_events.map { |event| event.get('[@metadata][_id]') }
+      end
+
+      it 'emits the hits on the first page of the first slice' do
+        expect(emitted_event_ids).to include('slice0-response0-item0')
+        expect(emitted_event_ids).to include('slice0-response0-item1')
+      end
+      it 'emits the hits on the second page of the first slice' do
+        expect(emitted_event_ids).to include('slice0-response1-item0')
+      end
+
+      it 'emits the hits on the first page of the second slice' do
+        expect(emitted_event_ids).to include('slice1-response0-item0')
+        expect(emitted_event_ids).to include('slice1-response0-item1')
+      end
+
+      it 'emits the hitson the second page of the second slice' do
+        expect(emitted_event_ids).to include('slice1-response1-item0')
+        expect(emitted_event_ids).to include('slice1-response1-item1')
+      end
+
+      it 'does not double-emit' do
+        expect(emitted_event_ids.uniq).to eq(emitted_event_ids)
+      end
+
+      it 'emits events with appropriate fields' do
+        emitted_events.each do |event|
+          expect(event).to be_a(LogStash::Event)
+          expect(event.get('message')).to eq(['hello, world'])
+          expect(event.get('[@metadata][_id]')).to_not be_nil
+          expect(event.get('[@metadata][_id]')).to_not be_empty
+          expect(event.get('[@metadata][_index]')).to start_with('logstash-')
+        end
+      end
+    end
+  end
+
   context "with Elasticsearch document information" do
     let!(:response) do
       {
@@ -142,15 +375,13 @@ describe LogStash::Inputs::Elasticsearch do
       end
 
       it 'merges the values if the `docinfo_target` already exist in the `_source` document' do
-        metadata_field = 'metadata_with_hash'
-
         config_metadata_with_hash = %Q[
             input {
               elasticsearch {
                 hosts => ["localhost"]
                 query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
                 docinfo => true
-                docinfo_target => '#{metadata_field}'
+                docinfo_target => 'metadata_with_hash'
               }
             }
         ]
@@ -159,33 +390,23 @@ describe LogStash::Inputs::Elasticsearch do
           queue.pop
         end
 
-        expect(event.get("[#{metadata_field}][_index]")).to eq('logstash-2014.10.12')
-        expect(event.get("[#{metadata_field}][_type]")).to eq('logs')
-        expect(event.get("[#{metadata_field}][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
-        expect(event.get("[#{metadata_field}][awesome]")).to eq("logstash")
+        expect(event.get("[metadata_with_hash][_index]")).to eq('logstash-2014.10.12')
+        expect(event.get("[metadata_with_hash][_type]")).to eq('logs')
+        expect(event.get("[metadata_with_hash][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
+        expect(event.get("[metadata_with_hash][awesome]")).to eq("logstash")
       end
 
-      it 'thows an exception if the `docinfo_target` exist but is not of type hash' do
-        metadata_field = 'metadata_with_string'
-
-        config_metadata_with_string = %Q[
-            input {
-              elasticsearch {
-                hosts => ["localhost"]
-                query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-                docinfo => true
-                docinfo_target => '#{metadata_field}'
-              }
-            }
-        ]
-
-        pipeline = new_pipeline_from_string(config_metadata_with_string)
-        queue = Queue.new
-        pipeline.instance_eval do
-          @output_func = lambda { |event| queue << event }
+      context 'if the `docinfo_target` exist but is not of type hash' do
+        let (:config) { {
+            "hosts" => ["localhost"],
+            "query" => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }',
+            "docinfo" => true,
+            "docinfo_target" => 'metadata_with_string'
+        } }
+        it 'thows an exception if the `docinfo_target` exist but is not of type hash' do
+          plugin.register
+          expect { plugin.run([]) }.to raise_error(Exception, /incompatible event/)
         end
-
-        expect { pipeline.run }.to raise_error(Exception, /incompatible event/)
       end
 
       it "should move the document info to the @metadata field" do

data/spec/inputs/integration/elasticsearch_spec.rb

@@ -0,0 +1,43 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/plugin"
+require "logstash/inputs/elasticsearch"
+require_relative "../../../spec/es_helper"
+
+describe LogStash::Inputs::Elasticsearch, :integration => true do
+
+  let(:config)   { { 'hosts' => [ESHelper.get_host_port],
+                     'index' => 'logs',
+                     'query' => '{ "query": { "match": { "message": "Not found"} }}' } }
+  let(:plugin) { described_class.new(config) }
+  let(:event)  { LogStash::Event.new({}) }
+
+  before(:each) do
+    @es = ESHelper.get_client
+    # Delete all templates first.
+    # Clean ES of data before we start.
+    @es.indices.delete_template(:name => "*")
+    # This can fail if there are no indexes, ignore failure.
+    @es.indices.delete(:index => "*") rescue nil
+    10.times do
+      ESHelper.index_doc(@es, :index => 'logs', :body => { :response => 404, :message=> 'Not Found'})
+    end
+    @es.indices.refresh
+    plugin.register
+  end
+
+  after(:each) do
+    @es.indices.delete_template(:name => "*")
+    @es.indices.delete(:index => "*") rescue nil
+  end
+
+  describe 'smoke test' do
+    it "should retrieve json event from elasticseach" do
+      queue = []
+      plugin.run(queue)
+      event = queue.pop
+      expect(event).to be_a(LogStash::Event)
+      expect(event.get("response")).to eql(404)
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.2.0
+  version: 4.3.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-01-15 00:00:00.000000000 Z
+date: 2019-11-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -36,9 +36,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 5.0.3
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: 6.0.0
   name: elasticsearch
   prerelease: false
   type: :runtime
@@ -47,9 +44,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 5.0.3
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: 6.0.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -179,7 +173,9 @@ files:
 - docs/index.asciidoc
 - lib/logstash/inputs/elasticsearch.rb
 - logstash-input-elasticsearch.gemspec
+- spec/es_helper.rb
 - spec/inputs/elasticsearch_spec.rb
+- spec/inputs/integration/elasticsearch_spec.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -207,4 +203,6 @@ signing_key:
 specification_version: 4
 summary: Reads query results from an Elasticsearch cluster
 test_files:
+- spec/es_helper.rb
 - spec/inputs/elasticsearch_spec.rb
+- spec/inputs/integration/elasticsearch_spec.rb