logstash-output-elasticsearch 3.0.2-java → 4.1.0-java

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -1,12 +1,12 @@
 require "logstash/outputs/elasticsearch"
 require "cabin"
 require "base64"
-require "elasticsearch"
-require "elasticsearch/transport/transport/http/manticore"
+require 'logstash/outputs/elasticsearch/http_client/pool'
+require 'logstash/outputs/elasticsearch/http_client/manticore_adapter'
 
 module LogStash; module Outputs; class ElasticSearch;
   class HttpClient
-    attr_reader :client, :options, :client_options, :sniffer_thread
+    attr_reader :client, :options, :logger, :pool, :action_count, :recv_count
     # This is here in case we use DEFAULT_OPTIONS in the future
     # DEFAULT_OPTIONS = {
     #   :setting => value
@@ -17,71 +17,50 @@ module LogStash; module Outputs; class ElasticSearch;
       # Again, in case we use DEFAULT_OPTIONS in the future, uncomment this.
       # @options = DEFAULT_OPTIONS.merge(options)
       @options = options
-      @client = build_client(@options)
+      @pool = build_pool(@options)
       # mutex to prevent requests and sniffing to access the
       # connection pool at the same time
-      @request_mutex = Mutex.new
-      start_sniffing!
     end
 
     def template_install(name, template, force=false)
-      @request_mutex.synchronize do
-        if template_exists?(name) && !force
-          @logger.debug("Found existing Elasticsearch template. Skipping template management", :name => name)
-          return
-        end
-        template_put(name, template)
+      if template_exists?(name) && !force
+        @logger.debug("Found existing Elasticsearch template. Skipping template management", :name => name)
+        return
       end
+      template_put(name, template)
     end
 
     def bulk(actions)
-      @request_mutex.synchronize { non_threadsafe_bulk(actions) }
-    end
-
-    def non_threadsafe_bulk(actions)
+      @action_count ||= 0
+      @action_count += actions.size
+      
       return if actions.empty?
       bulk_body = actions.collect do |action, args, source|
         args, source = update_action_builder(args, source) if action == 'update'
 
         if source && action != 'delete'
-          next [ { action => args.merge({ :data => source }) } ]
+          next [ { action => args }, source ]
         else
           next { action => args }
         end
-      end.flatten
-
-      @client.bulk(:body => bulk_body)
-    end
-
-    def start_sniffing!
-      if options[:sniffing]
-        @sniffer_thread = Thread.new do
-          loop do
-            @request_mutex.synchronize { sniff! }
-            sleep (options[:sniffing_delay].to_f || 30)
-          end
-        end
+      end.
+      flatten.
+      reduce("") do |acc,line|
+        acc << LogStash::Json.dump(line)
+        acc << "\n"
       end
-    end
 
-    def stop_sniffing!
-      @sniffer_thread.kill() if @sniffer_thread
+      # Discard the URL
+      url, response = @pool.post("_bulk", nil, bulk_body)
+      LogStash::Json.load(response.body)
     end
 
-    def sniff!
-      client.transport.reload_connections! if options[:sniffing]
-      hosts_by_name = client.transport.hosts.map {|h| h["name"]}.sort
-      @logger.debug({"count" => hosts_by_name.count, "hosts" => hosts_by_name})
-    rescue StandardError => e
-      @logger.error("Error while sniffing connection",
-                    :message => e.message,
-                    :class => e.class.name,
-                    :backtrace => e.backtrace)
+    def close
+      @pool.close
     end
 
     private
 
-    # Builds a client and returns an Elasticsearch::Client
     #
     # The `options` is a hash where the following symbol keys have meaning:
     #
@@ -100,7 +79,7 @@ module LogStash; module Outputs; class ElasticSearch;
     # * `:path` - String. The leading path for prefixing Elasticsearch
     #   requests. This is sometimes used if you are proxying Elasticsearch access
     #   through a special http path, such as using mod_rewrite.
-    def build_client(options)
+    def build_pool(options)
       hosts = options[:hosts] || ["127.0.0.1"]
       client_settings = options[:client_settings] || {}
       timeout = options[:timeout] || 0
@@ -108,25 +87,51 @@ module LogStash; module Outputs; class ElasticSearch;
       host_ssl_opt = client_settings[:ssl].nil? ? nil : client_settings[:ssl][:enabled]
       urls = hosts.map {|host| host_to_url(host, host_ssl_opt, client_settings[:path])}
 
-      @client_options = {
-        :hosts => urls,
-        :ssl => client_settings[:ssl],
-        :transport_options => {
-          :socket_timeout => timeout,
-          :request_timeout => timeout,
-          :proxy => client_settings[:proxy]
-        },
-        :transport_class => ::Elasticsearch::Transport::Transport::HTTP::Manticore
+      adapter_options = {
+        :socket_timeout => timeout,
+        :request_timeout => timeout,
+        :proxy => client_settings[:proxy]
       }
 
-      if options[:user] && options[:password] then
-        token = Base64.strict_encode64(options[:user] + ":" + options[:password])
-        @client_options[:headers] = { "Authorization" => "Basic #{token}" }
+      # Having this explicitly set to nil is an error
+      if client_settings[:pool_max]
+        adapter_options[:pool_max] = client_settings[:pool_max]
+      end
+
+      # Having this explicitly set to nil is an error
+      if client_settings[:pool_max_per_route]
+        adapter_options[:pool_max_per_route] = client_settings[:pool_max_per_route]
       end
 
-      @logger.debug? && @logger.debug("Elasticsearch HTTP client options", client_options)
+      adapter_options[:ssl] = client_settings[:ssl] if client_settings[:ssl]
+
+      if options[:user]
+        adapter_options[:auth] = {
+          :user => options[:user],
+          :password => options[:password],
+          :eager => true
+        }
+      end
+
+      adapter_class = ::LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter
+      adapter = adapter_class.new(@logger, adapter_options)
+
+      pool_options = {
+        :sniffing => options[:sniffing],
+        :sniffer_delay => options[:sniffer_delay],
+        :healthcheck_path => options[:healthcheck_path],
+        :resurrect_delay => options[:resurrect_delay]
+      }
+
+      ssl_options = options[:client_settings] ? options[:client_settings][:ssl] : {}
+      pool_options[:scheme] = ssl_options && ssl_options[:enabled] ? 'https' : 'http'
+
+      if options[:user]
+        pool_options[:auth] = {:user => options[:user], :password => options[:password]}
+      end
 
-      Elasticsearch::Client.new(client_options)
+      pool_class = ::LogStash::Outputs::ElasticSearch::HttpClient::Pool
+      pool_class.new(@logger, adapter, urls, pool_options)
     end
 
     HOSTNAME_PORT_REGEX=/\A(?<hostname>([A-Za-z0-9\.\-]+)|\[[0-9A-Fa-f\:]+\])(:(?<port>\d+))?\Z/
@@ -183,18 +188,19 @@ module LogStash; module Outputs; class ElasticSearch;
           "this may be logged to disk thus leaking credentials. Use the 'user' and 'password' options respectively"
       end
 
-      url.to_s
+      url
     end
 
     def template_exists?(name)
-      @client.indices.get_template(:name => name)
-      return true
-    rescue Elasticsearch::Transport::Transport::Errors::NotFound
-      return false
+      url, response = @pool.head("/_template/#{name}")
+      response.code >= 200 && response.code <= 299
     end
 
     def template_put(name, template)
-      @client.indices.put_template(:name => name, :body => template)
+      path = "_template/#{name}"
+      logger.info("Installing elasticsearch template to #{path}")
+      url, response = @pool.put(path, nil, LogStash::Json.dump(template))
+      response
     end
 
     # Build a bulk item for an elasticsearch update action

data/lib/logstash/outputs/elasticsearch/http_client_builder.rb

@@ -1,14 +1,22 @@
 module LogStash; module Outputs; class ElasticSearch;
   module HttpClientBuilder
     def self.build(logger, hosts, params)
-      client_settings = {}
+      client_settings = {
+        :pool_max => params["pool_max"],
+        :pool_max_per_route => params["pool_max_per_route"],
+      }
 
       common_options = {
         :client_settings => client_settings,
-        :sniffing => params["sniffing"],
-        :sniffing_delay => params["sniffing_delay"]
+        :resurrect_delay => params["resurrect_delay"],
+        :healthcheck_path => params["healthcheck_path"]
       }
 
+      if params["sniffing"]
+        common_options[:sniffing] = true
+        common_options[:sniffer_delay] = params["sniffing_delay"]
+      end
+
       common_options[:timeout] = params["timeout"] if params["timeout"]
 
       if params["path"]
@@ -22,7 +30,7 @@ module LogStash; module Outputs; class ElasticSearch;
       common_options.merge! setup_basic_auth(logger, params)
 
       # Update API setup
-      raise( Logstash::ConfigurationError,
+      raise( LogStash::ConfigurationError,
         "doc_as_upsert and scripted_upsert are mutually exclusive."
       ) if params["doc_as_upsert"] and params["scripted_upsert"]
 
@@ -63,7 +71,10 @@ module LogStash; module Outputs; class ElasticSearch;
     end
 
     def self.setup_ssl(logger, params)
+      # If we have HTTPS hosts we act like SSL is enabled
+      params["ssl"] = true if params["hosts"].any? {|h| h.start_with?("https://")}
       return {} if params["ssl"].nil?
+
       return {:ssl => {:enabled => false}} if params["ssl"] == false
 
       cacert, truststore, truststore_password, keystore, keystore_password =

data/lib/logstash/outputs/elasticsearch/template_manager.rb

@@ -32,4 +32,4 @@ module LogStash; module Outputs; class ElasticSearch
       LogStash::Json.load(template_data)
     end
   end
-end end end
+end end end

data/lib/logstash/outputs/elasticsearch.rb

@@ -41,11 +41,13 @@ require "uri" # for escaping user input
 # a global setting for the JVM.
 #
 # As an example, to set your DNS TTL to 1 second you would set
-# the `LS_JAVA_OPTS` environment variable to `-Dnetwordaddress.cache.ttl=1`.
+# the `LS_JAVA_OPTS` environment variable to `-Dnetworkaddress.cache.ttl=1`.
 #
 # Keep in mind that a connection with keepalive enabled will
 # not reevaluate its DNS value while the keepalive is in effect.
 class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
+  declare_threadsafe!
+
   require "logstash/outputs/elasticsearch/http_client"
   require "logstash/outputs/elasticsearch/http_client_builder"
   require "logstash/outputs/elasticsearch/common_configs"
@@ -126,18 +128,39 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # Note, this is NOT a SOCKS proxy, but a plain HTTP proxy
   config :proxy
 
-  # Set the timeout for network operations and requests sent Elasticsearch. If
+  # Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If
   # a timeout occurs, the request will be retried.
   config :timeout, :validate => :number
 
+  # While the output tries to reuse connections efficiently we have a maximum.
+  # This sets the maximum number of open connections the output will create.
+  # Setting this too low may mean frequently closing / opening connections
+  # which is bad.
+  config :pool_max, :validate => :number, :default => 1000
+
+  # While the output tries to reuse connections efficiently we have a maximum per endpoint.
+  # This sets the maximum number of open connections per endpoint the output will create.
+  # Setting this too low may mean frequently closing / opening connections
+  # which is bad.
+  config :pool_max_per_route, :validate => :number, :default => 100
+
+  # When a backend is marked down a HEAD request will be sent to this path in the
+  # background to see if it has come back again before it is once again eligible
+  # to service requests. If you have custom firewall rules you may need to change
+  # this
+  config :healthcheck_path, :validate => :string, :default => "/"
+
+  # How frequently, in seconds, to wait between resurrection attempts.
+  # Resurrection is the process by which backend endpoints marked 'down' are checked
+  # to see if they have come back to life
+  config :resurrect_delay, :validate => :number, :default => 5
+
   def build_client
     @client = ::LogStash::Outputs::ElasticSearch::HttpClientBuilder.build(@logger, @hosts, params)
   end
 
   def close
     @stopping.make_true
-    @client.stop_sniffing!
-    @buffer.stop
   end
 
   @@plugins = Gem::Specification.find_all{|spec| spec.name =~ /logstash-output-elasticsearch-/ }

data/logstash-output-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '3.0.2'
+  s.version         = '4.1.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Logstash Output to Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -19,13 +19,9 @@ Gem::Specification.new do |s|
   # Special flag to let us know this is actually a logstash plugin
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
 
-  # Gem dependencies
-  s.add_runtime_dependency 'concurrent-ruby'
-  s.add_runtime_dependency 'elasticsearch', ['>= 1.0.13', '~> 1.0']
   s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
   s.add_runtime_dependency 'cabin', ['~> 0.6']
   s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
-
   s.add_development_dependency 'ftw', '~> 0.0.42'
   s.add_development_dependency 'logstash-codec-plain'
 
@@ -37,4 +33,6 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'longshoreman'
   s.add_development_dependency 'flores'
+  # Still used in some specs, we should remove this ASAP
+  s.add_development_dependency 'elasticsearch'
 end

data/spec/es_spec_helper.rb

@@ -5,6 +5,7 @@ require "logstash/json"
 require "stud/try"
 require "longshoreman"
 require "logstash/outputs/elasticsearch"
+require 'elasticsearch'
 
 CONTAINER_NAME = "logstash-output-elasticsearch-#{rand(999).to_s}"
 CONTAINER_IMAGE = "elasticsearch"

data/spec/fixtures/5x_node_resp.json

@@ -0,0 +1,2 @@
+
+{"cluster_name":"elasticsearch","nodes":{"vW8V2o_KSnOa-i97FACWFw":{"name":"Blockbuster","transport_address":"127.0.0.1:9300","host":"127.0.0.1","ip":"127.0.0.1","version":"5.0.0-alpha2","build_hash":"e3126df","http_address":"127.0.0.1:9200","roles":["master","data","ingest"],"settings":{"client":{"type":"node"},"cluster":{"name":"elasticsearch"},"node":{"name":"Blockbuster"},"path":{"logs":"/Users/andrewvc/Downloads/elasticsearch-5.0.0-alpha2/logs","home":"/Users/andrewvc/Downloads/elasticsearch-5.0.0-alpha2"},"config":{"ignore_system_properties":"true"}},"os":{"refresh_interval_in_millis":1000,"name":"Mac OS X","arch":"x86_64","version":"10.11.4","available_processors":8,"allocated_processors":8},"process":{"refresh_interval_in_millis":1000,"id":19048,"mlockall":false},"jvm":{"pid":19048,"version":"1.8.0_51","vm_name":"Java HotSpot(TM) 64-Bit Server VM","vm_version":"25.51-b03","vm_vendor":"Oracle Corporation","start_time_in_millis":1463781724873,"mem":{"heap_init_in_bytes":268435456,"heap_max_in_bytes":1037959168,"non_heap_init_in_bytes":2555904,"non_heap_max_in_bytes":0,"direct_max_in_bytes":1037959168},"gc_collectors":["ParNew","ConcurrentMarkSweep"],"memory_pools":["Code Cache","Metaspace","Compressed Class Space","Par Eden Space","Par Survivor Space","CMS Old Gen"],"using_compressed_ordinary_object_pointers":"true"},"thread_pool":{"force_merge":{"type":"fixed","min":1,"max":1,"queue_size":-1},"fetch_shard_started":{"type":"scaling","min":1,"max":16,"keep_alive":"5m","queue_size":-1},"listener":{"type":"fixed","min":4,"max":4,"queue_size":-1},"index":{"type":"fixed","min":8,"max":8,"queue_size":200},"refresh":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1},"generic":{"type":"cached","keep_alive":"30s","queue_size":-1},"warmer":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1},"search":{"type":"fixed","min":13,"max":13,"queue_size":1000},"flush":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1},"fetch_shard_store":{"type":"scaling","min":1,"max":16,"keep_alive":"5m","queue_size":-1},"management":{"type":"scaling","min":1,"max":5,"keep_alive":"5m","queue_size":-1},"get":{"type":"fixed","min":8,"max":8,"queue_size":1000},"bulk":{"type":"fixed","min":8,"max":8,"queue_size":50},"snapshot":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1}},"transport":{"bound_address":["[fe80::1]:9300","[::1]:9300","127.0.0.1:9300"],"publish_address":"127.0.0.1:9300","profiles":{}},"http":{"bound_address":["[fe80::1]:9200","[::1]:9200","127.0.0.1:9200"],"publish_address":"127.0.0.1:9200","max_content_length_in_bytes":104857600},"plugins":[],"modules":[{"name":"ingest-grok","version":"5.0.0-alpha2","description":"Ingest processor that uses grok patterns to split text","classname":"org.elasticsearch.ingest.grok.IngestGrokPlugin"},{"name":"lang-expression","version":"5.0.0-alpha2","description":"Lucene expressions integration for Elasticsearch","classname":"org.elasticsearch.script.expression.ExpressionPlugin"},{"name":"lang-groovy","version":"5.0.0-alpha2","description":"Groovy scripting integration for Elasticsearch","classname":"org.elasticsearch.script.groovy.GroovyPlugin"},{"name":"lang-mustache","version":"5.0.0-alpha2","description":"Mustache scripting integration for Elasticsearch","classname":"org.elasticsearch.script.mustache.MustachePlugin"},{"name":"lang-painless","version":"5.0.0-alpha2","description":"An easy, safe and fast scripting language for Elasticsearch","classname":"org.elasticsearch.painless.PainlessPlugin"},{"name":"reindex","version":"5.0.0-alpha2","description":"The Reindex module adds APIs to reindex from one index to another or update documents in place.","classname":"org.elasticsearch.index.reindex.ReindexPlugin"}],"ingest":{"processors":[{"type":"append"},{"type":"convert"},{"type":"date"},{"type":"fail"},{"type":"foreach"},{"type":"grok"},{"type":"gsub"},{"type":"join"},{"type":"lowercase"},{"type":"remove"},{"type":"rename"},{"type":"set"},{"type":"split"},{"type":"trim"},{"type":"uppercase"}]}},"BIAIepXSTYufETY06CUpYw":{"name":"Spyne","transport_address":"127.0.0.1:9301","host":"127.0.0.1","ip":"127.0.0.1","version":"5.0.0-alpha2","build_hash":"e3126df","http_address":"127.0.0.1:9201","roles":["master","data","ingest"],"settings":{"client":{"type":"node"},"cluster":{"name":"elasticsearch"},"node":{"name":"Spyne"},"path":{"logs":"/Users/andrewvc/Downloads/elasticsearch-5.0.0-alpha2/logs","home":"/Users/andrewvc/Downloads/elasticsearch-5.0.0-alpha2"},"config":{"ignore_system_properties":"true"}},"os":{"refresh_interval_in_millis":1000,"name":"Mac OS X","arch":"x86_64","version":"10.11.4","available_processors":8,"allocated_processors":8},"process":{"refresh_interval_in_millis":1000,"id":19029,"mlockall":false},"jvm":{"pid":19029,"version":"1.8.0_51","vm_name":"Java HotSpot(TM) 64-Bit Server VM","vm_version":"25.51-b03","vm_vendor":"Oracle Corporation","start_time_in_millis":1463781692985,"mem":{"heap_init_in_bytes":268435456,"heap_max_in_bytes":1037959168,"non_heap_init_in_bytes":2555904,"non_heap_max_in_bytes":0,"direct_max_in_bytes":1037959168},"gc_collectors":["ParNew","ConcurrentMarkSweep"],"memory_pools":["Code Cache","Metaspace","Compressed Class Space","Par Eden Space","Par Survivor Space","CMS Old Gen"],"using_compressed_ordinary_object_pointers":"true"},"thread_pool":{"force_merge":{"type":"fixed","min":1,"max":1,"queue_size":-1},"fetch_shard_started":{"type":"scaling","min":1,"max":16,"keep_alive":"5m","queue_size":-1},"listener":{"type":"fixed","min":4,"max":4,"queue_size":-1},"index":{"type":"fixed","min":8,"max":8,"queue_size":200},"refresh":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1},"generic":{"type":"cached","keep_alive":"30s","queue_size":-1},"warmer":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1},"search":{"type":"fixed","min":13,"max":13,"queue_size":1000},"flush":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1},"fetch_shard_store":{"type":"scaling","min":1,"max":16,"keep_alive":"5m","queue_size":-1},"management":{"type":"scaling","min":1,"max":5,"keep_alive":"5m","queue_size":-1},"get":{"type":"fixed","min":8,"max":8,"queue_size":1000},"bulk":{"type":"fixed","min":8,"max":8,"queue_size":50},"snapshot":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1}},"transport":{"bound_address":["[fe80::1]:9301","[::1]:9301","127.0.0.1:9301"],"publish_address":"127.0.0.1:9301","profiles":{}},"http":{"bound_address":["[fe80::1]:9201","[::1]:9201","127.0.0.1:9201"],"publish_address":"127.0.0.1:9201","max_content_length_in_bytes":104857600},"plugins":[],"modules":[{"name":"ingest-grok","version":"5.0.0-alpha2","description":"Ingest processor that uses grok patterns to split text","classname":"org.elasticsearch.ingest.grok.IngestGrokPlugin"},{"name":"lang-expression","version":"5.0.0-alpha2","description":"Lucene expressions integration for Elasticsearch","classname":"org.elasticsearch.script.expression.ExpressionPlugin"},{"name":"lang-groovy","version":"5.0.0-alpha2","description":"Groovy scripting integration for Elasticsearch","classname":"org.elasticsearch.script.groovy.GroovyPlugin"},{"name":"lang-mustache","version":"5.0.0-alpha2","description":"Mustache scripting integration for Elasticsearch","classname":"org.elasticsearch.script.mustache.MustachePlugin"},{"name":"lang-painless","version":"5.0.0-alpha2","description":"An easy, safe and fast scripting language for Elasticsearch","classname":"org.elasticsearch.painless.PainlessPlugin"},{"name":"reindex","version":"5.0.0-alpha2","description":"The Reindex module adds APIs to reindex from one index to another or update documents in place.","classname":"org.elasticsearch.index.reindex.ReindexPlugin"}],"ingest":{"processors":[{"type":"append"},{"type":"convert"},{"type":"date"},{"type":"fail"},{"type":"foreach"},{"type":"grok"},{"type":"gsub"},{"type":"join"},{"type":"lowercase"},{"type":"remove"},{"type":"rename"},{"type":"set"},{"type":"split"},{"type":"trim"},{"type":"uppercase"}]}}}}

data/spec/integration/outputs/create_spec.rb

@@ -2,7 +2,6 @@ require_relative "../../../spec/es_spec_helper"
 
 describe "client create actions", :integration => true do
   require "logstash/outputs/elasticsearch"
-  require "elasticsearch"
 
   def get_es_output(action, id = nil)
     settings = {
@@ -29,8 +28,7 @@ describe "client create actions", :integration => true do
     it "should create new documents with or without id" do
       subject = get_es_output("create", "id123")
       subject.register
-      subject.receive(LogStash::Event.new("message" => "sample message here"))
-      subject.flush
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
       @es.indices.refresh
       # Wait or fail until everything's indexed.
       Stud::try(3.times) do
@@ -42,8 +40,7 @@ describe "client create actions", :integration => true do
     it "should create new documents without id" do
       subject = get_es_output("create")
       subject.register
-      subject.receive(LogStash::Event.new("message" => "sample message here"))
-      subject.flush
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
       @es.indices.refresh
       # Wait or fail until everything's indexed.
       Stud::try(3.times) do

data/spec/integration/outputs/index_spec.rb

@@ -11,7 +11,7 @@ shared_examples "an indexer" do
     before do
       subject.register
       event_count.times do
-        subject.receive(LogStash::Event.new("message" => "Hello World!", "type" => type))
+        subject.multi_receive([LogStash::Event.new("message" => "Hello World!", "type" => type)])
       end
     end
 

data/spec/integration/outputs/parent_spec.rb

@@ -19,9 +19,7 @@ shared_examples "a parent indexer" do
       ftw.put!("#{index_url}/#{type}_parent/test", :body => pdoc.to_json)
       
       subject.register
-      event_count.times do
-        subject.receive(LogStash::Event.new("link_to" => "test", "message" => "Hello World!", "type" => type))
-      end
+      subject.multi_receive(event_count.times.map { LogStash::Event.new("link_to" => "test", "message" => "Hello World!", "type" => type) })
     end
 
 

data/spec/integration/outputs/pipeline_spec.rb

@@ -49,8 +49,7 @@ describe "Ingest pipeline execution behavior", :integration => true, :version_5x
     #@es.ingest.put_pipeline :id => 'apache_pipeline', :body => pipeline_defintion
 
     subject.register
-    subject.receive(LogStash::Event.new("message" => '183.60.215.50 - - [01/Jun/2015:18:00:00 +0000] "GET /scripts/netcat-webserver HTTP/1.1" 200 182 "-" "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)"'))
-    subject.flush
+    subject.multi_receive([LogStash::Event.new("message" => '183.60.215.50 - - [01/Jun/2015:18:00:00 +0000] "GET /scripts/netcat-webserver HTTP/1.1" 200 182 "-" "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)"')])
     @es.indices.refresh
 
     #Wait or fail until everything's indexed.

data/spec/integration/outputs/retry_spec.rb

@@ -10,6 +10,8 @@ describe "failures in bulk class expected behavior", :integration => true do
   let(:invalid_event) { LogStash::Event.new("geoip" => { "location" => "notlatlon" }, "@timestamp" => "2014-11-17T20:37:17.223Z") }
 
   def mock_actions_with_response(*resp)
+    raise ArgumentError, "Cannot mock actions until subject is registered and has a client!" unless subject.client
+
     expanded_responses = resp.map do |resp|
       items = resp["statuses"] && resp["statuses"].map do |status|
         {"create" => {"status" => status, "error" => "Error for #{status}"}}
@@ -21,7 +23,7 @@ describe "failures in bulk class expected behavior", :integration => true do
       }
     end
 
-    allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient).to receive(:bulk).and_return(*expanded_responses)
+    allow(subject.client).to receive(:bulk).and_return(*expanded_responses)
   end
 
   subject! do
@@ -30,8 +32,8 @@ describe "failures in bulk class expected behavior", :integration => true do
       "index" => "logstash-2014.11.17",
       "template_overwrite" => true,
       "hosts" => get_host_port(),
-      "retry_max_items" => 10,
-      "retry_max_interval" => 1,
+      "retry_max_interval" => 64,
+      "retry_initial_interval" => 2
     }
     next LogStash::Outputs::ElasticSearch.new(settings)
   end
@@ -39,6 +41,7 @@ describe "failures in bulk class expected behavior", :integration => true do
   before :each do
     # Delete all templates first.
     require "elasticsearch"
+    allow(Stud).to receive(:stoppable_sleep)
 
     # Clean ES of data before we start.
     @es = get_client
@@ -51,14 +54,11 @@ describe "failures in bulk class expected behavior", :integration => true do
     subject.close
   end
 
-  it "should return no errors if all bulk actions are successful" do
-    mock_actions_with_response({"errors" => false})
+  it "should retry exactly once if all bulk actions are successful" do
     expect(subject).to receive(:submit).with([action1, action2]).once.and_call_original
     subject.register
-    subject.receive(event1)
-    subject.receive(event2)
-    subject.flush
-    sleep(2)
+    mock_actions_with_response({"errors" => false})
+    subject.multi_receive([event1, event2])
   end
 
   it "retry exceptions within the submit body" do
@@ -73,25 +73,19 @@ describe "failures in bulk class expected behavior", :integration => true do
       end
     end
 
-    subject.receive(event1)
-    subject.flush
+    subject.multi_receive([event1])
   end
 
-  it "should retry actions with response status of 503" do
-    mock_actions_with_response({"errors" => true, "statuses" => [200, 200, 503, 503]},
-                               {"errors" => true, "statuses" => [200, 503]},
-                               {"errors" => false})
-    expect(subject).to receive(:submit).with([action1, action1, action1, action2]).ordered.once.and_call_original
+  it "should retry actions with response status of 503" do    expect(subject).to receive(:submit).with([action1, action1, action1, action2]).ordered.once.and_call_original
     expect(subject).to receive(:submit).with([action1, action2]).ordered.once.and_call_original
     expect(subject).to receive(:submit).with([action2]).ordered.once.and_call_original
 
     subject.register
-    subject.receive(event1)
-    subject.receive(event1)
-    subject.receive(event1)
-    subject.receive(event2)
-    subject.flush
-    sleep(3)
+    mock_actions_with_response({"errors" => true, "statuses" => [200, 200, 503, 503]},
+                               {"errors" => true, "statuses" => [200, 503]},
+                               {"errors" => false})
+
+    subject.multi_receive([event1, event1, event1, event2])
   end
 
   it "should retry actions with response status of 429" do
@@ -101,63 +95,71 @@ describe "failures in bulk class expected behavior", :integration => true do
                                {"errors" => false})
     expect(subject).to receive(:submit).with([action1]).twice.and_call_original
 
-    subject.receive(event1)
-    subject.flush
-    sleep(3)
+    subject.multi_receive([event1])
   end
 
   it "should retry an event infinitely until a non retryable status occurs" do
+    expect(subject).to receive(:submit).with([action1]).exactly(6).times.and_call_original
+    subject.register
+
     mock_actions_with_response({"errors" => true, "statuses" => [429]},
                                {"errors" => true, "statuses" => [429]},
                                {"errors" => true, "statuses" => [429]},
                                {"errors" => true, "statuses" => [429]},
                                {"errors" => true, "statuses" => [429]},
                                {"errors" => true, "statuses" => [500]})
-    expect(subject).to receive(:submit).with([action1]).exactly(6).times.and_call_original
+
+    subject.multi_receive([event1])
+  end
+
+  it "should sleep for an exponentially increasing amount of time on each retry, capped by the max" do
+    [2, 4, 8, 16, 32, 64, 64].each_with_index do |interval,i|
+      expect(Stud).to receive(:stoppable_sleep).with(interval).ordered
+    end
+
     subject.register
-    subject.receive(event1)
-    subject.flush
-    sleep(5)
+
+    mock_actions_with_response({"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [500]})
+
+    subject.multi_receive([event1])
   end
 
   it "non-retryable errors like mapping errors (400) should be dropped and not be retried (unfortunately)" do
     subject.register
-    subject.receive(invalid_event)
     expect(subject).to receive(:submit).once.and_call_original
+    subject.multi_receive([invalid_event])
     subject.close
 
     @es.indices.refresh
-    sleep(5)
-    Stud::try(10.times) do
-      r = @es.search
-      insist { r["hits"]["total"] } == 0
-    end
+    r = @es.search
+    expect(r["hits"]["total"]).to eql(0)
   end
 
   it "successful requests should not be appended to retry queue" do
-    subject.register
-    subject.receive(event1)
     expect(subject).to receive(:submit).once.and_call_original
+
+    subject.register
+    subject.multi_receive([event1])
     subject.close
     @es.indices.refresh
-    sleep(5)
-    Stud::try(10.times) do
-      r = @es.search
-      insist { r["hits"]["total"] } == 1
-    end
+    r = @es.search
+    expect(r["hits"]["total"]).to eql(1)
   end
 
   it "should only index proper events" do
     subject.register
-    subject.receive(invalid_event)
-    subject.receive(event1)
+    subject.multi_receive([invalid_event, event1])
     subject.close
 
     @es.indices.refresh
-    sleep(5)
-    Stud::try(10.times) do
-      r = @es.search
-      insist { r["hits"]["total"] } == 1
-    end
+    r = @es.search
+    expect(r["hits"]["total"]).to eql(1)
   end
 end