logstash-output-elasticsearch 3.0.2-java → 4.1.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +16 -3
  3. data/Gemfile +1 -1
  4. data/lib/logstash/outputs/elasticsearch/common.rb +90 -58
  5. data/lib/logstash/outputs/elasticsearch/common_configs.rb +12 -32
  6. data/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb +63 -0
  7. data/lib/logstash/outputs/elasticsearch/http_client/pool.rb +378 -0
  8. data/lib/logstash/outputs/elasticsearch/http_client.rb +70 -64
  9. data/lib/logstash/outputs/elasticsearch/http_client_builder.rb +15 -4
  10. data/lib/logstash/outputs/elasticsearch/template_manager.rb +1 -1
  11. data/lib/logstash/outputs/elasticsearch.rb +27 -4
  12. data/logstash-output-elasticsearch.gemspec +3 -5
  13. data/spec/es_spec_helper.rb +1 -0
  14. data/spec/fixtures/5x_node_resp.json +2 -0
  15. data/spec/integration/outputs/create_spec.rb +2 -5
  16. data/spec/integration/outputs/index_spec.rb +1 -1
  17. data/spec/integration/outputs/parent_spec.rb +1 -3
  18. data/spec/integration/outputs/pipeline_spec.rb +1 -2
  19. data/spec/integration/outputs/retry_spec.rb +51 -49
  20. data/spec/integration/outputs/routing_spec.rb +1 -1
  21. data/spec/integration/outputs/secure_spec.rb +4 -8
  22. data/spec/integration/outputs/templates_spec.rb +12 -8
  23. data/spec/integration/outputs/update_spec.rb +13 -27
  24. data/spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb +25 -0
  25. data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb +142 -0
  26. data/spec/unit/outputs/elasticsearch/http_client_spec.rb +8 -22
  27. data/spec/unit/outputs/elasticsearch_proxy_spec.rb +5 -6
  28. data/spec/unit/outputs/elasticsearch_spec.rb +33 -30
  29. data/spec/unit/outputs/elasticsearch_ssl_spec.rb +10 -6
  30. metadata +72 -87
  31. data/lib/logstash/outputs/elasticsearch/buffer.rb +0 -124
  32. data/spec/unit/buffer_spec.rb +0 -118
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 7b8baebc24fbd14637d41ef87a5a0eeff7daf94f
4
- data.tar.gz: e1ebb6428b9f00175f1cd816dc19d0c0ed4c36d9
3
+ metadata.gz: 1463814b1c058872439a7cff351407b2c2a6d442
4
+ data.tar.gz: 28b8279a5cf3bb64e2003bc7e5272989918a2eb7
5
5
  SHA512:
6
- metadata.gz: ccb0eec4af4e4dcadcf39f3db0ee986799887f202627722c3eb88677d5a1c7996e4338ce60ad123ebbeca44d90cc235d23a95885eda47e6ac0b3c5115853d053
7
- data.tar.gz: 534769a76b11249b73a91622928271aa9a7c8f1ab2ebde6d50bc117e0d0046356655f4652825110c7a88ab57ae75e8c896ea17f51ecadc200f8ed57dddd8c8ed
6
+ metadata.gz: 7d7ac9a08e59b1698121a9aaf13b195a032961a3df092c7a58ed183611869efe95803928e5dce25fe8b8e9a0394a1eae7e15cb4b2fa4b7b44a0bee1362321bdb
7
+ data.tar.gz: a97ecd3ea4a6d391b7b7e77d38956d9c7c8089ca5406513301a2e46e5f1e12fb82edd4f0a0dce048a473e903965012246c8b99978fb207eae384fb35a4887238
data/CHANGELOG.md CHANGED
@@ -1,12 +1,25 @@
1
+ ## 4.1.0
2
+ - breaking,config: Removed obsolete config `host` and `port`. Please use the `hosts` config with the `[host:port]` syntax.
3
+ - breaking,config: Removed obsolete config `index_type`. Please use `document_type` instead.
4
+ - breaking,config: Set config `max_retries` and `retry_max_items` as obsolete
5
+
6
+ ## 4.0.0
7
+ - Make this plugin threadsafe. Workers no longer needed or supported
8
+ - Add pool_max and pool_max_per_route options
9
+
1
10
  ## 3.0.2
2
- - Fix issues where URI based paths in 'hosts' would not function correctly
11
+ - Fix issues where URI based paths in 'hosts' would not function correctly
12
+
3
13
  ## 3.0.1
4
- - Republish all the gems under jruby.
14
+ - Republish all the gems under jruby.
15
+
5
16
  ## 3.0.0
6
- - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
17
+ - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
18
+
7
19
  ## 2.7.0
8
20
  - Add `pipeline` configuration option for setting an ingest pipeline to run upon indexing
9
21
 
22
+
10
23
  ## 2.6.2
11
24
  - Fix bug where update index actions would not work with events with 'data' field
12
25
 
data/Gemfile CHANGED
@@ -1,4 +1,4 @@
1
1
  source 'https://rubygems.org'
2
2
 
3
- # Specify your gem's dependencies in logstash-mass_effect.gemspec
3
+
4
4
  gemspec
@@ -1,5 +1,4 @@
1
1
  require "logstash/outputs/elasticsearch/template_manager"
2
- require "logstash/outputs/elasticsearch/buffer"
3
2
 
4
3
  module LogStash; module Outputs; class ElasticSearch;
5
4
  module Common
@@ -13,16 +12,11 @@ module LogStash; module Outputs; class ElasticSearch;
13
12
  setup_hosts # properly sets @hosts
14
13
  build_client
15
14
  install_template
16
- setup_buffer_and_handler
17
15
  check_action_validity
18
16
 
19
17
  @logger.info("New Elasticsearch output", :class => self.class.name, :hosts => @hosts)
20
18
  end
21
19
 
22
- def receive(event)
23
- @buffer << event_action_tuple(event)
24
- end
25
-
26
20
  # Receive an array of events and immediately attempt to index them (no buffering)
27
21
  def multi_receive(events)
28
22
  events.each_slice(@flush_size) do |slice|
@@ -37,10 +31,6 @@ module LogStash; module Outputs; class ElasticSearch;
37
31
  [action, params, event]
38
32
  end
39
33
 
40
- def flush
41
- @buffer.flush
42
- end
43
-
44
34
  def setup_hosts
45
35
  @hosts = Array(@hosts)
46
36
  if @hosts.empty?
@@ -53,12 +43,6 @@ module LogStash; module Outputs; class ElasticSearch;
53
43
  TemplateManager.install_template(self)
54
44
  end
55
45
 
56
- def setup_buffer_and_handler
57
- @buffer = ::LogStash::Outputs::ElasticSearch::Buffer.new(@logger, @flush_size, @idle_flush_time) do |actions|
58
- retrying_submit(actions)
59
- end
60
- end
61
-
62
46
  def check_action_validity
63
47
  raise LogStash::ConfigurationError, "No action specified!" unless @action
64
48
 
@@ -75,33 +59,55 @@ module LogStash; module Outputs; class ElasticSearch;
75
59
  VALID_HTTP_ACTIONS
76
60
  end
77
61
 
78
- def retrying_submit(actions)
62
+ def retrying_submit(actions)
79
63
  # Initially we submit the full list of actions
80
64
  submit_actions = actions
81
65
 
66
+ sleep_interval = @retry_initial_interval
67
+
82
68
  while submit_actions && submit_actions.length > 0
83
- return if !submit_actions || submit_actions.empty? # If everything's a success we move along
69
+
84
70
  # We retry with whatever is didn't succeed
85
71
  begin
86
72
  submit_actions = submit(submit_actions)
73
+ if submit_actions && submit_actions.size > 0
74
+ @logger.error("Retrying individual actions")
75
+ submit_actions.each {|action| @logger.error("Action", action) }
76
+ end
87
77
  rescue => e
88
- @logger.warn("Encountered an unexpected error submitting a bulk request! Will retry.",
89
- :message => e.message,
78
+ @logger.error("Encountered an unexpected error submitting a bulk request! Will retry.",
79
+ :error_message => e.message,
90
80
  :class => e.class.name,
91
81
  :backtrace => e.backtrace)
92
82
  end
93
83
 
94
- sleep @retry_max_interval if submit_actions && submit_actions.length > 0
84
+ # Everything was a success!
85
+ break if !submit_actions || submit_actions.empty?
86
+
87
+ # If we're retrying the action sleep for the recommended interval
88
+ # Double the interval for the next time through to achieve exponential backoff
89
+ Stud.stoppable_sleep(sleep_interval) { @stopping.true? }
90
+ sleep_interval = next_sleep_interval(sleep_interval)
95
91
  end
96
92
  end
97
93
 
98
- def submit(actions)
99
- es_actions = actions.map { |a, doc, event| [a, doc, event.to_hash]}
94
+ def sleep_for_interval(sleep_interval)
95
+ Stud.stoppable_sleep(sleep_interval) { @stopping.true? }
96
+ next_sleep_interval(sleep_interval)
97
+ end
100
98
 
101
- bulk_response = safe_bulk(es_actions,actions)
99
+ def next_sleep_interval(current_interval)
100
+ doubled = current_interval * 2
101
+ doubled > @retry_max_interval ? @retry_max_interval : doubled
102
+ end
102
103
 
103
- # If there are no errors, we're done here!
104
- return unless bulk_response["errors"]
104
+ def submit(actions)
105
+ bulk_response = safe_bulk(actions)
106
+
107
+ # If the response is nil that means we were in a retry loop
108
+ # and aborted since we're shutting down
109
+ # If it did return and there are no errors we're good as well
110
+ return if bulk_response.nil? || !bulk_response["errors"]
105
111
 
106
112
  actions_to_retry = []
107
113
  bulk_response["items"].each_with_index do |response,idx|
@@ -168,38 +174,64 @@ module LogStash; module Outputs; class ElasticSearch;
168
174
  end
169
175
 
170
176
  # Rescue retryable errors during bulk submission
171
- def safe_bulk(es_actions,actions)
172
- @client.bulk(es_actions)
173
- rescue Manticore::SocketException, Manticore::SocketTimeout => e
174
- # If we can't even connect to the server let's just print out the URL (:hosts is actually a URL)
175
- # and let the user sort it out from there
176
- @logger.error(
177
- "Attempted to send a bulk request to Elasticsearch configured at '#{@client.client_options[:hosts]}',"+
178
- " but Elasticsearch appears to be unreachable or down!",
179
- :error_message => e.message,
180
- :class => e.class.name,
181
- :client_config => @client.client_options,
182
- )
183
- @logger.debug("Failed actions for last bad bulk request!", :actions => actions)
184
-
185
- # We retry until there are no errors! Errors should all go to the retry queue
186
- sleep @retry_max_interval
187
- retry unless @stopping.true?
188
- rescue => e
189
- # For all other errors print out full connection issues
190
- @logger.error(
191
- "Attempted to send a bulk request to Elasticsearch configured at '#{@client.client_options[:hosts]}'," +
192
- " but an error occurred and it failed! Are you sure you can reach elasticsearch from this machine using " +
193
- "the configuration provided?",
194
- :error_message => e.message,
195
- :error_class => e.class.name,
196
- :backtrace => e.backtrace,
197
- :client_config => @client.client_options,
198
- )
199
-
200
- @logger.debug("Failed actions for last bad bulk request!", :actions => actions)
201
-
202
- raise e
177
+ def safe_bulk(actions)
178
+ sleep_interval = @retry_initial_interval
179
+ begin
180
+ es_actions = actions.map {|action_type, params, event| [action_type, params, event.to_hash]}
181
+ response = @client.bulk(es_actions)
182
+ response
183
+ rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError => e
184
+ # If we can't even connect to the server let's just print out the URL (:hosts is actually a URL)
185
+ # and let the user sort it out from there
186
+ @logger.error(
187
+ "Attempted to send a bulk request to elasticsearch'"+
188
+ " but Elasticsearch appears to be unreachable or down!",
189
+ :error_message => e.message,
190
+ :class => e.class.name,
191
+ :will_retry_in_seconds => sleep_interval
192
+ )
193
+ @logger.debug("Failed actions for last bad bulk request!", :actions => actions)
194
+
195
+ # We retry until there are no errors! Errors should all go to the retry queue
196
+ sleep_interval = sleep_for_interval(sleep_interval)
197
+ retry unless @stopping.true?
198
+ rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::NoConnectionAvailableError => e
199
+ @logger.error(
200
+ "Attempted to send a bulk request to elasticsearch, but no there are no living connections in the connection pool. Perhaps Elasticsearch is unreachable or down?",
201
+ :error_message => e.message,
202
+ :class => e.class.name,
203
+ :will_retry_in_seconds => sleep_interval
204
+ )
205
+ Stud.stoppable_sleep(sleep_interval) { @stopping.true? }
206
+ sleep_interval = next_sleep_interval(sleep_interval)
207
+ retry unless @stopping.true?
208
+ rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
209
+ if RETRYABLE_CODES.include?(e.response_code)
210
+ log_hash = {:code => e.response_code, :url => e.url}
211
+ log_hash[:body] = e.body if @logger.debug? # Generally this is too verbose
212
+ @logger.error("Attempted to send a bulk request to elasticsearch but received a bad HTTP response code!", log_hash)
213
+
214
+ sleep_interval = sleep_for_interval(sleep_interval)
215
+ retry unless @stopping.true?
216
+ else
217
+ @logger.error("Got a bad response code from server, but this code is not considered retryable. Request will be dropped", :code => e.code)
218
+ end
219
+ rescue => e
220
+ # Stuff that should never happen
221
+ # For all other errors print out full connection issues
222
+ @logger.error(
223
+ "An unknown error occurred sending a bulk request to Elasticsearch. We will retry indefinitely",
224
+ :error_message => e.message,
225
+ :error_class => e.class.name,
226
+ :backtrace => e.backtrace
227
+ )
228
+
229
+ @logger.debug("Failed actions for last bad bulk request!", :actions => actions)
230
+
231
+ # We retry until there are no errors! Errors should all go to the retry queue
232
+ sleep_interval = sleep_for_interval(sleep_interval)
233
+ retry unless @stopping.true?
234
+ end
203
235
  end
204
236
  end
205
237
  end; end; end
@@ -6,16 +6,10 @@ module LogStash; module Outputs; class ElasticSearch
6
6
  # delete old data or only search specific date ranges.
7
7
  # Indexes may not contain uppercase characters.
8
8
  # For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}.
9
- # LS uses Joda to format the index pattern from event timestamp.
10
- # Joda formats are defined http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[here].
9
+ # LS uses Joda to format the index pattern from event timestamp.
10
+ # Joda formats are defined http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[here].
11
11
  mod.config :index, :validate => :string, :default => "logstash-%{+YYYY.MM.dd}"
12
12
 
13
- # The index type to write events to. Generally you should try to write only
14
- # similar events to the same 'type'. String expansion `%{foo}` works here.
15
- #
16
- # Deprecated in favor of `docoument_type` field.
17
- mod.config :index_type, :validate => :string, :obsolete => "Please use the 'document_type' setting instead. It has the same effect, but is more appropriately named."
18
-
19
13
  # The document type to write events to. Generally you should try to write only
20
14
  # similar events to the same 'type'. String expansion `%{foo}` works here.
21
15
  # Unless you set 'document_type', the event 'type' will be used if it exists
@@ -81,28 +75,12 @@ module LogStash; module Outputs; class ElasticSearch
81
75
  # to prevent LS from sending bulk requests to the master nodes. So this parameter should only reference either data or client nodes in Elasticsearch.
82
76
  mod.config :hosts, :validate => :array, :default => ["127.0.0.1"]
83
77
 
84
- mod.config :host, :obsolete => "Please use the 'hosts' setting instead. You can specify multiple entries separated by comma in 'host:port' format."
85
-
86
- # The port setting is obsolete. Please use the 'hosts' setting instead.
87
- # Hosts entries can be in "host:port" format.
88
- mod.config :port, :obsolete => "Please use the 'hosts' setting instead. Hosts entries can be in 'host:port' format."
89
-
90
78
  # This plugin uses the bulk index API for improved indexing performance.
91
- # In Logstashes >= 2.2 this setting defines the maximum sized bulk request Logstash will make
79
+ # This setting defines the maximum sized bulk request Logstash will make
92
80
  # You you may want to increase this to be in line with your pipeline's batch size.
93
81
  # If you specify a number larger than the batch size of your pipeline it will have no effect,
94
82
  # save for the case where a filter increases the size of an inflight batch by outputting
95
83
  # events.
96
- #
97
- # In Logstashes <= 2.1 this plugin uses its own internal buffer of events.
98
- # This config option sets that size. In these older logstashes this size may
99
- # have a significant impact on heap usage, whereas in 2.2+ it will never increase it.
100
- # To make efficient bulk API calls, we will buffer a certain number of
101
- # events before flushing that out to Elasticsearch. This setting
102
- # controls how many events will be buffered before sending a batch
103
- # of events. Increasing the `flush_size` has an effect on Logstash's heap size.
104
- # Remember to also increase the heap size using `LS_HEAP_SIZE` if you are sending big documents
105
- # or have increased the `flush_size` to a higher value.
106
84
  mod.config :flush_size, :validate => :number, :default => 500
107
85
 
108
86
  # The amount of time since last flush before a flush is forced.
@@ -124,8 +102,8 @@ module LogStash; module Outputs; class ElasticSearch
124
102
  # Create a new document with source if `document_id` doesn't exist in Elasticsearch
125
103
  mod.config :doc_as_upsert, :validate => :boolean, :default => false
126
104
 
127
- # DEPRECATED This setting no longer does anything. It will be marked obsolete in a future version.
128
- mod.config :max_retries, :validate => :number, :default => 3
105
+ #Obsolete since 4.1.0
106
+ mod.config :max_retries, :obsolete => "This setting no longer does anything. Please remove it from your config"
129
107
 
130
108
  # Set script name for scripted update mode
131
109
  mod.config :script, :validate => :string, :default => ""
@@ -145,12 +123,14 @@ module LogStash; module Outputs; class ElasticSearch
145
123
  # if enabled, script is in charge of creating non-existent document (scripted update)
146
124
  mod.config :scripted_upsert, :validate => :boolean, :default => false
147
125
 
148
- # Set max interval between bulk retries.
149
- mod.config :retry_max_interval, :validate => :number, :default => 2
126
+ # Set initial interval in seconds between bulk retries. Doubled on each retry up to `retry_max_interval`
127
+ mod.config :retry_initial_interval, :validate => :number, :default => 2
128
+
129
+ # Set max interval in seconds between bulk retries.
130
+ mod.config :retry_max_interval, :validate => :number, :default => 64
150
131
 
151
- # DEPRECATED This setting no longer does anything. If you need to change the number of retries in flight
152
- # try increasing the total number of workers to better handle this.
153
- mod.config :retry_max_items, :validate => :number, :default => 500, :deprecated => true
132
+ #Obsolete since 4.1.0
133
+ mod.config :retry_max_items, :obsolete => "This setting no longer does anything. Please remove it from your config"
154
134
 
155
135
  # The number of times Elasticsearch should internally retry an update/upserted document
156
136
  # See the https://www.elastic.co/guide/en/elasticsearch/guide/current/partial-updates.html[partial updates]
@@ -0,0 +1,63 @@
1
+ require 'manticore'
2
+
3
+ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
4
+ class ManticoreAdapter
5
+ attr_reader :manticore, :logger
6
+
7
+ def initialize(logger, options={})
8
+ @logger = logger
9
+ @options = options || {}
10
+ @options[:ssl] = @options[:ssl] || {}
11
+
12
+ # We manage our own retries directly, so let's disable them here
13
+ @options[:automatic_retries] = 0
14
+ # We definitely don't need cookies
15
+ @options[:cookies] = false
16
+
17
+ @request_options = @options[:headers] ? {:headers => @options[:headers]} : {}
18
+ @manticore = ::Manticore::Client.new(@options)
19
+ end
20
+
21
+ def client
22
+ @manticore
23
+ end
24
+
25
+ # Performs the request by invoking {Transport::Base#perform_request} with a block.
26
+ #
27
+ # @return [Response]
28
+ # @see Transport::Base#perform_request
29
+ #
30
+ def perform_request(url, method, path, params={}, body=nil)
31
+
32
+
33
+ params = (params || {}).merge @request_options
34
+ params[:body] = body if body
35
+ url_and_path = (url + path).to_s # Convert URI object to string
36
+
37
+
38
+ resp = @manticore.send(method.downcase, url_and_path, params)
39
+
40
+ # Manticore returns lazy responses by default
41
+ # We want to block for our usage, this will wait for the repsonse
42
+ # to finish
43
+ resp.call
44
+
45
+ # 404s are excluded because they are valid codes in the case of
46
+ # template installation. We might need a better story around this later
47
+ # but for our current purposes this is correct
48
+ if resp.code < 200 || resp.code > 299 && resp.code != 404
49
+ raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(resp.code, url_and_path, body)
50
+ end
51
+
52
+ resp
53
+ end
54
+
55
+ def close
56
+ @manticore.close
57
+ end
58
+
59
+ def host_unreachable_exceptions
60
+ [::Manticore::Timeout,::Manticore::SocketException, ::Manticore::ClientProtocolException, ::Manticore::ResolutionFailure, Manticore::SocketTimeout]
61
+ end
62
+ end
63
+ end; end; end; end
@@ -0,0 +1,378 @@
1
+ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
2
+ class Pool
3
+ class NoConnectionAvailableError < Error; end
4
+ class BadResponseCodeError < Error
5
+ attr_reader :url, :response_code, :body
6
+
7
+ def initialize(response_code, url, body)
8
+ @response_code = response_code
9
+ @url = url
10
+ @body = body
11
+ end
12
+
13
+ def message
14
+ "Got response code '#{response_code}' contact Elasticsrearch at URL '#{@url}'"
15
+ end
16
+ end
17
+ class HostUnreachableError < Error;
18
+ attr_reader :original_error, :url
19
+
20
+ def initialize(original_error, url)
21
+ @original_error = original_error
22
+ @url = url
23
+ end
24
+
25
+ def message
26
+ "Elasticsearch Unreachable: [#{@url}][#{original_error.class}] #{original_error.message}"
27
+ end
28
+ end
29
+
30
+ attr_reader :logger, :adapter, :sniffing, :sniffer_delay, :resurrect_delay, :auth, :healthcheck_path
31
+
32
+ DEFAULT_OPTIONS = {
33
+ :healthcheck_path => '/'.freeze,
34
+ :scheme => 'http',
35
+ :resurrect_delay => 5,
36
+ :auth => nil, # Can be set to {:user => 'user', :password => 'pass'}
37
+ :sniffing => false,
38
+ :sniffer_delay => 10,
39
+ }.freeze
40
+
41
+ def initialize(logger, adapter, initial_urls=[], options={})
42
+ @logger = logger
43
+ @adapter = adapter
44
+
45
+ DEFAULT_OPTIONS.merge(options).tap do |merged|
46
+ @healthcheck_path = merged[:healthcheck_path]
47
+ @scheme = merged[:scheme]
48
+ @resurrect_delay = merged[:resurrect_delay]
49
+ @auth = merged[:auth]
50
+ @sniffing = merged[:sniffing]
51
+ @sniffer_delay = merged[:sniffer_delay]
52
+ end
53
+
54
+ # Override the scheme if one is explicitly set in urls
55
+ if initial_urls.any? {|u| u.scheme == 'https'} && @scheme == 'http'
56
+ raise ArgumentError, "HTTP was set as scheme, but an HTTPS URL was passed in!"
57
+ end
58
+
59
+ # Used for all concurrent operations in this class
60
+ @state_mutex = Mutex.new
61
+
62
+ # Holds metadata about all URLs
63
+ @url_info = {}
64
+ @stopping = false
65
+
66
+ update_urls(initial_urls)
67
+ start_resurrectionist
68
+ start_sniffer if @sniffing
69
+ end
70
+
71
+ def close
72
+ @state_mutex.synchronize { @stopping = true }
73
+
74
+ logger.debug "Stopping sniffer"
75
+ stop_sniffer
76
+
77
+ logger.debug "Stopping resurrectionist"
78
+ stop_resurrectionist
79
+
80
+ logger.debug "Waiting for in use manticore connections"
81
+ wait_for_in_use_connections
82
+
83
+ logger.debug("Closing adapter #{@adapter}")
84
+ @adapter.close
85
+ end
86
+
87
+ def wait_for_in_use_connections
88
+ until in_use_connections.empty?
89
+ logger.info "Blocked on shutdown to in use connections #{@state_mutex.synchronize {@url_info}}"
90
+ sleep 1
91
+ end
92
+ end
93
+
94
+ def in_use_connections
95
+ @state_mutex.synchronize { @url_info.values.select {|v| v[:in_use] > 0 } }
96
+ end
97
+
98
+ def alive_urls_count
99
+ @state_mutex.synchronize { @url_info.values.select {|v| !v[:dead] }.count }
100
+ end
101
+
102
+ def url_info
103
+ @state_mutex.synchronize { @url_info }
104
+ end
105
+
106
+ def urls
107
+ url_info.keys
108
+ end
109
+
110
+ def until_stopped(task_name, delay)
111
+ last_done = Time.now
112
+ until @state_mutex.synchronize { @stopping }
113
+ begin
114
+ now = Time.now
115
+ if (now - last_done) >= delay
116
+ last_done = now
117
+ yield
118
+ end
119
+ sleep 1
120
+ rescue => e
121
+ logger.warn(
122
+ "Error while performing #{task_name}",
123
+ :error_message => e.message,
124
+ :class => e.class.name,
125
+ :backtrace => e.backtrace
126
+ )
127
+ end
128
+ end
129
+ end
130
+
131
+ def start_sniffer
132
+ @sniffer = Thread.new do
133
+ until_stopped("sniffing", sniffer_delay) do
134
+ begin
135
+ sniff!
136
+ rescue NoConnectionAvailableError => e
137
+ @state_mutex.synchronize { # Synchronize around @url_info
138
+ logger.warn("Elasticsearch output attempted to sniff for new connections but cannot. No living connections are detected. Pool contains the following current URLs", :url_info => @url_info) }
139
+ end
140
+ end
141
+ end
142
+ end
143
+
144
+ # Sniffs the cluster then updates the internal URLs
145
+ def sniff!
146
+ update_urls(check_sniff)
147
+ end
148
+
149
+ ES1_SNIFF_RE_URL = /\[([^\/]*)?\/?([^:]*):([0-9]+)\]/
150
+ ES2_SNIFF_RE_URL = /([^\/]*)?\/?([^:]*):([0-9]+)/
151
+ # Sniffs and returns the results. Does not update internal URLs!
152
+ def check_sniff
153
+ url, resp = perform_request(:get, '_nodes')
154
+ parsed = LogStash::Json.load(resp.body)
155
+ parsed['nodes'].map do |id,info|
156
+ # TODO Make sure this works with shield. Does that listed
157
+ # stuff as 'https_address?'
158
+ addr_str = info['http_address'].to_s
159
+ next unless addr_str # Skip hosts with HTTP disabled
160
+
161
+
162
+ # Only connect to nodes that serve data
163
+ # this will skip connecting to client, tribe, and master only nodes
164
+ # Note that if 'attributes' is NOT set, then that's just a regular node
165
+ # with master + data + client enabled, so we allow that
166
+ attributes = info['attributes']
167
+ next if attributes && attributes['data'] == 'false'
168
+
169
+ matches = addr_str.match(ES1_SNIFF_RE_URL) || addr_str.match(ES2_SNIFF_RE_URL)
170
+ if matches
171
+ host = matches[1].empty? ? matches[2] : matches[1]
172
+ port = matches[3]
173
+ URI.parse("#{@scheme}://#{host}:#{port}")
174
+ end
175
+ end.compact
176
+ end
177
+
178
+ def stop_sniffer
179
+ @sniffer.join if @sniffer
180
+ end
181
+
182
+ def sniffer_alive?
183
+ @sniffer ? @sniffer.alive? : nil
184
+ end
185
+
186
+ def start_resurrectionist
187
+ @resurrectionist = Thread.new do
188
+ until_stopped("resurrection", @resurrect_delay) do
189
+ resurrect_dead!
190
+ end
191
+ end
192
+ end
193
+
194
+ def resurrect_dead!
195
+ # Try to keep locking granularity low such that we don't affect IO...
196
+ @state_mutex.synchronize { @url_info.select {|url,meta| meta[:dead] } }.each do |url,meta|
197
+ begin
198
+ @logger.info("Checking url #{url} with path #{@healthcheck_path} to see if node resurrected")
199
+ perform_request_to_url(url, "HEAD", @healthcheck_path)
200
+ # If no exception was raised it must have succeeded!
201
+ logger.warn("Resurrected connection to dead ES instance at #{url}")
202
+ @state_mutex.synchronize { meta[:dead] = false }
203
+ rescue HostUnreachableError => e
204
+ logger.debug("Attempted to resurrect connection to dead ES instance at #{url}, got an error [#{e.class}] #{e.message}")
205
+ end
206
+ end
207
+ end
208
+
209
+ def stop_resurrectionist
210
+ @resurrectionist.join
211
+ end
212
+
213
+ def resurrectionist_alive?
214
+ @resurrectionist.alive?
215
+ end
216
+
217
+ def perform_request(method, path, params={}, body=nil)
218
+ with_connection do |url|
219
+ resp = perform_request_to_url(url, method, path, params, body)
220
+ [url, resp]
221
+ end
222
+ end
223
+
224
+ [:get, :put, :post, :delete, :patch, :head].each do |method|
225
+ define_method(method) do |path, params={}, body=nil|
226
+ perform_request(method, path, params, body)
227
+ end
228
+ end
229
+
230
+ def perform_request_to_url(url, method, path, params={}, body=nil)
231
+ res = @adapter.perform_request(url, method, path, params, body)
232
+ rescue *@adapter.host_unreachable_exceptions => e
233
+ raise HostUnreachableError.new(e, url), "Could not reach host #{e.class}: #{e.message}"
234
+ end
235
+
236
+ def normalize_url(uri)
237
+ raise ArgumentError, "Only URI objects may be passed in!" unless uri.is_a?(URI)
238
+ uri = uri.clone
239
+
240
+ # Set credentials if need be
241
+ if @auth && !uri.user
242
+ uri.user ||= @auth[:user]
243
+ uri.password ||= @auth[:password]
244
+ end
245
+
246
+ uri.scheme = @scheme
247
+
248
+ uri
249
+ end
250
+
251
+ def update_urls(new_urls)
252
+ # Normalize URLs
253
+ new_urls = new_urls.map(&method(:normalize_url))
254
+
255
+ # Used for logging nicely
256
+ state_changes = {:removed => [], :added => []}
257
+ @state_mutex.synchronize do
258
+ # Add new connections
259
+ new_urls.each do |url|
260
+ # URI objects don't have real hash equality! So, since this isn't perf sensitive we do a linear scan
261
+ unless @url_info.keys.include?(url)
262
+ state_changes[:added] << url.to_s
263
+ add_url(url)
264
+ end
265
+ end
266
+
267
+ # Delete connections not in the new list
268
+ @url_info.each do |url,_|
269
+ unless new_urls.include?(url)
270
+ state_changes[:removed] << url.to_s
271
+ remove_url(url)
272
+ end
273
+ end
274
+ end
275
+
276
+ if state_changes[:removed].size > 0 || state_changes[:added].size > 0
277
+ logger.info("Elasticsearch pool URLs updated", :changes => state_changes)
278
+ end
279
+ end
280
+
281
+ def size
282
+ @state_mutex.synchronize { @url_info.size }
283
+ end
284
+
285
+ def add_url(url)
286
+ @url_info[url] ||= empty_url_meta
287
+ end
288
+
289
+ def remove_url(url)
290
+ @url_info.delete(url)
291
+ end
292
+
293
+ def empty_url_meta
294
+ {
295
+ :in_use => 0,
296
+ :dead => false
297
+ }
298
+ end
299
+
300
+ def with_connection
301
+ url, url_meta = get_connection
302
+
303
+ # Custom error class used here so that users may retry attempts if they receive this error
304
+ # should they choose to
305
+ raise NoConnectionAvailableError, "No Available connections" unless url
306
+ yield url
307
+ rescue HostUnreachableError => e
308
+ # Mark the connection as dead here since this is likely not transient
309
+ mark_dead(url, e)
310
+ raise e
311
+ rescue BadResponseCodeError => e
312
+ # These aren't discarded from the pool because these are often very transient
313
+ # errors
314
+ raise e
315
+ rescue => e
316
+ logger.warn("UNEXPECTED POOL ERROR", :e => e)
317
+ raise e
318
+ ensure
319
+ return_connection(url)
320
+ end
321
+
322
+ def mark_dead(url, error)
323
+ @state_mutex.synchronize do
324
+ meta = @url_info[url]
325
+ # In case a sniff happened removing the metadata just before there's nothing to mark
326
+ # This is an extreme edge case, but it can happen!
327
+ return unless meta
328
+ logger.warn("Marking url as dead. Last error: [#{error.class}] #{error.message}",
329
+ :url => url, :error_message => error.message, :error_class => error.class.name)
330
+ meta[:dead] = true
331
+ meta[:last_error] = error
332
+ meta[:last_errored_at] = Time.now
333
+ end
334
+ end
335
+
336
+ def url_meta(url)
337
+ @state_mutex.synchronize do
338
+ @url_info[url]
339
+ end
340
+ end
341
+
342
+ def get_connection
343
+ @state_mutex.synchronize do
344
+ # The goal here is to pick a random connection from the least-in-use connections
345
+ # We want some randomness so that we don't hit the same node over and over, but
346
+ # we also want more 'fair' behavior in the event of high concurrency
347
+ eligible_set = nil
348
+ lowest_value_seen = nil
349
+ @url_info.each do |url,meta|
350
+ meta_in_use = meta[:in_use]
351
+ next if meta[:dead]
352
+
353
+ if lowest_value_seen.nil? || meta_in_use < lowest_value_seen
354
+ lowest_value_seen = meta_in_use
355
+ eligible_set = [[url, meta]]
356
+ elsif lowest_value_seen == meta_in_use
357
+ eligible_set << [url, meta]
358
+ end
359
+ end
360
+
361
+ return nil if eligible_set.nil?
362
+
363
+ pick, pick_meta = eligible_set.sample
364
+ pick_meta[:in_use] += 1
365
+
366
+ [pick, pick_meta]
367
+ end
368
+ end
369
+
370
+ def return_connection(url)
371
+ @state_mutex.synchronize do
372
+ if @url_info[url] # Guard against the condition where the connection has already been deleted
373
+ @url_info[url][:in_use] -= 1
374
+ end
375
+ end
376
+ end
377
+ end
378
+ end; end; end; end;