logstash-output-amazon_es 7.0-java → 7.0.1-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +10 -7
- data/lib/logstash/outputs/amazon_es.rb +7 -4
- data/lib/logstash/outputs/amazon_es/common.rb +4 -4
- data/lib/logstash/outputs/amazon_es/common_configs.rb +1 -1
- data/lib/logstash/outputs/amazon_es/http_client.rb +9 -21
- data/lib/logstash/outputs/amazon_es/http_client/manticore_adapter.rb +2 -2
- data/lib/logstash/outputs/amazon_es/http_client/pool.rb +1 -1
- data/lib/logstash/outputs/amazon_es/http_client_builder.rb +5 -3
- data/lib/logstash/outputs/amazon_es/template_manager.rb +1 -1
- data/logstash-output-amazon_es.gemspec +3 -5
- data/spec/es_spec_helper.rb +0 -5
- data/spec/unit/http_client_builder_spec.rb +2 -2
- data/spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb +3 -3
- data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb +2 -2
- data/spec/unit/outputs/elasticsearch/http_client_spec.rb +7 -5
- data/spec/unit/outputs/elasticsearch/template_manager_spec.rb +1 -1
- data/spec/unit/outputs/elasticsearch_spec.rb +2 -2
- data/spec/unit/outputs/error_whitelist_spec.rb +1 -1
- metadata +12 -20
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8c5f134b8fa5c15b4487b4db40169c975fe4980a86c15859e914c2f2834484c2
|
4
|
+
data.tar.gz: 7b603d2e39205e83e67925369f0b197970133a7602ebba7f6e7c1e5a4a4612e8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 642d6f09125457e3ff52b67b3c43684230ea351ee6b067332d49f1432e782fb30d5b5c6a81da62f0cdffd696bddaa71e1e749913f0a79084ca4dfd89bbf10488
|
7
|
+
data.tar.gz: b205a14fd96273c85d4c33422ce29a57b5142ef26822c9b582de76625669f2eafc4b8bdeec291b0ef75cb4046e88c0c7861753ecccc108266176e593535d985d
|
data/README.md
CHANGED
@@ -15,9 +15,11 @@ The following table shows the versions of logstash and logstash-output-amazon_es
|
|
15
15
|
| 6.0.0 | <6.0.0 |
|
16
16
|
| 6.4.0 | >6.0.0 |
|
17
17
|
|
18
|
-
|
18
|
+
logstash-output-amazon_es plugin versions 6.4.0 and newer are tested to be compatible with Elasticsearch 6.5 and greater.
|
19
19
|
|
20
|
-
|
20
|
+
## Configuration for Amazon Elasticsearch Service Output Plugin
|
21
|
+
|
22
|
+
To run the Logstash Output Amazon Elasticsearch Service plugin, simply add a configuration following the below documentation.
|
21
23
|
|
22
24
|
An example configuration:
|
23
25
|
|
@@ -46,17 +48,18 @@ output {
|
|
46
48
|
* aws_access_key_id, :validate => :string - optional AWS access key
|
47
49
|
* aws_secret_access_key, :validate => :string - optional AWS secret key
|
48
50
|
|
49
|
-
|
51
|
+
The credential resolution logic can be described as follows:
|
50
52
|
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
53
|
+
- User passed `aws_access_key_id` and `aws_secret_access_key` in `amazon_es` configuration
|
54
|
+
- Environment variables - `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` (RECOMMENDED since they are recognized by all the AWS SDKs and CLI except for .NET), or `AWS_ACCESS_KEY` and `AWS_SECRET_KEY` (only recognized by Java SDK)
|
55
|
+
- Credential profiles file at the default location (`~/.aws/credentials`) shared by all AWS SDKs and the AWS CLI
|
56
|
+
- Instance profile credentials delivered through the Amazon EC2 metadata service
|
55
57
|
|
56
58
|
- template (path) - You can set the path to your own template here, if you so desire. If not set, the included template will be used.
|
57
59
|
- template_name (string, default => "logstash") - defines how the template is named inside Elasticsearch
|
58
60
|
- port (string, default 443) - Amazon Elasticsearch Service listens on port 443 for HTTPS (default) and port 80 for HTTP. Tweak this value for a custom proxy.
|
59
61
|
- protocol (string, default https) - The protocol used to connect to the Amazon Elasticsearch Service
|
62
|
+
- max_bulk_bytes - The max size for a bulk request in bytes. Default is 20MB. It is recommended not to change this value unless needed. For guidance on changing this value, please consult the table for network limits for your instance type: https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/aes-limits.html#network-limits
|
60
63
|
|
61
64
|
After 6.4.0, users can't set batch size in this output plugin config. However, users can still set batch size in logstash.yml file.
|
62
65
|
## Developing
|
@@ -83,7 +83,7 @@ require "forwardable"
|
|
83
83
|
# For requests compression, regardless of the Elasticsearch version, users have to enable `http_compression`
|
84
84
|
# setting in their Logstash config file.
|
85
85
|
#
|
86
|
-
class LogStash::Outputs::
|
86
|
+
class LogStash::Outputs::AmazonElasticSearch < LogStash::Outputs::Base
|
87
87
|
declare_threadsafe!
|
88
88
|
|
89
89
|
require "logstash/outputs/amazon_es/http_client"
|
@@ -92,10 +92,10 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
92
92
|
require "logstash/outputs/amazon_es/common"
|
93
93
|
|
94
94
|
# Protocol agnostic (i.e. non-http, non-java specific) configs go here
|
95
|
-
include(LogStash::Outputs::
|
95
|
+
include(LogStash::Outputs::AmazonElasticSearch::CommonConfigs)
|
96
96
|
|
97
97
|
# Protocol agnostic methods
|
98
|
-
include(LogStash::Outputs::
|
98
|
+
include(LogStash::Outputs::AmazonElasticSearch::Common)
|
99
99
|
|
100
100
|
config_name "amazon_es"
|
101
101
|
|
@@ -248,9 +248,12 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
248
248
|
# Custom Headers to send on each request to amazon_es nodes
|
249
249
|
config :custom_headers, :validate => :hash, :default => {}
|
250
250
|
|
251
|
+
#Max bulk size in bytes
|
252
|
+
config :max_bulk_bytes, :validate => :number, :default => 20 * 1024 * 1024
|
253
|
+
|
251
254
|
def build_client
|
252
255
|
params["metric"] = metric
|
253
|
-
@client ||= ::LogStash::Outputs::
|
256
|
+
@client ||= ::LogStash::Outputs::AmazonElasticSearch::HttpClientBuilder.build(@logger, @hosts, params)
|
254
257
|
end
|
255
258
|
|
256
259
|
def close
|
@@ -1,6 +1,6 @@
|
|
1
1
|
require "logstash/outputs/amazon_es/template_manager"
|
2
2
|
|
3
|
-
module LogStash; module Outputs; class
|
3
|
+
module LogStash; module Outputs; class AmazonElasticSearch;
|
4
4
|
module Common
|
5
5
|
attr_reader :client, :hosts
|
6
6
|
|
@@ -274,7 +274,7 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
274
274
|
es_actions = actions.map {|action_type, params, event| [action_type, params, event.to_hash]}
|
275
275
|
response = @client.bulk(es_actions)
|
276
276
|
response
|
277
|
-
rescue ::LogStash::Outputs::
|
277
|
+
rescue ::LogStash::Outputs::AmazonElasticSearch::HttpClient::Pool::HostUnreachableError => e
|
278
278
|
# If we can't even connect to the server let's just print out the URL (:hosts is actually a URL)
|
279
279
|
# and let the user sort it out from there
|
280
280
|
@logger.error(
|
@@ -290,7 +290,7 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
290
290
|
sleep_interval = sleep_for_interval(sleep_interval)
|
291
291
|
@bulk_request_metrics.increment(:failures)
|
292
292
|
retry unless @stopping.true?
|
293
|
-
rescue ::LogStash::Outputs::
|
293
|
+
rescue ::LogStash::Outputs::AmazonElasticSearch::HttpClient::Pool::NoConnectionAvailableError => e
|
294
294
|
@logger.error(
|
295
295
|
"Attempted to send a bulk request to elasticsearch, but no there are no living connections in the connection pool. Perhaps Elasticsearch is unreachable or down?",
|
296
296
|
:error_message => e.message,
|
@@ -301,7 +301,7 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
301
301
|
sleep_interval = next_sleep_interval(sleep_interval)
|
302
302
|
@bulk_request_metrics.increment(:failures)
|
303
303
|
retry unless @stopping.true?
|
304
|
-
rescue ::LogStash::Outputs::
|
304
|
+
rescue ::LogStash::Outputs::AmazonElasticSearch::HttpClient::Pool::BadResponseCodeError => e
|
305
305
|
@bulk_request_metrics.increment(:failures)
|
306
306
|
log_hash = {:code => e.response_code, :url => e.url.sanitized.to_s}
|
307
307
|
log_hash[:body] = e.response_body if @logger.debug? # Generally this is too verbose
|
@@ -1,6 +1,6 @@
|
|
1
1
|
require 'forwardable' # Needed for logstash core SafeURI. We need to patch this in core: https://github.com/elastic/logstash/pull/5978
|
2
2
|
|
3
|
-
module LogStash; module Outputs; class
|
3
|
+
module LogStash; module Outputs; class AmazonElasticSearch
|
4
4
|
module CommonConfigs
|
5
5
|
def self.included(mod)
|
6
6
|
# The index to write events to. This can be dynamic using the `%{foo}` syntax.
|
@@ -7,24 +7,11 @@ require 'cgi'
|
|
7
7
|
require 'zlib'
|
8
8
|
require 'stringio'
|
9
9
|
|
10
|
-
module LogStash; module Outputs; class
|
11
|
-
# This is a constant instead of a config option because
|
12
|
-
# there really isn't a good reason to configure it.
|
13
|
-
#
|
14
|
-
# The criteria used are:
|
15
|
-
# 1. We need a number that's less than 100MiB because ES
|
16
|
-
# won't accept bulks larger than that.
|
17
|
-
# 2. It must be large enough to amortize the connection constant
|
18
|
-
# across multiple requests.
|
19
|
-
# 3. It must be small enough that even if multiple threads hit this size
|
20
|
-
# we won't use a lot of heap.
|
21
|
-
#
|
22
|
-
# We wound up agreeing that a number greater than 10 MiB and less than 100MiB
|
23
|
-
# made sense. We picked one on the lowish side to not use too much heap.
|
24
|
-
TARGET_BULK_BYTES = 20 * 1024 * 1024 # 20MiB
|
25
|
-
|
10
|
+
module LogStash; module Outputs; class AmazonElasticSearch;
|
26
11
|
class HttpClient
|
27
|
-
attr_reader :client, :options, :logger, :pool, :action_count, :recv_count
|
12
|
+
attr_reader :client, :options, :logger, :pool, :action_count, :recv_count, :max_bulk_bytes
|
13
|
+
|
14
|
+
|
28
15
|
# This is here in case we use DEFAULT_OPTIONS in the future
|
29
16
|
# DEFAULT_OPTIONS = {
|
30
17
|
# :setting => value
|
@@ -54,6 +41,7 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
54
41
|
@metric = options[:metric]
|
55
42
|
@bulk_request_metrics = @metric.namespace(:bulk_requests)
|
56
43
|
@bulk_response_metrics = @bulk_request_metrics.namespace(:responses)
|
44
|
+
@max_bulk_bytes = options[:max_bulk_bytes]
|
57
45
|
|
58
46
|
# Again, in case we use DEFAULT_OPTIONS in the future, uncomment this.
|
59
47
|
# @options = DEFAULT_OPTIONS.merge(options)
|
@@ -121,7 +109,7 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
121
109
|
action.map {|line| LogStash::Json.dump(line)}.join("\n") :
|
122
110
|
LogStash::Json.dump(action)
|
123
111
|
as_json << "\n"
|
124
|
-
if (body_stream.size + as_json.bytesize) >
|
112
|
+
if (body_stream.size + as_json.bytesize) > @max_bulk_bytes
|
125
113
|
bulk_responses << bulk_send(body_stream) unless body_stream.size == 0
|
126
114
|
end
|
127
115
|
stream_writer.write(as_json)
|
@@ -152,7 +140,7 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
152
140
|
|
153
141
|
if response.code != 200
|
154
142
|
url = ::LogStash::Util::SafeURI.new(response.final_url)
|
155
|
-
raise ::LogStash::Outputs::
|
143
|
+
raise ::LogStash::Outputs::AmazonElasticSearch::HttpClient::Pool::BadResponseCodeError.new(
|
156
144
|
response.code, url, body_stream.to_s, response.body
|
157
145
|
)
|
158
146
|
end
|
@@ -292,7 +280,7 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
292
280
|
|
293
281
|
adapter_options[:aws_secret_access_key] = options[:aws_secret_access_key]
|
294
282
|
|
295
|
-
adapter_class = ::LogStash::Outputs::
|
283
|
+
adapter_class = ::LogStash::Outputs::AmazonElasticSearch::HttpClient::ManticoreAdapter
|
296
284
|
adapter = adapter_class.new(@logger, adapter_options)
|
297
285
|
end
|
298
286
|
|
@@ -310,7 +298,7 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
310
298
|
}
|
311
299
|
pool_options[:scheme] = self.scheme if self.scheme
|
312
300
|
|
313
|
-
pool_class = ::LogStash::Outputs::
|
301
|
+
pool_class = ::LogStash::Outputs::AmazonElasticSearch::HttpClient::Pool
|
314
302
|
full_urls = @options[:hosts].map {|h| host_to_url(h) }
|
315
303
|
pool = pool_class.new(@logger, adapter, full_urls, pool_options)
|
316
304
|
pool.start
|
@@ -3,7 +3,7 @@ require 'cgi'
|
|
3
3
|
require 'aws-sdk-core'
|
4
4
|
require 'uri'
|
5
5
|
|
6
|
-
module LogStash; module Outputs; class
|
6
|
+
module LogStash; module Outputs; class AmazonElasticSearch; class HttpClient;
|
7
7
|
DEFAULT_HEADERS = { "content-type" => "application/json" }
|
8
8
|
|
9
9
|
CredentialConfig = Struct.new(
|
@@ -123,7 +123,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
|
|
123
123
|
# template installation. We might need a better story around this later
|
124
124
|
# but for our current purposes this is correct
|
125
125
|
if resp.code < 200 || resp.code > 299 && resp.code != 404
|
126
|
-
raise ::LogStash::Outputs::
|
126
|
+
raise ::LogStash::Outputs::AmazonElasticSearch::HttpClient::Pool::BadResponseCodeError.new(resp.code, request_uri, body, resp.body)
|
127
127
|
end
|
128
128
|
|
129
129
|
resp
|
@@ -1,6 +1,6 @@
|
|
1
1
|
require 'cgi'
|
2
2
|
|
3
|
-
module LogStash; module Outputs; class
|
3
|
+
module LogStash; module Outputs; class AmazonElasticSearch;
|
4
4
|
module HttpClientBuilder
|
5
5
|
def self.build(logger, hosts, params)
|
6
6
|
client_settings = {
|
@@ -101,11 +101,13 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
101
101
|
:port => params["port"],
|
102
102
|
:region => params["region"],
|
103
103
|
:aws_access_key_id => params["aws_access_key_id"],
|
104
|
-
:aws_secret_access_key => params["aws_secret_access_key"]
|
104
|
+
:aws_secret_access_key => params["aws_secret_access_key"],
|
105
|
+
:max_bulk_bytes => params["max_bulk_bytes"])
|
106
|
+
)
|
105
107
|
end
|
106
108
|
|
107
109
|
def self.create_http_client(options)
|
108
|
-
LogStash::Outputs::
|
110
|
+
LogStash::Outputs::AmazonElasticSearch::HttpClient.new(options)
|
109
111
|
end
|
110
112
|
|
111
113
|
def self.setup_ssl(logger, params)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-amazon_es'
|
3
|
-
s.version = '7.0'
|
3
|
+
s.version = '7.0.1'
|
4
4
|
s.licenses = ['Apache-2.0']
|
5
5
|
s.summary = "Logstash Output to Amazon Elasticsearch Service"
|
6
6
|
s.description = "Output events to Amazon Elasticsearch Service with V4 signing"
|
@@ -26,9 +26,7 @@ Gem::Specification.new do |s|
|
|
26
26
|
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
27
27
|
s.add_runtime_dependency 'aws-sdk', '>= 2.3.22', '~> 2'
|
28
28
|
|
29
|
-
s.add_development_dependency 'logstash-codec-plain'
|
30
|
-
s.add_development_dependency 'logstash-devutils',
|
29
|
+
s.add_development_dependency 'logstash-codec-plain'
|
30
|
+
s.add_development_dependency 'logstash-devutils', "~> 1.3", ">= 1.3.1"
|
31
31
|
s.add_development_dependency 'flores', '~> 0'
|
32
|
-
# Still used in some specs, we should remove this ASAP
|
33
|
-
s.add_development_dependency 'elasticsearch', '~> 0'
|
34
32
|
end
|
data/spec/es_spec_helper.rb
CHANGED
@@ -1,6 +1,5 @@
|
|
1
1
|
require "logstash/devutils/rspec/spec_helper"
|
2
2
|
require 'manticore'
|
3
|
-
require 'elasticsearch'
|
4
3
|
|
5
4
|
# by default exclude secure_integration tests unless requested
|
6
5
|
# normal integration specs are already excluded by devutils' spec helper
|
@@ -13,10 +12,6 @@ module ESHelper
|
|
13
12
|
"127.0.0.1"
|
14
13
|
end
|
15
14
|
|
16
|
-
def get_client
|
17
|
-
Elasticsearch::Client.new(:hosts => [get_host_port])
|
18
|
-
end
|
19
|
-
|
20
15
|
def self.es_version
|
21
16
|
RSpec.configuration.filter[:es_version] || ENV['ES_VERSION']
|
22
17
|
end
|
@@ -3,9 +3,9 @@ require "logstash/outputs/amazon_es"
|
|
3
3
|
require "logstash/outputs/amazon_es/http_client"
|
4
4
|
require "logstash/outputs/amazon_es/http_client_builder"
|
5
5
|
|
6
|
-
describe LogStash::Outputs::
|
6
|
+
describe LogStash::Outputs::AmazonElasticSearch::HttpClientBuilder do
|
7
7
|
describe "auth setup with url encodable passwords" do
|
8
|
-
let(:klass) { LogStash::Outputs::
|
8
|
+
let(:klass) { LogStash::Outputs::AmazonElasticSearch::HttpClientBuilder }
|
9
9
|
let(:user) { "foo@bar"}
|
10
10
|
let(:password) {"baz@blah" }
|
11
11
|
let(:password_secured) do
|
@@ -1,7 +1,7 @@
|
|
1
1
|
require "logstash/devutils/rspec/spec_helper"
|
2
2
|
require "logstash/outputs/amazon_es/http_client"
|
3
3
|
|
4
|
-
describe LogStash::Outputs::
|
4
|
+
describe LogStash::Outputs::AmazonElasticSearch::HttpClient::ManticoreAdapter do
|
5
5
|
let(:logger) { Cabin::Channel.get }
|
6
6
|
let(:options) { {:aws_access_key_id => 'AAAAAAAAAAAAAAAAAAAA',
|
7
7
|
:aws_secret_access_key => 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'} }
|
@@ -34,12 +34,12 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter do
|
|
34
34
|
uri_with_path = uri.clone
|
35
35
|
uri_with_path.path = "/"
|
36
36
|
|
37
|
-
expect(::LogStash::Outputs::
|
37
|
+
expect(::LogStash::Outputs::AmazonElasticSearch::HttpClient::Pool::BadResponseCodeError).to receive(:new).
|
38
38
|
with(resp.code, uri_with_path, nil, resp.body).and_call_original
|
39
39
|
|
40
40
|
expect do
|
41
41
|
subject.perform_request(uri, :get, "/")
|
42
|
-
end.to raise_error(::LogStash::Outputs::
|
42
|
+
end.to raise_error(::LogStash::Outputs::AmazonElasticSearch::HttpClient::Pool::BadResponseCodeError)
|
43
43
|
end
|
44
44
|
end
|
45
45
|
|
@@ -2,9 +2,9 @@ require "logstash/devutils/rspec/spec_helper"
|
|
2
2
|
require "logstash/outputs/amazon_es/http_client"
|
3
3
|
require "json"
|
4
4
|
|
5
|
-
describe LogStash::Outputs::
|
5
|
+
describe LogStash::Outputs::AmazonElasticSearch::HttpClient::Pool do
|
6
6
|
let(:logger) { Cabin::Channel.get }
|
7
|
-
let(:adapter) { LogStash::Outputs::
|
7
|
+
let(:adapter) { LogStash::Outputs::AmazonElasticSearch::HttpClient::ManticoreAdapter.new(logger,
|
8
8
|
{:aws_access_key_id => 'AAAAAAAAAAAAAAAAAAAA',
|
9
9
|
:aws_secret_access_key => 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'}) }
|
10
10
|
let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
|
@@ -2,7 +2,7 @@ require "logstash/devutils/rspec/spec_helper"
|
|
2
2
|
require "logstash/outputs/amazon_es/http_client"
|
3
3
|
require "java"
|
4
4
|
|
5
|
-
describe LogStash::Outputs::
|
5
|
+
describe LogStash::Outputs::AmazonElasticSearch::HttpClient do
|
6
6
|
let(:ssl) { nil }
|
7
7
|
let(:base_options) do
|
8
8
|
opts = {
|
@@ -12,7 +12,8 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
|
|
12
12
|
:protocol => "http",
|
13
13
|
:port => 9200,
|
14
14
|
:aws_access_key_id => "AAAAAAAAAAAAAAAAAAAA",
|
15
|
-
:aws_secret_access_key => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
|
15
|
+
:aws_secret_access_key => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
|
16
|
+
:max_bulk_bytes => 20 * 1024 * 1024
|
16
17
|
}
|
17
18
|
|
18
19
|
if !ssl.nil? # Shortcut to set this
|
@@ -29,6 +30,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
|
|
29
30
|
let(:ipv6_hostname) { "[::1]" }
|
30
31
|
let(:ipv4_hostname) { "127.0.0.1" }
|
31
32
|
let(:port) { 9200 }
|
33
|
+
let(:max_bulk_bytes) { 20 * 1024 * 1024 }
|
32
34
|
let(:protocol) {"http"}
|
33
35
|
let(:aws_access_key_id) {"AAAAAAAAAAAAAAAAAAAA"}
|
34
36
|
let(:aws_secret_access_key) {"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"}
|
@@ -163,7 +165,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
|
|
163
165
|
]}
|
164
166
|
|
165
167
|
context "if a message is over TARGET_BULK_BYTES" do
|
166
|
-
let(:target_bulk_bytes) {
|
168
|
+
let(:target_bulk_bytes) { 20 * 1024 * 1024 }
|
167
169
|
let(:message) { "a" * (target_bulk_bytes + 1) }
|
168
170
|
|
169
171
|
it "should be handled properly" do
|
@@ -189,7 +191,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
|
|
189
191
|
end
|
190
192
|
|
191
193
|
context "if one exceeds TARGET_BULK_BYTES" do
|
192
|
-
let(:target_bulk_bytes) {
|
194
|
+
let(:target_bulk_bytes) { 20 * 1024 * 1024 }
|
193
195
|
let(:message1) { "a" * (target_bulk_bytes + 1) }
|
194
196
|
it "executes two bulk_send operations" do
|
195
197
|
allow(subject).to receive(:join_bulk_responses)
|
@@ -201,7 +203,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
|
|
201
203
|
end
|
202
204
|
|
203
205
|
describe "sniffing" do
|
204
|
-
let(:client) { LogStash::Outputs::
|
206
|
+
let(:client) { LogStash::Outputs::AmazonElasticSearch::HttpClient.new(base_options.merge(client_opts)) }
|
205
207
|
|
206
208
|
context "with sniffing enabled" do
|
207
209
|
let(:client_opts) { {:sniffing => true, :sniffing_delay => 1 } }
|
@@ -3,7 +3,7 @@ require "logstash/outputs/amazon_es/http_client"
|
|
3
3
|
require "java"
|
4
4
|
require "json"
|
5
5
|
|
6
|
-
describe LogStash::Outputs::
|
6
|
+
describe LogStash::Outputs::AmazonElasticSearch::TemplateManager do
|
7
7
|
|
8
8
|
describe ".default_template_path" do
|
9
9
|
context "amazon_es 1.x" do
|
@@ -2,7 +2,7 @@ require_relative "../../../spec/es_spec_helper"
|
|
2
2
|
require "flores/random"
|
3
3
|
require "logstash/outputs/amazon_es"
|
4
4
|
|
5
|
-
describe LogStash::Outputs::
|
5
|
+
describe LogStash::Outputs::AmazonElasticSearch do
|
6
6
|
subject { described_class.new(options) }
|
7
7
|
let(:options) { { "aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA",
|
8
8
|
"aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} }
|
@@ -222,7 +222,7 @@ describe LogStash::Outputs::ElasticSearch do
|
|
222
222
|
context "429 errors" do
|
223
223
|
let(:event) { ::LogStash::Event.new("foo" => "bar") }
|
224
224
|
let(:error) do
|
225
|
-
::LogStash::Outputs::
|
225
|
+
::LogStash::Outputs::AmazonElasticSearch::HttpClient::Pool::BadResponseCodeError.new(
|
226
226
|
429, double("url").as_null_object, double("request body"), double("response body")
|
227
227
|
)
|
228
228
|
end
|
@@ -14,7 +14,7 @@ describe "whitelisting error types in expected behavior" do
|
|
14
14
|
"aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA",
|
15
15
|
"aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} }
|
16
16
|
|
17
|
-
subject { LogStash::Outputs::
|
17
|
+
subject { LogStash::Outputs::AmazonElasticSearch.new(settings) }
|
18
18
|
|
19
19
|
before :each do
|
20
20
|
allow(subject.logger).to receive(:warn)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-amazon_es
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 7.0.1
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Amazon
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-03-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -107,7 +107,7 @@ dependencies:
|
|
107
107
|
- !ruby/object:Gem::Dependency
|
108
108
|
requirement: !ruby/object:Gem::Requirement
|
109
109
|
requirements:
|
110
|
-
- - "
|
110
|
+
- - ">="
|
111
111
|
- !ruby/object:Gem::Version
|
112
112
|
version: '0'
|
113
113
|
name: logstash-codec-plain
|
@@ -115,7 +115,7 @@ dependencies:
|
|
115
115
|
type: :development
|
116
116
|
version_requirements: !ruby/object:Gem::Requirement
|
117
117
|
requirements:
|
118
|
-
- - "
|
118
|
+
- - ">="
|
119
119
|
- !ruby/object:Gem::Version
|
120
120
|
version: '0'
|
121
121
|
- !ruby/object:Gem::Dependency
|
@@ -123,7 +123,10 @@ dependencies:
|
|
123
123
|
requirements:
|
124
124
|
- - "~>"
|
125
125
|
- !ruby/object:Gem::Version
|
126
|
-
version: '
|
126
|
+
version: '1.3'
|
127
|
+
- - ">="
|
128
|
+
- !ruby/object:Gem::Version
|
129
|
+
version: 1.3.1
|
127
130
|
name: logstash-devutils
|
128
131
|
prerelease: false
|
129
132
|
type: :development
|
@@ -131,28 +134,17 @@ dependencies:
|
|
131
134
|
requirements:
|
132
135
|
- - "~>"
|
133
136
|
- !ruby/object:Gem::Version
|
134
|
-
version: '
|
135
|
-
-
|
136
|
-
requirement: !ruby/object:Gem::Requirement
|
137
|
-
requirements:
|
138
|
-
- - "~>"
|
139
|
-
- !ruby/object:Gem::Version
|
140
|
-
version: '0'
|
141
|
-
name: flores
|
142
|
-
prerelease: false
|
143
|
-
type: :development
|
144
|
-
version_requirements: !ruby/object:Gem::Requirement
|
145
|
-
requirements:
|
146
|
-
- - "~>"
|
137
|
+
version: '1.3'
|
138
|
+
- - ">="
|
147
139
|
- !ruby/object:Gem::Version
|
148
|
-
version:
|
140
|
+
version: 1.3.1
|
149
141
|
- !ruby/object:Gem::Dependency
|
150
142
|
requirement: !ruby/object:Gem::Requirement
|
151
143
|
requirements:
|
152
144
|
- - "~>"
|
153
145
|
- !ruby/object:Gem::Version
|
154
146
|
version: '0'
|
155
|
-
name:
|
147
|
+
name: flores
|
156
148
|
prerelease: false
|
157
149
|
type: :development
|
158
150
|
version_requirements: !ruby/object:Gem::Requirement
|