logstash-output-amazon_es 1.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0c5f98a6e4c8fcd85d45b66ad156290bb73e6f7f
|
4
|
+
data.tar.gz: f7244a69869f93394924f217cce12daa5d3bb91f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 72ce932d32910a609a80fbb13479dd190a812cc642951f0eedbf011a13bf0176acffb0416adf357df8880ad2c335e4566f21e673db6fcc619ff99c33635971b5
|
7
|
+
data.tar.gz: 0ae5201cf31bf696513332671646c5952f52e674c8d39731a9bcd2438f4d38533a9073639b57e0a652b78481ae4c6170c9a2067d6065fb413fdab11677d13155
|
data/README.md
CHANGED
@@ -16,6 +16,7 @@ if above does not work, or you would like to patch code here is a workaround to
|
|
16
16
|
1. Check out/clone this code from github
|
17
17
|
2. Build plugin using - `gem build logstash-output-amazon_es.gemspec` ( this works with jruby and rubygem versions > 1.9)
|
18
18
|
3. Install plugin using `<logstash-home>/bin/plugin install logstash-output-amazon_es-0.2.0-java.gem` (or the non java variant)
|
19
|
+
4. For the 2.3 version , please use '<logstash-home>/bin/logstash-plugin install logstash-output-amazon_es-1.0-java.gem'
|
19
20
|
|
20
21
|
## Configuration for Amazon Elasticsearch Output plugin
|
21
22
|
|
@@ -288,7 +288,7 @@ class LogStash::Outputs::AmazonES < LogStash::Outputs::Base
|
|
288
288
|
@retry_queue_not_full.wait(@retry_flush_mutex) while @retry_queue.size > @retry_max_items
|
289
289
|
}
|
290
290
|
|
291
|
-
event[
|
291
|
+
event.set('[@metadata][retry_count]', 0)
|
292
292
|
|
293
293
|
# Set the 'type' value for the index.
|
294
294
|
type = if @document_type
|
@@ -296,7 +296,7 @@ class LogStash::Outputs::AmazonES < LogStash::Outputs::Base
|
|
296
296
|
elsif @index_type # deprecated
|
297
297
|
event.sprintf(@index_type)
|
298
298
|
else
|
299
|
-
event
|
299
|
+
event.get('type') || 'logs'
|
300
300
|
end
|
301
301
|
|
302
302
|
params = {
|
@@ -438,9 +438,9 @@ class LogStash::Outputs::AmazonES < LogStash::Outputs::Base
|
|
438
438
|
unless @retry_queue.empty?
|
439
439
|
buffer = @retry_queue.size.times.map do
|
440
440
|
next_action, next_doc, next_event = @retry_queue.pop
|
441
|
-
next_event['@metadata
|
441
|
+
next_event.set('[@metadata][retry_count]', next_event.get('[@metadata][retry_count]') + 1)
|
442
442
|
|
443
|
-
if next_event[
|
443
|
+
if next_event.get('[@metadata][retry_count]') > @max_retries
|
444
444
|
@logger.error "too many attempts at sending event. dropping: #{next_event}"
|
445
445
|
nil
|
446
446
|
else
|
@@ -33,7 +33,10 @@ module Elasticsearch
|
|
33
33
|
:access_key_id,
|
34
34
|
:secret_access_key,
|
35
35
|
:session_token,
|
36
|
-
:profile
|
36
|
+
:profile,
|
37
|
+
:instance_profile_credentials_retries,
|
38
|
+
:instance_profile_credentials_timeout,
|
39
|
+
:region
|
37
40
|
)
|
38
41
|
|
39
42
|
# Performs the request by invoking {Transport::Base#perform_request} with a block.
|
@@ -63,8 +66,11 @@ module Elasticsearch
|
|
63
66
|
secret_access_key = options[:aws_secret_access_key] || nil
|
64
67
|
session_token = options[:session_token] || nil
|
65
68
|
profile = options[:profile] || 'default'
|
69
|
+
instance_cred_retries = options[:instance_profile_credentials_retries] || 0
|
70
|
+
instance_cred_timeout = options[:instance_profile_credentials_timeout] || 1
|
66
71
|
|
67
|
-
credential_config = CredentialConfig.new(access_key_id, secret_access_key, session_token, profile
|
72
|
+
credential_config = CredentialConfig.new(access_key_id, secret_access_key, session_token, profile,
|
73
|
+
instance_cred_retries, instance_cred_timeout, region)
|
68
74
|
credentials = Aws::CredentialProviderChain.new(credential_config).resolve
|
69
75
|
|
70
76
|
Connections::Collection.new \
|
@@ -100,4 +106,4 @@ module Elasticsearch
|
|
100
106
|
end
|
101
107
|
end
|
102
108
|
end
|
103
|
-
end
|
109
|
+
end
|
@@ -69,21 +69,22 @@ module LogStash::Outputs::AES
|
|
69
69
|
@client_options = {
|
70
70
|
:hosts => uris,
|
71
71
|
:region => options[:region],
|
72
|
-
:aws_access_key_id => options[:aws_access_key_id],
|
73
|
-
:aws_secret_access_key => options[:aws_secret_access_key],
|
74
72
|
:transport_options => {
|
75
73
|
:request => {:open_timeout => 0, :timeout => 60}, # ELB timeouts are set at 60
|
76
74
|
:proxy => client_settings[:proxy],
|
77
75
|
},
|
78
76
|
:transport_class => Elasticsearch::Transport::Transport::HTTP::AWS
|
79
77
|
}
|
78
|
+
internal_options = @client_options.clone
|
79
|
+
internal_options[:aws_access_key_id] = options[:aws_access_key_id]
|
80
|
+
internal_options[:aws_secret_access_key] = options[:aws_secret_access_key]
|
80
81
|
|
81
82
|
if options[:user] && options[:password] then
|
82
83
|
token = Base64.strict_encode64(options[:user] + ":" + options[:password])
|
83
|
-
|
84
|
+
internal_options[:headers] = { "Authorization" => "Basic #{token}" }
|
84
85
|
end
|
85
86
|
|
86
|
-
Elasticsearch::Client.new(
|
87
|
+
Elasticsearch::Client.new(internal_options)
|
87
88
|
end
|
88
89
|
|
89
90
|
def self.normalize_bulk_response(bulk_response)
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-output-amazon_es'
|
4
|
-
s.version = '1.0'
|
4
|
+
s.version = '1.1.0'
|
5
5
|
s.licenses = ['apache-2.0']
|
6
6
|
s.summary = "Logstash Output to Amazon Elasticsearch Service"
|
7
7
|
s.description = "Output events to Amazon Elasticsearch Service with V4 signing"
|
@@ -21,14 +21,15 @@ Gem::Specification.new do |s|
|
|
21
21
|
|
22
22
|
# Gem dependencies
|
23
23
|
s.add_runtime_dependency 'concurrent-ruby'
|
24
|
-
s.add_runtime_dependency 'elasticsearch',
|
24
|
+
s.add_runtime_dependency 'elasticsearch', '~> 1.0', '>= 1.0.10'
|
25
25
|
s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
|
26
26
|
s.add_runtime_dependency 'cabin', ['~> 0.6']
|
27
|
-
s.add_runtime_dependency
|
28
|
-
s.add_runtime_dependency
|
27
|
+
s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99'
|
28
|
+
s.add_runtime_dependency 'aws-sdk', '>= 2.1.14', '~> 2'
|
29
29
|
s.add_runtime_dependency "faraday", '~> 0.9.1'
|
30
30
|
s.add_runtime_dependency "faraday_middleware", '~> 0.10.0'
|
31
31
|
|
32
|
+
s.add_development_dependency 'addressable', '< 2.5.0'
|
32
33
|
s.add_development_dependency 'ftw', '~> 0.0.42'
|
33
34
|
s.add_development_dependency 'logstash-input-generator'
|
34
35
|
|
metadata
CHANGED
@@ -1,199 +1,219 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-amazon_es
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Amazon
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-01-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: concurrent-ruby
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- -
|
17
|
+
- - '>='
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- -
|
24
|
+
- - '>='
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: elasticsearch
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- -
|
31
|
+
- - ~>
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '1.0'
|
34
|
+
- - '>='
|
32
35
|
- !ruby/object:Gem::Version
|
33
36
|
version: 1.0.10
|
34
37
|
type: :runtime
|
35
38
|
prerelease: false
|
36
39
|
version_requirements: !ruby/object:Gem::Requirement
|
37
40
|
requirements:
|
38
|
-
- -
|
41
|
+
- - ~>
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: '1.0'
|
44
|
+
- - '>='
|
39
45
|
- !ruby/object:Gem::Version
|
40
46
|
version: 1.0.10
|
41
47
|
- !ruby/object:Gem::Dependency
|
42
48
|
name: stud
|
43
49
|
requirement: !ruby/object:Gem::Requirement
|
44
50
|
requirements:
|
45
|
-
- -
|
51
|
+
- - '>='
|
46
52
|
- !ruby/object:Gem::Version
|
47
53
|
version: 0.0.17
|
48
|
-
- -
|
54
|
+
- - ~>
|
49
55
|
- !ruby/object:Gem::Version
|
50
56
|
version: '0.0'
|
51
57
|
type: :runtime
|
52
58
|
prerelease: false
|
53
59
|
version_requirements: !ruby/object:Gem::Requirement
|
54
60
|
requirements:
|
55
|
-
- -
|
61
|
+
- - '>='
|
56
62
|
- !ruby/object:Gem::Version
|
57
63
|
version: 0.0.17
|
58
|
-
- -
|
64
|
+
- - ~>
|
59
65
|
- !ruby/object:Gem::Version
|
60
66
|
version: '0.0'
|
61
67
|
- !ruby/object:Gem::Dependency
|
62
68
|
name: cabin
|
63
69
|
requirement: !ruby/object:Gem::Requirement
|
64
70
|
requirements:
|
65
|
-
- -
|
71
|
+
- - ~>
|
66
72
|
- !ruby/object:Gem::Version
|
67
73
|
version: '0.6'
|
68
74
|
type: :runtime
|
69
75
|
prerelease: false
|
70
76
|
version_requirements: !ruby/object:Gem::Requirement
|
71
77
|
requirements:
|
72
|
-
- -
|
78
|
+
- - ~>
|
73
79
|
- !ruby/object:Gem::Version
|
74
80
|
version: '0.6'
|
75
81
|
- !ruby/object:Gem::Dependency
|
76
|
-
name: logstash-core
|
82
|
+
name: logstash-core-plugin-api
|
77
83
|
requirement: !ruby/object:Gem::Requirement
|
78
84
|
requirements:
|
79
|
-
- -
|
85
|
+
- - '>='
|
80
86
|
- !ruby/object:Gem::Version
|
81
|
-
version: 1.
|
82
|
-
- -
|
87
|
+
version: '1.60'
|
88
|
+
- - <=
|
83
89
|
- !ruby/object:Gem::Version
|
84
|
-
version:
|
90
|
+
version: '2.99'
|
85
91
|
type: :runtime
|
86
92
|
prerelease: false
|
87
93
|
version_requirements: !ruby/object:Gem::Requirement
|
88
94
|
requirements:
|
89
|
-
- -
|
95
|
+
- - '>='
|
90
96
|
- !ruby/object:Gem::Version
|
91
|
-
version: 1.
|
92
|
-
- -
|
97
|
+
version: '1.60'
|
98
|
+
- - <=
|
93
99
|
- !ruby/object:Gem::Version
|
94
|
-
version:
|
100
|
+
version: '2.99'
|
95
101
|
- !ruby/object:Gem::Dependency
|
96
102
|
name: aws-sdk
|
97
103
|
requirement: !ruby/object:Gem::Requirement
|
98
104
|
requirements:
|
99
|
-
- -
|
105
|
+
- - '>='
|
100
106
|
- !ruby/object:Gem::Version
|
101
107
|
version: 2.1.14
|
102
|
-
- -
|
108
|
+
- - ~>
|
103
109
|
- !ruby/object:Gem::Version
|
104
110
|
version: '2'
|
105
111
|
type: :runtime
|
106
112
|
prerelease: false
|
107
113
|
version_requirements: !ruby/object:Gem::Requirement
|
108
114
|
requirements:
|
109
|
-
- -
|
115
|
+
- - '>='
|
110
116
|
- !ruby/object:Gem::Version
|
111
117
|
version: 2.1.14
|
112
|
-
- -
|
118
|
+
- - ~>
|
113
119
|
- !ruby/object:Gem::Version
|
114
120
|
version: '2'
|
115
121
|
- !ruby/object:Gem::Dependency
|
116
122
|
name: faraday
|
117
123
|
requirement: !ruby/object:Gem::Requirement
|
118
124
|
requirements:
|
119
|
-
- -
|
125
|
+
- - ~>
|
120
126
|
- !ruby/object:Gem::Version
|
121
127
|
version: 0.9.1
|
122
128
|
type: :runtime
|
123
129
|
prerelease: false
|
124
130
|
version_requirements: !ruby/object:Gem::Requirement
|
125
131
|
requirements:
|
126
|
-
- -
|
132
|
+
- - ~>
|
127
133
|
- !ruby/object:Gem::Version
|
128
134
|
version: 0.9.1
|
129
135
|
- !ruby/object:Gem::Dependency
|
130
136
|
name: faraday_middleware
|
131
137
|
requirement: !ruby/object:Gem::Requirement
|
132
138
|
requirements:
|
133
|
-
- -
|
139
|
+
- - ~>
|
134
140
|
- !ruby/object:Gem::Version
|
135
141
|
version: 0.10.0
|
136
142
|
type: :runtime
|
137
143
|
prerelease: false
|
138
144
|
version_requirements: !ruby/object:Gem::Requirement
|
139
145
|
requirements:
|
140
|
-
- -
|
146
|
+
- - ~>
|
141
147
|
- !ruby/object:Gem::Version
|
142
148
|
version: 0.10.0
|
149
|
+
- !ruby/object:Gem::Dependency
|
150
|
+
name: addressable
|
151
|
+
requirement: !ruby/object:Gem::Requirement
|
152
|
+
requirements:
|
153
|
+
- - <
|
154
|
+
- !ruby/object:Gem::Version
|
155
|
+
version: 2.5.0
|
156
|
+
type: :development
|
157
|
+
prerelease: false
|
158
|
+
version_requirements: !ruby/object:Gem::Requirement
|
159
|
+
requirements:
|
160
|
+
- - <
|
161
|
+
- !ruby/object:Gem::Version
|
162
|
+
version: 2.5.0
|
143
163
|
- !ruby/object:Gem::Dependency
|
144
164
|
name: ftw
|
145
165
|
requirement: !ruby/object:Gem::Requirement
|
146
166
|
requirements:
|
147
|
-
- -
|
167
|
+
- - ~>
|
148
168
|
- !ruby/object:Gem::Version
|
149
169
|
version: 0.0.42
|
150
170
|
type: :development
|
151
171
|
prerelease: false
|
152
172
|
version_requirements: !ruby/object:Gem::Requirement
|
153
173
|
requirements:
|
154
|
-
- -
|
174
|
+
- - ~>
|
155
175
|
- !ruby/object:Gem::Version
|
156
176
|
version: 0.0.42
|
157
177
|
- !ruby/object:Gem::Dependency
|
158
178
|
name: logstash-input-generator
|
159
179
|
requirement: !ruby/object:Gem::Requirement
|
160
180
|
requirements:
|
161
|
-
- -
|
181
|
+
- - '>='
|
162
182
|
- !ruby/object:Gem::Version
|
163
183
|
version: '0'
|
164
184
|
type: :development
|
165
185
|
prerelease: false
|
166
186
|
version_requirements: !ruby/object:Gem::Requirement
|
167
187
|
requirements:
|
168
|
-
- -
|
188
|
+
- - '>='
|
169
189
|
- !ruby/object:Gem::Version
|
170
190
|
version: '0'
|
171
191
|
- !ruby/object:Gem::Dependency
|
172
192
|
name: logstash-devutils
|
173
193
|
requirement: !ruby/object:Gem::Requirement
|
174
194
|
requirements:
|
175
|
-
- -
|
195
|
+
- - '>='
|
176
196
|
- !ruby/object:Gem::Version
|
177
197
|
version: '0'
|
178
198
|
type: :development
|
179
199
|
prerelease: false
|
180
200
|
version_requirements: !ruby/object:Gem::Requirement
|
181
201
|
requirements:
|
182
|
-
- -
|
202
|
+
- - '>='
|
183
203
|
- !ruby/object:Gem::Version
|
184
204
|
version: '0'
|
185
205
|
- !ruby/object:Gem::Dependency
|
186
206
|
name: longshoreman
|
187
207
|
requirement: !ruby/object:Gem::Requirement
|
188
208
|
requirements:
|
189
|
-
- -
|
209
|
+
- - '>='
|
190
210
|
- !ruby/object:Gem::Version
|
191
211
|
version: '0'
|
192
212
|
type: :development
|
193
213
|
prerelease: false
|
194
214
|
version_requirements: !ruby/object:Gem::Requirement
|
195
215
|
requirements:
|
196
|
-
- -
|
216
|
+
- - '>='
|
197
217
|
- !ruby/object:Gem::Version
|
198
218
|
version: '0'
|
199
219
|
description: Output events to Amazon Elasticsearch Service with V4 signing
|
@@ -202,21 +222,21 @@ executables: []
|
|
202
222
|
extensions: []
|
203
223
|
extra_rdoc_files: []
|
204
224
|
files:
|
205
|
-
- Gemfile
|
206
|
-
- LICENSE
|
207
|
-
- NOTICE.TXT
|
208
|
-
- README.md
|
209
|
-
- lib/logstash/outputs/amazon_es.rb
|
210
225
|
- lib/logstash/outputs/amazon_es/aws_transport.rb
|
211
226
|
- lib/logstash/outputs/amazon_es/aws_v4_signer.rb
|
212
227
|
- lib/logstash/outputs/amazon_es/aws_v4_signer_impl.rb
|
213
228
|
- lib/logstash/outputs/amazon_es/elasticsearch-template.json
|
214
229
|
- lib/logstash/outputs/amazon_es/http_client.rb
|
215
|
-
- logstash
|
230
|
+
- lib/logstash/outputs/amazon_es.rb
|
216
231
|
- spec/amazon_es_spec_helper.rb
|
217
232
|
- spec/unit/outputs/amazon_es_spec.rb
|
218
233
|
- spec/unit/outputs/elasticsearch/protocol_spec.rb
|
219
234
|
- spec/unit/outputs/elasticsearch_proxy_spec.rb
|
235
|
+
- logstash-output-amazon_es.gemspec
|
236
|
+
- README.md
|
237
|
+
- Gemfile
|
238
|
+
- LICENSE
|
239
|
+
- NOTICE.TXT
|
220
240
|
homepage: http://logstash.net/
|
221
241
|
licenses:
|
222
242
|
- apache-2.0
|
@@ -229,17 +249,17 @@ require_paths:
|
|
229
249
|
- lib
|
230
250
|
required_ruby_version: !ruby/object:Gem::Requirement
|
231
251
|
requirements:
|
232
|
-
- -
|
252
|
+
- - '>='
|
233
253
|
- !ruby/object:Gem::Version
|
234
254
|
version: '0'
|
235
255
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
236
256
|
requirements:
|
237
|
-
- -
|
257
|
+
- - '>='
|
238
258
|
- !ruby/object:Gem::Version
|
239
259
|
version: '0'
|
240
260
|
requirements: []
|
241
261
|
rubyforge_project:
|
242
|
-
rubygems_version: 2.
|
262
|
+
rubygems_version: 2.0.14.1
|
243
263
|
signing_key:
|
244
264
|
specification_version: 4
|
245
265
|
summary: Logstash Output to Amazon Elasticsearch Service
|