logstash-output-amazon_es 1.0-java → 1.1.0-java

Sign up to get free protection for your applications and to get access to all the features.
Files changed (7) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1 -0
  3. data/lib/logstash/outputs/amazon_es.rb +4 -4
  4. data/lib/logstash/outputs/amazon_es/aws_transport.rb +9 -3
  5. data/lib/logstash/outputs/amazon_es/http_client.rb +5 -4
  6. data/logstash-output-amazon_es.gemspec +5 -4
  7. metadata +64 -44
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 9a5aa1eae9dcf3ff783b7eae6a03af1d6b1f1fd6
4
- data.tar.gz: e8628f198b297023f200c52ec2ce12f9d7dc825c
3
+ metadata.gz: 286292688e20f9f6f1760d64ca008b7efcf40051
4
+ data.tar.gz: 151fa5283bf52d5a7f32cd97aa204590fd30ecb6
5
5
  SHA512:
6
- metadata.gz: 3124f478af6545161580d741646350d40952a8a2f1af2ba6ebd4ff7ed2c20614442bfca6e3ad3ca23743af189dbf1d86a9a05da4e290fb2202a8eab085540e87
7
- data.tar.gz: 3cbbfb9af493f00ffa84cc0fc6960be2fa55dec1ab36d4181ca276097091604752e2509dff2f9c844b247d19e52e6ad0fa23a474a457249be1aacdd40c3e0986
6
+ metadata.gz: 8d078f26da791eaaa6a332ce5d56d2a6df2361850f8574bee45a1a897fe93693a78a0ceee40c114693eab095e54bd2bd319b4f01e9e3e87ff00ed6b9c499912d
7
+ data.tar.gz: b0af0146d4e514be49041b2e5b0b1f3764647652924d0a5f44ac9e118d4d73dc86aed838844def36d50529dd7def73a23c030786480f7c3636bb47060f8610fb
data/README.md CHANGED
@@ -16,6 +16,7 @@ if above does not work, or you would like to patch code here is a workaround to
16
16
  1. Check out/clone this code from github
17
17
  2. Build plugin using - `gem build logstash-output-amazon_es.gemspec` ( this works with jruby and rubygem versions > 1.9)
18
18
  3. Install plugin using `<logstash-home>/bin/plugin install logstash-output-amazon_es-0.2.0-java.gem` (or the non java variant)
19
+ 4. For the 2.3 version , please use '<logstash-home>/bin/logstash-plugin install logstash-output-amazon_es-1.0-java.gem'
19
20
 
20
21
  ## Configuration for Amazon Elasticsearch Output plugin
21
22
 
data/lib/logstash/outputs/amazon_es.rb CHANGED
@@ -288,7 +288,7 @@ class LogStash::Outputs::AmazonES < LogStash::Outputs::Base
288
288
  @retry_queue_not_full.wait(@retry_flush_mutex) while @retry_queue.size > @retry_max_items
289
289
  }
290
290
 
291
- event['@metadata']['retry_count'] = 0
291
+ event.set('[@metadata][retry_count]', 0)
292
292
 
293
293
  # Set the 'type' value for the index.
294
294
  type = if @document_type
@@ -296,7 +296,7 @@ class LogStash::Outputs::AmazonES < LogStash::Outputs::Base
296
296
  elsif @index_type # deprecated
297
297
  event.sprintf(@index_type)
298
298
  else
299
- event["type"] || "logs"
299
+ event.get('type') || 'logs'
300
300
  end
301
301
 
302
302
  params = {
@@ -438,9 +438,9 @@ class LogStash::Outputs::AmazonES < LogStash::Outputs::Base
438
438
  unless @retry_queue.empty?
439
439
  buffer = @retry_queue.size.times.map do
440
440
  next_action, next_doc, next_event = @retry_queue.pop
441
- next_event['@metadata']['retry_count'] += 1
441
+ next_event.set('[@metadata][retry_count]', next_event.get('[@metadata][retry_count]') + 1)
442
442
 
443
- if next_event['@metadata']['retry_count'] > @max_retries
443
+ if next_event.get('[@metadata][retry_count]') > @max_retries
444
444
  @logger.error "too many attempts at sending event. dropping: #{next_event}"
445
445
  nil
446
446
  else
data/lib/logstash/outputs/amazon_es/aws_transport.rb CHANGED
@@ -33,7 +33,10 @@ module Elasticsearch
33
33
  :access_key_id,
34
34
  :secret_access_key,
35
35
  :session_token,
36
- :profile
36
+ :profile,
37
+ :instance_profile_credentials_retries,
38
+ :instance_profile_credentials_timeout,
39
+ :region
37
40
  )
38
41
 
39
42
  # Performs the request by invoking {Transport::Base#perform_request} with a block.
@@ -63,8 +66,11 @@ module Elasticsearch
63
66
  secret_access_key = options[:aws_secret_access_key] || nil
64
67
  session_token = options[:session_token] || nil
65
68
  profile = options[:profile] || 'default'
69
+ instance_cred_retries = options[:instance_profile_credentials_retries] || 0
70
+ instance_cred_timeout = options[:instance_profile_credentials_timeout] || 1
66
71
 
67
- credential_config = CredentialConfig.new(access_key_id, secret_access_key, session_token, profile)
72
+ credential_config = CredentialConfig.new(access_key_id, secret_access_key, session_token, profile,
73
+ instance_cred_retries, instance_cred_timeout, region)
68
74
  credentials = Aws::CredentialProviderChain.new(credential_config).resolve
69
75
 
70
76
  Connections::Collection.new \
@@ -100,4 +106,4 @@ module Elasticsearch
100
106
  end
101
107
  end
102
108
  end
103
- end
109
+ end
data/lib/logstash/outputs/amazon_es/http_client.rb CHANGED
@@ -69,21 +69,22 @@ module LogStash::Outputs::AES
69
69
  @client_options = {
70
70
  :hosts => uris,
71
71
  :region => options[:region],
72
- :aws_access_key_id => options[:aws_access_key_id],
73
- :aws_secret_access_key => options[:aws_secret_access_key],
74
72
  :transport_options => {
75
73
  :request => {:open_timeout => 0, :timeout => 60}, # ELB timeouts are set at 60
76
74
  :proxy => client_settings[:proxy],
77
75
  },
78
76
  :transport_class => Elasticsearch::Transport::Transport::HTTP::AWS
79
77
  }
78
+ internal_options = @client_options.clone
79
+ internal_options[:aws_access_key_id] = options[:aws_access_key_id]
80
+ internal_options[:aws_secret_access_key] = options[:aws_secret_access_key]
80
81
 
81
82
  if options[:user] && options[:password] then
82
83
  token = Base64.strict_encode64(options[:user] + ":" + options[:password])
83
- @client_options[:headers] = { "Authorization" => "Basic #{token}" }
84
+ internal_options[:headers] = { "Authorization" => "Basic #{token}" }
84
85
  end
85
86
 
86
- Elasticsearch::Client.new(client_options)
87
+ Elasticsearch::Client.new(internal_options)
87
88
  end
88
89
 
89
90
  def self.normalize_bulk_response(bulk_response)
data/logstash-output-amazon_es.gemspec CHANGED
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-output-amazon_es'
4
- s.version = '1.0'
4
+ s.version = '1.1.0'
5
5
  s.licenses = ['apache-2.0']
6
6
  s.summary = "Logstash Output to Amazon Elasticsearch Service"
7
7
  s.description = "Output events to Amazon Elasticsearch Service with V4 signing"
@@ -21,14 +21,15 @@ Gem::Specification.new do |s|
21
21
 
22
22
  # Gem dependencies
23
23
  s.add_runtime_dependency 'concurrent-ruby'
24
- s.add_runtime_dependency 'elasticsearch', ['>= 1.0.10']
24
+ s.add_runtime_dependency 'elasticsearch', '~> 1.0', '>= 1.0.10'
25
25
  s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
26
26
  s.add_runtime_dependency 'cabin', ['~> 0.6']
27
- s.add_runtime_dependency "logstash-core", ">= 1.4.0", "< 3.0.0"
28
- s.add_runtime_dependency "aws-sdk", ['>= 2.1.14', '~> 2']
27
+ s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99'
28
+ s.add_runtime_dependency 'aws-sdk', '>= 2.1.14', '~> 2'
29
29
  s.add_runtime_dependency "faraday", '~> 0.9.1'
30
30
  s.add_runtime_dependency "faraday_middleware", '~> 0.10.0'
31
31
 
32
+ s.add_development_dependency 'addressable', '< 2.5.0'
32
33
  s.add_development_dependency 'ftw', '~> 0.0.42'
33
34
  s.add_development_dependency 'logstash-input-generator'
34
35
 
metadata CHANGED
@@ -1,19 +1,19 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-amazon_es
3
3
  version: !ruby/object:Gem::Version
4
- version: '1.0'
4
+ version: 1.1.0
5
5
  platform: java
6
6
  authors:
7
7
  - Amazon
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-06-16 00:00:00.000000000 Z
11
+ date: 2017-01-11 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
15
15
  requirements:
16
- - - '>='
16
+ - - ">="
17
17
  - !ruby/object:Gem::Version
18
18
  version: '0'
19
19
  name: concurrent-ruby
@@ -21,13 +21,16 @@ dependencies:
21
21
  type: :runtime
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - '>='
24
+ - - ">="
25
25
  - !ruby/object:Gem::Version
26
26
  version: '0'
27
27
  - !ruby/object:Gem::Dependency
28
28
  requirement: !ruby/object:Gem::Requirement
29
29
  requirements:
30
- - - '>='
30
+ - - "~>"
31
+ - !ruby/object:Gem::Version
32
+ version: '1.0'
33
+ - - ">="
31
34
  - !ruby/object:Gem::Version
32
35
  version: 1.0.10
33
36
  name: elasticsearch
@@ -35,16 +38,19 @@ dependencies:
35
38
  type: :runtime
36
39
  version_requirements: !ruby/object:Gem::Requirement
37
40
  requirements:
38
- - - '>='
41
+ - - "~>"
42
+ - !ruby/object:Gem::Version
43
+ version: '1.0'
44
+ - - ">="
39
45
  - !ruby/object:Gem::Version
40
46
  version: 1.0.10
41
47
  - !ruby/object:Gem::Dependency
42
48
  requirement: !ruby/object:Gem::Requirement
43
49
  requirements:
44
- - - '>='
50
+ - - ">="
45
51
  - !ruby/object:Gem::Version
46
52
  version: 0.0.17
47
- - - ~>
53
+ - - "~>"
48
54
  - !ruby/object:Gem::Version
49
55
  version: '0.0'
50
56
  name: stud
@@ -52,16 +58,16 @@ dependencies:
52
58
  type: :runtime
53
59
  version_requirements: !ruby/object:Gem::Requirement
54
60
  requirements:
55
- - - '>='
61
+ - - ">="
56
62
  - !ruby/object:Gem::Version
57
63
  version: 0.0.17
58
- - - ~>
64
+ - - "~>"
59
65
  - !ruby/object:Gem::Version
60
66
  version: '0.0'
61
67
  - !ruby/object:Gem::Dependency
62
68
  requirement: !ruby/object:Gem::Requirement
63
69
  requirements:
64
- - - ~>
70
+ - - "~>"
65
71
  - !ruby/object:Gem::Version
66
72
  version: '0.6'
67
73
  name: cabin
@@ -69,36 +75,36 @@ dependencies:
69
75
  type: :runtime
70
76
  version_requirements: !ruby/object:Gem::Requirement
71
77
  requirements:
72
- - - ~>
78
+ - - "~>"
73
79
  - !ruby/object:Gem::Version
74
80
  version: '0.6'
75
81
  - !ruby/object:Gem::Dependency
76
82
  requirement: !ruby/object:Gem::Requirement
77
83
  requirements:
78
- - - '>='
84
+ - - ">="
79
85
  - !ruby/object:Gem::Version
80
- version: 1.4.0
81
- - - <
86
+ version: '1.60'
87
+ - - "<="
82
88
  - !ruby/object:Gem::Version
83
- version: 3.0.0
84
- name: logstash-core
89
+ version: '2.99'
90
+ name: logstash-core-plugin-api
85
91
  prerelease: false
86
92
  type: :runtime
87
93
  version_requirements: !ruby/object:Gem::Requirement
88
94
  requirements:
89
- - - '>='
95
+ - - ">="
90
96
  - !ruby/object:Gem::Version
91
- version: 1.4.0
92
- - - <
97
+ version: '1.60'
98
+ - - "<="
93
99
  - !ruby/object:Gem::Version
94
- version: 3.0.0
100
+ version: '2.99'
95
101
  - !ruby/object:Gem::Dependency
96
102
  requirement: !ruby/object:Gem::Requirement
97
103
  requirements:
98
- - - '>='
104
+ - - ">="
99
105
  - !ruby/object:Gem::Version
100
106
  version: 2.1.14
101
- - - ~>
107
+ - - "~>"
102
108
  - !ruby/object:Gem::Version
103
109
  version: '2'
104
110
  name: aws-sdk
@@ -106,16 +112,16 @@ dependencies:
106
112
  type: :runtime
107
113
  version_requirements: !ruby/object:Gem::Requirement
108
114
  requirements:
109
- - - '>='
115
+ - - ">="
110
116
  - !ruby/object:Gem::Version
111
117
  version: 2.1.14
112
- - - ~>
118
+ - - "~>"
113
119
  - !ruby/object:Gem::Version
114
120
  version: '2'
115
121
  - !ruby/object:Gem::Dependency
116
122
  requirement: !ruby/object:Gem::Requirement
117
123
  requirements:
118
- - - ~>
124
+ - - "~>"
119
125
  - !ruby/object:Gem::Version
120
126
  version: 0.9.1
121
127
  name: faraday
@@ -123,13 +129,13 @@ dependencies:
123
129
  type: :runtime
124
130
  version_requirements: !ruby/object:Gem::Requirement
125
131
  requirements:
126
- - - ~>
132
+ - - "~>"
127
133
  - !ruby/object:Gem::Version
128
134
  version: 0.9.1
129
135
  - !ruby/object:Gem::Dependency
130
136
  requirement: !ruby/object:Gem::Requirement
131
137
  requirements:
132
- - - ~>
138
+ - - "~>"
133
139
  - !ruby/object:Gem::Version
134
140
  version: 0.10.0
135
141
  name: faraday_middleware
@@ -137,13 +143,27 @@ dependencies:
137
143
  type: :runtime
138
144
  version_requirements: !ruby/object:Gem::Requirement
139
145
  requirements:
140
- - - ~>
146
+ - - "~>"
141
147
  - !ruby/object:Gem::Version
142
148
  version: 0.10.0
143
149
  - !ruby/object:Gem::Dependency
144
150
  requirement: !ruby/object:Gem::Requirement
145
151
  requirements:
146
- - - ~>
152
+ - - "<"
153
+ - !ruby/object:Gem::Version
154
+ version: 2.5.0
155
+ name: addressable
156
+ prerelease: false
157
+ type: :development
158
+ version_requirements: !ruby/object:Gem::Requirement
159
+ requirements:
160
+ - - "<"
161
+ - !ruby/object:Gem::Version
162
+ version: 2.5.0
163
+ - !ruby/object:Gem::Dependency
164
+ requirement: !ruby/object:Gem::Requirement
165
+ requirements:
166
+ - - "~>"
147
167
  - !ruby/object:Gem::Version
148
168
  version: 0.0.42
149
169
  name: ftw
@@ -151,13 +171,13 @@ dependencies:
151
171
  type: :development
152
172
  version_requirements: !ruby/object:Gem::Requirement
153
173
  requirements:
154
- - - ~>
174
+ - - "~>"
155
175
  - !ruby/object:Gem::Version
156
176
  version: 0.0.42
157
177
  - !ruby/object:Gem::Dependency
158
178
  requirement: !ruby/object:Gem::Requirement
159
179
  requirements:
160
- - - '>='
180
+ - - ">="
161
181
  - !ruby/object:Gem::Version
162
182
  version: '0'
163
183
  name: logstash-input-generator
@@ -165,16 +185,16 @@ dependencies:
165
185
  type: :development
166
186
  version_requirements: !ruby/object:Gem::Requirement
167
187
  requirements:
168
- - - '>='
188
+ - - ">="
169
189
  - !ruby/object:Gem::Version
170
190
  version: '0'
171
191
  - !ruby/object:Gem::Dependency
172
192
  requirement: !ruby/object:Gem::Requirement
173
193
  requirements:
174
- - - '>='
194
+ - - ">="
175
195
  - !ruby/object:Gem::Version
176
196
  version: 0.5.2
177
- - - <
197
+ - - "<"
178
198
  - !ruby/object:Gem::Version
179
199
  version: 1.0.0
180
200
  name: manticore
@@ -182,16 +202,16 @@ dependencies:
182
202
  type: :runtime
183
203
  version_requirements: !ruby/object:Gem::Requirement
184
204
  requirements:
185
- - - '>='
205
+ - - ">="
186
206
  - !ruby/object:Gem::Version
187
207
  version: 0.5.2
188
- - - <
208
+ - - "<"
189
209
  - !ruby/object:Gem::Version
190
210
  version: 1.0.0
191
211
  - !ruby/object:Gem::Dependency
192
212
  requirement: !ruby/object:Gem::Requirement
193
213
  requirements:
194
- - - '>='
214
+ - - ">="
195
215
  - !ruby/object:Gem::Version
196
216
  version: '0'
197
217
  name: logstash-devutils
@@ -199,13 +219,13 @@ dependencies:
199
219
  type: :development
200
220
  version_requirements: !ruby/object:Gem::Requirement
201
221
  requirements:
202
- - - '>='
222
+ - - ">="
203
223
  - !ruby/object:Gem::Version
204
224
  version: '0'
205
225
  - !ruby/object:Gem::Dependency
206
226
  requirement: !ruby/object:Gem::Requirement
207
227
  requirements:
208
- - - '>='
228
+ - - ">="
209
229
  - !ruby/object:Gem::Version
210
230
  version: '0'
211
231
  name: longshoreman
@@ -213,7 +233,7 @@ dependencies:
213
233
  type: :development
214
234
  version_requirements: !ruby/object:Gem::Requirement
215
235
  requirements:
216
- - - '>='
236
+ - - ">="
217
237
  - !ruby/object:Gem::Version
218
238
  version: '0'
219
239
  description: Output events to Amazon Elasticsearch Service with V4 signing
@@ -249,17 +269,17 @@ require_paths:
249
269
  - lib
250
270
  required_ruby_version: !ruby/object:Gem::Requirement
251
271
  requirements:
252
- - - '>='
272
+ - - ">="
253
273
  - !ruby/object:Gem::Version
254
274
  version: '0'
255
275
  required_rubygems_version: !ruby/object:Gem::Requirement
256
276
  requirements:
257
- - - '>='
277
+ - - ">="
258
278
  - !ruby/object:Gem::Version
259
279
  version: '0'
260
280
  requirements: []
261
281
  rubyforge_project:
262
- rubygems_version: 2.4.8
282
+ rubygems_version: 2.6.8
263
283
  signing_key:
264
284
  specification_version: 4
265
285
  summary: Logstash Output to Amazon Elasticsearch Service