logstash-output-amazon_es 1.0-java → 1.1.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1 -0
  3. data/lib/logstash/outputs/amazon_es.rb +4 -4
  4. data/lib/logstash/outputs/amazon_es/aws_transport.rb +9 -3
  5. data/lib/logstash/outputs/amazon_es/http_client.rb +5 -4
  6. data/logstash-output-amazon_es.gemspec +5 -4
  7. metadata +64 -44
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 9a5aa1eae9dcf3ff783b7eae6a03af1d6b1f1fd6
4
- data.tar.gz: e8628f198b297023f200c52ec2ce12f9d7dc825c
3
+ metadata.gz: 286292688e20f9f6f1760d64ca008b7efcf40051
4
+ data.tar.gz: 151fa5283bf52d5a7f32cd97aa204590fd30ecb6
5
5
  SHA512:
6
- metadata.gz: 3124f478af6545161580d741646350d40952a8a2f1af2ba6ebd4ff7ed2c20614442bfca6e3ad3ca23743af189dbf1d86a9a05da4e290fb2202a8eab085540e87
7
- data.tar.gz: 3cbbfb9af493f00ffa84cc0fc6960be2fa55dec1ab36d4181ca276097091604752e2509dff2f9c844b247d19e52e6ad0fa23a474a457249be1aacdd40c3e0986
6
+ metadata.gz: 8d078f26da791eaaa6a332ce5d56d2a6df2361850f8574bee45a1a897fe93693a78a0ceee40c114693eab095e54bd2bd319b4f01e9e3e87ff00ed6b9c499912d
7
+ data.tar.gz: b0af0146d4e514be49041b2e5b0b1f3764647652924d0a5f44ac9e118d4d73dc86aed838844def36d50529dd7def73a23c030786480f7c3636bb47060f8610fb
data/README.md CHANGED
@@ -16,6 +16,7 @@ if above does not work, or you would like to patch code here is a workaround to
16
16
  1. Check out/clone this code from github
17
17
  2. Build plugin using - `gem build logstash-output-amazon_es.gemspec` ( this works with jruby and rubygem versions > 1.9)
18
18
  3. Install plugin using `<logstash-home>/bin/plugin install logstash-output-amazon_es-0.2.0-java.gem` (or the non java variant)
19
+ 4. For the 2.3 version , please use '<logstash-home>/bin/logstash-plugin install logstash-output-amazon_es-1.0-java.gem'
19
20
 
20
21
  ## Configuration for Amazon Elasticsearch Output plugin
21
22
 
data/lib/logstash/outputs/amazon_es.rb CHANGED
@@ -288,7 +288,7 @@ class LogStash::Outputs::AmazonES < LogStash::Outputs::Base
288
288
  @retry_queue_not_full.wait(@retry_flush_mutex) while @retry_queue.size > @retry_max_items
289
289
  }
290
290
 
291
- event['@metadata']['retry_count'] = 0
291
+ event.set('[@metadata][retry_count]', 0)
292
292
 
293
293
  # Set the 'type' value for the index.
294
294
  type = if @document_type
@@ -296,7 +296,7 @@ class LogStash::Outputs::AmazonES < LogStash::Outputs::Base
296
296
  elsif @index_type # deprecated
297
297
  event.sprintf(@index_type)
298
298
  else
299
- event["type"] || "logs"
299
+ event.get('type') || 'logs'
300
300
  end
301
301
 
302
302
  params = {
@@ -438,9 +438,9 @@ class LogStash::Outputs::AmazonES < LogStash::Outputs::Base
438
438
  unless @retry_queue.empty?
439
439
  buffer = @retry_queue.size.times.map do
440
440
  next_action, next_doc, next_event = @retry_queue.pop
441
- next_event['@metadata']['retry_count'] += 1
441
+ next_event.set('[@metadata][retry_count]', next_event.get('[@metadata][retry_count]') + 1)
442
442
 
443
- if next_event['@metadata']['retry_count'] > @max_retries
443
+ if next_event.get('[@metadata][retry_count]') > @max_retries
444
444
  @logger.error "too many attempts at sending event. dropping: #{next_event}"
445
445
  nil
446
446
  else
data/lib/logstash/outputs/amazon_es/aws_transport.rb CHANGED
@@ -33,7 +33,10 @@ module Elasticsearch
33
33
  :access_key_id,
34
34
  :secret_access_key,
35
35
  :session_token,
36
- :profile
36
+ :profile,
37
+ :instance_profile_credentials_retries,
38
+ :instance_profile_credentials_timeout,
39
+ :region
37
40
  )
38
41
 
39
42
  # Performs the request by invoking {Transport::Base#perform_request} with a block.
@@ -63,8 +66,11 @@ module Elasticsearch
63
66
  secret_access_key = options[:aws_secret_access_key] || nil
64
67
  session_token = options[:session_token] || nil
65
68
  profile = options[:profile] || 'default'
69
+ instance_cred_retries = options[:instance_profile_credentials_retries] || 0
70
+ instance_cred_timeout = options[:instance_profile_credentials_timeout] || 1
66
71
 
67
- credential_config = CredentialConfig.new(access_key_id, secret_access_key, session_token, profile)
72
+ credential_config = CredentialConfig.new(access_key_id, secret_access_key, session_token, profile,
73
+ instance_cred_retries, instance_cred_timeout, region)
68
74
  credentials = Aws::CredentialProviderChain.new(credential_config).resolve
69
75
 
70
76
  Connections::Collection.new \
@@ -100,4 +106,4 @@ module Elasticsearch
100
106
  end
101
107
  end
102
108
  end
103
- end
109
+ end
data/lib/logstash/outputs/amazon_es/http_client.rb CHANGED
@@ -69,21 +69,22 @@ module LogStash::Outputs::AES
69
69
  @client_options = {
70
70
  :hosts => uris,
71
71
  :region => options[:region],
72
- :aws_access_key_id => options[:aws_access_key_id],
73
- :aws_secret_access_key => options[:aws_secret_access_key],
74
72
  :transport_options => {
75
73
  :request => {:open_timeout => 0, :timeout => 60}, # ELB timeouts are set at 60
76
74
  :proxy => client_settings[:proxy],
77
75
  },
78
76
  :transport_class => Elasticsearch::Transport::Transport::HTTP::AWS
79
77
  }
78
+ internal_options = @client_options.clone
79
+ internal_options[:aws_access_key_id] = options[:aws_access_key_id]
80
+ internal_options[:aws_secret_access_key] = options[:aws_secret_access_key]
80
81
 
81
82
  if options[:user] && options[:password] then
82
83
  token = Base64.strict_encode64(options[:user] + ":" + options[:password])
83
- @client_options[:headers] = { "Authorization" => "Basic #{token}" }
84
+ internal_options[:headers] = { "Authorization" => "Basic #{token}" }
84
85
  end
85
86
 
86
- Elasticsearch::Client.new(client_options)
87
+ Elasticsearch::Client.new(internal_options)
87
88
  end
88
89
 
89
90
  def self.normalize_bulk_response(bulk_response)
data/logstash-output-amazon_es.gemspec CHANGED
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-output-amazon_es'
4
- s.version = '1.0'
4
+ s.version = '1.1.0'
5
5
  s.licenses = ['apache-2.0']
6
6
  s.summary = "Logstash Output to Amazon Elasticsearch Service"
7
7
  s.description = "Output events to Amazon Elasticsearch Service with V4 signing"
@@ -21,14 +21,15 @@ Gem::Specification.new do |s|
21
21
 
22
22
  # Gem dependencies
23
23
  s.add_runtime_dependency 'concurrent-ruby'
24
- s.add_runtime_dependency 'elasticsearch', ['>= 1.0.10']
24
+ s.add_runtime_dependency 'elasticsearch', '~> 1.0', '>= 1.0.10'
25
25
  s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
26
26
  s.add_runtime_dependency 'cabin', ['~> 0.6']
27
- s.add_runtime_dependency "logstash-core", ">= 1.4.0", "< 3.0.0"
28
- s.add_runtime_dependency "aws-sdk", ['>= 2.1.14', '~> 2']
27
+ s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99'
28
+ s.add_runtime_dependency 'aws-sdk', '>= 2.1.14', '~> 2'
29
29
  s.add_runtime_dependency "faraday", '~> 0.9.1'
30
30
  s.add_runtime_dependency "faraday_middleware", '~> 0.10.0'
31
31
 
32
+ s.add_development_dependency 'addressable', '< 2.5.0'
32
33
  s.add_development_dependency 'ftw', '~> 0.0.42'
33
34
  s.add_development_dependency 'logstash-input-generator'
34
35
 
metadata CHANGED
@@ -1,19 +1,19 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-amazon_es
3
3
  version: !ruby/object:Gem::Version
4
- version: '1.0'
4
+ version: 1.1.0
5
5
  platform: java
6
6
  authors:
7
7
  - Amazon
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-06-16 00:00:00.000000000 Z
11
+ date: 2017-01-11 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
15
15
  requirements:
16
- - - '>='
16
+ - - ">="
17
17
  - !ruby/object:Gem::Version
18
18
  version: '0'
19
19
  name: concurrent-ruby
@@ -21,13 +21,16 @@ dependencies:
21
21
  type: :runtime
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - '>='
24
+ - - ">="
25
25
  - !ruby/object:Gem::Version
26
26
  version: '0'
27
27
  - !ruby/object:Gem::Dependency
28
28
  requirement: !ruby/object:Gem::Requirement
29
29
  requirements:
30
- - - '>='
30
+ - - "~>"
31
+ - !ruby/object:Gem::Version
32
+ version: '1.0'
33
+ - - ">="
31
34
  - !ruby/object:Gem::Version
32
35
  version: 1.0.10
33
36
  name: elasticsearch
@@ -35,16 +38,19 @@ dependencies:
35
38
  type: :runtime
36
39
  version_requirements: !ruby/object:Gem::Requirement
37
40
  requirements:
38
- - - '>='
41
+ - - "~>"
42
+ - !ruby/object:Gem::Version
43
+ version: '1.0'
44
+ - - ">="
39
45
  - !ruby/object:Gem::Version
40
46
  version: 1.0.10
41
47
  - !ruby/object:Gem::Dependency
42
48
  requirement: !ruby/object:Gem::Requirement
43
49
  requirements:
44
- - - '>='
50
+ - - ">="
45
51
  - !ruby/object:Gem::Version
46
52
  version: 0.0.17
47
- - - ~>
53
+ - - "~>"
48
54
  - !ruby/object:Gem::Version
49
55
  version: '0.0'
50
56
  name: stud
@@ -52,16 +58,16 @@ dependencies:
52
58
  type: :runtime
53
59
  version_requirements: !ruby/object:Gem::Requirement
54
60
  requirements:
55
- - - '>='
61
+ - - ">="
56
62
  - !ruby/object:Gem::Version
57
63
  version: 0.0.17
58
- - - ~>
64
+ - - "~>"
59
65
  - !ruby/object:Gem::Version
60
66
  version: '0.0'
61
67
  - !ruby/object:Gem::Dependency
62
68
  requirement: !ruby/object:Gem::Requirement
63
69
  requirements:
64
- - - ~>
70
+ - - "~>"
65
71
  - !ruby/object:Gem::Version
66
72
  version: '0.6'
67
73
  name: cabin
@@ -69,36 +75,36 @@ dependencies:
69
75
  type: :runtime
70
76
  version_requirements: !ruby/object:Gem::Requirement
71
77
  requirements:
72
- - - ~>
78
+ - - "~>"
73
79
  - !ruby/object:Gem::Version
74
80
  version: '0.6'
75
81
  - !ruby/object:Gem::Dependency
76
82
  requirement: !ruby/object:Gem::Requirement
77
83
  requirements:
78
- - - '>='
84
+ - - ">="
79
85
  - !ruby/object:Gem::Version
80
- version: 1.4.0
81
- - - <
86
+ version: '1.60'
87
+ - - "<="
82
88
  - !ruby/object:Gem::Version
83
- version: 3.0.0
84
- name: logstash-core
89
+ version: '2.99'
90
+ name: logstash-core-plugin-api
85
91
  prerelease: false
86
92
  type: :runtime
87
93
  version_requirements: !ruby/object:Gem::Requirement
88
94
  requirements:
89
- - - '>='
95
+ - - ">="
90
96
  - !ruby/object:Gem::Version
91
- version: 1.4.0
92
- - - <
97
+ version: '1.60'
98
+ - - "<="
93
99
  - !ruby/object:Gem::Version
94
- version: 3.0.0
100
+ version: '2.99'
95
101
  - !ruby/object:Gem::Dependency
96
102
  requirement: !ruby/object:Gem::Requirement
97
103
  requirements:
98
- - - '>='
104
+ - - ">="
99
105
  - !ruby/object:Gem::Version
100
106
  version: 2.1.14
101
- - - ~>
107
+ - - "~>"
102
108
  - !ruby/object:Gem::Version
103
109
  version: '2'
104
110
  name: aws-sdk
@@ -106,16 +112,16 @@ dependencies:
106
112
  type: :runtime
107
113
  version_requirements: !ruby/object:Gem::Requirement
108
114
  requirements:
109
- - - '>='
115
+ - - ">="
110
116
  - !ruby/object:Gem::Version
111
117
  version: 2.1.14
112
- - - ~>
118
+ - - "~>"
113
119
  - !ruby/object:Gem::Version
114
120
  version: '2'
115
121
  - !ruby/object:Gem::Dependency
116
122
  requirement: !ruby/object:Gem::Requirement
117
123
  requirements:
118
- - - ~>
124
+ - - "~>"
119
125
  - !ruby/object:Gem::Version
120
126
  version: 0.9.1
121
127
  name: faraday
@@ -123,13 +129,13 @@ dependencies:
123
129
  type: :runtime
124
130
  version_requirements: !ruby/object:Gem::Requirement
125
131
  requirements:
126
- - - ~>
132
+ - - "~>"
127
133
  - !ruby/object:Gem::Version
128
134
  version: 0.9.1
129
135
  - !ruby/object:Gem::Dependency
130
136
  requirement: !ruby/object:Gem::Requirement
131
137
  requirements:
132
- - - ~>
138
+ - - "~>"
133
139
  - !ruby/object:Gem::Version
134
140
  version: 0.10.0
135
141
  name: faraday_middleware
@@ -137,13 +143,27 @@ dependencies:
137
143
  type: :runtime
138
144
  version_requirements: !ruby/object:Gem::Requirement
139
145
  requirements:
140
- - - ~>
146
+ - - "~>"
141
147
  - !ruby/object:Gem::Version
142
148
  version: 0.10.0
143
149
  - !ruby/object:Gem::Dependency
144
150
  requirement: !ruby/object:Gem::Requirement
145
151
  requirements:
146
- - - ~>
152
+ - - "<"
153
+ - !ruby/object:Gem::Version
154
+ version: 2.5.0
155
+ name: addressable
156
+ prerelease: false
157
+ type: :development
158
+ version_requirements: !ruby/object:Gem::Requirement
159
+ requirements:
160
+ - - "<"
161
+ - !ruby/object:Gem::Version
162
+ version: 2.5.0
163
+ - !ruby/object:Gem::Dependency
164
+ requirement: !ruby/object:Gem::Requirement
165
+ requirements:
166
+ - - "~>"
147
167
  - !ruby/object:Gem::Version
148
168
  version: 0.0.42
149
169
  name: ftw
@@ -151,13 +171,13 @@ dependencies:
151
171
  type: :development
152
172
  version_requirements: !ruby/object:Gem::Requirement
153
173
  requirements:
154
- - - ~>
174
+ - - "~>"
155
175
  - !ruby/object:Gem::Version
156
176
  version: 0.0.42
157
177
  - !ruby/object:Gem::Dependency
158
178
  requirement: !ruby/object:Gem::Requirement
159
179
  requirements:
160
- - - '>='
180
+ - - ">="
161
181
  - !ruby/object:Gem::Version
162
182
  version: '0'
163
183
  name: logstash-input-generator
@@ -165,16 +185,16 @@ dependencies:
165
185
  type: :development
166
186
  version_requirements: !ruby/object:Gem::Requirement
167
187
  requirements:
168
- - - '>='
188
+ - - ">="
169
189
  - !ruby/object:Gem::Version
170
190
  version: '0'
171
191
  - !ruby/object:Gem::Dependency
172
192
  requirement: !ruby/object:Gem::Requirement
173
193
  requirements:
174
- - - '>='
194
+ - - ">="
175
195
  - !ruby/object:Gem::Version
176
196
  version: 0.5.2
177
- - - <
197
+ - - "<"
178
198
  - !ruby/object:Gem::Version
179
199
  version: 1.0.0
180
200
  name: manticore
@@ -182,16 +202,16 @@ dependencies:
182
202
  type: :runtime
183
203
  version_requirements: !ruby/object:Gem::Requirement
184
204
  requirements:
185
- - - '>='
205
+ - - ">="
186
206
  - !ruby/object:Gem::Version
187
207
  version: 0.5.2
188
- - - <
208
+ - - "<"
189
209
  - !ruby/object:Gem::Version
190
210
  version: 1.0.0
191
211
  - !ruby/object:Gem::Dependency
192
212
  requirement: !ruby/object:Gem::Requirement
193
213
  requirements:
194
- - - '>='
214
+ - - ">="
195
215
  - !ruby/object:Gem::Version
196
216
  version: '0'
197
217
  name: logstash-devutils
@@ -199,13 +219,13 @@ dependencies:
199
219
  type: :development
200
220
  version_requirements: !ruby/object:Gem::Requirement
201
221
  requirements:
202
- - - '>='
222
+ - - ">="
203
223
  - !ruby/object:Gem::Version
204
224
  version: '0'
205
225
  - !ruby/object:Gem::Dependency
206
226
  requirement: !ruby/object:Gem::Requirement
207
227
  requirements:
208
- - - '>='
228
+ - - ">="
209
229
  - !ruby/object:Gem::Version
210
230
  version: '0'
211
231
  name: longshoreman
@@ -213,7 +233,7 @@ dependencies:
213
233
  type: :development
214
234
  version_requirements: !ruby/object:Gem::Requirement
215
235
  requirements:
216
- - - '>='
236
+ - - ">="
217
237
  - !ruby/object:Gem::Version
218
238
  version: '0'
219
239
  description: Output events to Amazon Elasticsearch Service with V4 signing
@@ -249,17 +269,17 @@ require_paths:
249
269
  - lib
250
270
  required_ruby_version: !ruby/object:Gem::Requirement
251
271
  requirements:
252
- - - '>='
272
+ - - ">="
253
273
  - !ruby/object:Gem::Version
254
274
  version: '0'
255
275
  required_rubygems_version: !ruby/object:Gem::Requirement
256
276
  requirements:
257
- - - '>='
277
+ - - ">="
258
278
  - !ruby/object:Gem::Version
259
279
  version: '0'
260
280
  requirements: []
261
281
  rubyforge_project:
262
- rubygems_version: 2.4.8
282
+ rubygems_version: 2.6.8
263
283
  signing_key:
264
284
  specification_version: 4
265
285
  summary: Logstash Output to Amazon Elasticsearch Service