logstash-output-amazon_es 1.0 → 1.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (7) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1 -0
  3. data/lib/logstash/outputs/amazon_es.rb +4 -4
  4. data/lib/logstash/outputs/amazon_es/aws_transport.rb +9 -3
  5. data/lib/logstash/outputs/amazon_es/http_client.rb +5 -4
  6. data/logstash-output-amazon_es.gemspec +5 -4
  7. metadata +66 -46
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: fbcd7505af8bdb4cf52bc2479b6f4e8d710925e8
4
- data.tar.gz: 03c45c246fb13c690c1e2bf72c223c208c0aae26
3
+ metadata.gz: 0c5f98a6e4c8fcd85d45b66ad156290bb73e6f7f
4
+ data.tar.gz: f7244a69869f93394924f217cce12daa5d3bb91f
5
5
  SHA512:
6
- metadata.gz: 9be8ee882ebbce45ec093737f60664875323ecc87535df4b9ff5971b3e23c01e9dffd9dcac1cce37a1ff010514aa5f05b0e2ad7c1bfb69961d51b5bbb25e6e00
7
- data.tar.gz: dd680f48ec1c8e1e4dbf38c8d7947e3525e70d55e4912b4ff0426ca0828c4d0f4db7ba8f8a4198341413ef346097009b3d28d43bd5617a5fc721c6caff85b3a3
6
+ metadata.gz: 72ce932d32910a609a80fbb13479dd190a812cc642951f0eedbf011a13bf0176acffb0416adf357df8880ad2c335e4566f21e673db6fcc619ff99c33635971b5
7
+ data.tar.gz: 0ae5201cf31bf696513332671646c5952f52e674c8d39731a9bcd2438f4d38533a9073639b57e0a652b78481ae4c6170c9a2067d6065fb413fdab11677d13155
data/README.md CHANGED
@@ -16,6 +16,7 @@ if above does not work, or you would like to patch code here is a workaround to
16
16
  1. Check out/clone this code from github
17
17
  2. Build plugin using - `gem build logstash-output-amazon_es.gemspec` ( this works with jruby and rubygem versions > 1.9)
18
18
  3. Install plugin using `<logstash-home>/bin/plugin install logstash-output-amazon_es-0.2.0-java.gem` (or the non java variant)
19
+ 4. For the 2.3 version , please use '<logstash-home>/bin/logstash-plugin install logstash-output-amazon_es-1.0-java.gem'
19
20
 
20
21
  ## Configuration for Amazon Elasticsearch Output plugin
21
22
 
data/lib/logstash/outputs/amazon_es.rb CHANGED
@@ -288,7 +288,7 @@ class LogStash::Outputs::AmazonES < LogStash::Outputs::Base
288
288
  @retry_queue_not_full.wait(@retry_flush_mutex) while @retry_queue.size > @retry_max_items
289
289
  }
290
290
 
291
- event['@metadata']['retry_count'] = 0
291
+ event.set('[@metadata][retry_count]', 0)
292
292
 
293
293
  # Set the 'type' value for the index.
294
294
  type = if @document_type
@@ -296,7 +296,7 @@ class LogStash::Outputs::AmazonES < LogStash::Outputs::Base
296
296
  elsif @index_type # deprecated
297
297
  event.sprintf(@index_type)
298
298
  else
299
- event["type"] || "logs"
299
+ event.get('type') || 'logs'
300
300
  end
301
301
 
302
302
  params = {
@@ -438,9 +438,9 @@ class LogStash::Outputs::AmazonES < LogStash::Outputs::Base
438
438
  unless @retry_queue.empty?
439
439
  buffer = @retry_queue.size.times.map do
440
440
  next_action, next_doc, next_event = @retry_queue.pop
441
- next_event['@metadata']['retry_count'] += 1
441
+ next_event.set('[@metadata][retry_count]', next_event.get('[@metadata][retry_count]') + 1)
442
442
 
443
- if next_event['@metadata']['retry_count'] > @max_retries
443
+ if next_event.get('[@metadata][retry_count]') > @max_retries
444
444
  @logger.error "too many attempts at sending event. dropping: #{next_event}"
445
445
  nil
446
446
  else
data/lib/logstash/outputs/amazon_es/aws_transport.rb CHANGED
@@ -33,7 +33,10 @@ module Elasticsearch
33
33
  :access_key_id,
34
34
  :secret_access_key,
35
35
  :session_token,
36
- :profile
36
+ :profile,
37
+ :instance_profile_credentials_retries,
38
+ :instance_profile_credentials_timeout,
39
+ :region
37
40
  )
38
41
 
39
42
  # Performs the request by invoking {Transport::Base#perform_request} with a block.
@@ -63,8 +66,11 @@ module Elasticsearch
63
66
  secret_access_key = options[:aws_secret_access_key] || nil
64
67
  session_token = options[:session_token] || nil
65
68
  profile = options[:profile] || 'default'
69
+ instance_cred_retries = options[:instance_profile_credentials_retries] || 0
70
+ instance_cred_timeout = options[:instance_profile_credentials_timeout] || 1
66
71
 
67
- credential_config = CredentialConfig.new(access_key_id, secret_access_key, session_token, profile)
72
+ credential_config = CredentialConfig.new(access_key_id, secret_access_key, session_token, profile,
73
+ instance_cred_retries, instance_cred_timeout, region)
68
74
  credentials = Aws::CredentialProviderChain.new(credential_config).resolve
69
75
 
70
76
  Connections::Collection.new \
@@ -100,4 +106,4 @@ module Elasticsearch
100
106
  end
101
107
  end
102
108
  end
103
- end
109
+ end
data/lib/logstash/outputs/amazon_es/http_client.rb CHANGED
@@ -69,21 +69,22 @@ module LogStash::Outputs::AES
69
69
  @client_options = {
70
70
  :hosts => uris,
71
71
  :region => options[:region],
72
- :aws_access_key_id => options[:aws_access_key_id],
73
- :aws_secret_access_key => options[:aws_secret_access_key],
74
72
  :transport_options => {
75
73
  :request => {:open_timeout => 0, :timeout => 60}, # ELB timeouts are set at 60
76
74
  :proxy => client_settings[:proxy],
77
75
  },
78
76
  :transport_class => Elasticsearch::Transport::Transport::HTTP::AWS
79
77
  }
78
+ internal_options = @client_options.clone
79
+ internal_options[:aws_access_key_id] = options[:aws_access_key_id]
80
+ internal_options[:aws_secret_access_key] = options[:aws_secret_access_key]
80
81
 
81
82
  if options[:user] && options[:password] then
82
83
  token = Base64.strict_encode64(options[:user] + ":" + options[:password])
83
- @client_options[:headers] = { "Authorization" => "Basic #{token}" }
84
+ internal_options[:headers] = { "Authorization" => "Basic #{token}" }
84
85
  end
85
86
 
86
- Elasticsearch::Client.new(client_options)
87
+ Elasticsearch::Client.new(internal_options)
87
88
  end
88
89
 
89
90
  def self.normalize_bulk_response(bulk_response)
data/logstash-output-amazon_es.gemspec CHANGED
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-output-amazon_es'
4
- s.version = '1.0'
4
+ s.version = '1.1.0'
5
5
  s.licenses = ['apache-2.0']
6
6
  s.summary = "Logstash Output to Amazon Elasticsearch Service"
7
7
  s.description = "Output events to Amazon Elasticsearch Service with V4 signing"
@@ -21,14 +21,15 @@ Gem::Specification.new do |s|
21
21
 
22
22
  # Gem dependencies
23
23
  s.add_runtime_dependency 'concurrent-ruby'
24
- s.add_runtime_dependency 'elasticsearch', ['>= 1.0.10']
24
+ s.add_runtime_dependency 'elasticsearch', '~> 1.0', '>= 1.0.10'
25
25
  s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
26
26
  s.add_runtime_dependency 'cabin', ['~> 0.6']
27
- s.add_runtime_dependency "logstash-core", ">= 1.4.0", "< 3.0.0"
28
- s.add_runtime_dependency "aws-sdk", ['>= 2.1.14', '~> 2']
27
+ s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99'
28
+ s.add_runtime_dependency 'aws-sdk', '>= 2.1.14', '~> 2'
29
29
  s.add_runtime_dependency "faraday", '~> 0.9.1'
30
30
  s.add_runtime_dependency "faraday_middleware", '~> 0.10.0'
31
31
 
32
+ s.add_development_dependency 'addressable', '< 2.5.0'
32
33
  s.add_development_dependency 'ftw', '~> 0.0.42'
33
34
  s.add_development_dependency 'logstash-input-generator'
34
35
 
metadata CHANGED
@@ -1,199 +1,219 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-amazon_es
3
3
  version: !ruby/object:Gem::Version
4
- version: '1.0'
4
+ version: 1.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Amazon
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-06-16 00:00:00.000000000 Z
11
+ date: 2017-01-11 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: concurrent-ruby
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - ">="
17
+ - - '>='
18
18
  - !ruby/object:Gem::Version
19
19
  version: '0'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - ">="
24
+ - - '>='
25
25
  - !ruby/object:Gem::Version
26
26
  version: '0'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: elasticsearch
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - ">="
31
+ - - ~>
32
+ - !ruby/object:Gem::Version
33
+ version: '1.0'
34
+ - - '>='
32
35
  - !ruby/object:Gem::Version
33
36
  version: 1.0.10
34
37
  type: :runtime
35
38
  prerelease: false
36
39
  version_requirements: !ruby/object:Gem::Requirement
37
40
  requirements:
38
- - - ">="
41
+ - - ~>
42
+ - !ruby/object:Gem::Version
43
+ version: '1.0'
44
+ - - '>='
39
45
  - !ruby/object:Gem::Version
40
46
  version: 1.0.10
41
47
  - !ruby/object:Gem::Dependency
42
48
  name: stud
43
49
  requirement: !ruby/object:Gem::Requirement
44
50
  requirements:
45
- - - ">="
51
+ - - '>='
46
52
  - !ruby/object:Gem::Version
47
53
  version: 0.0.17
48
- - - "~>"
54
+ - - ~>
49
55
  - !ruby/object:Gem::Version
50
56
  version: '0.0'
51
57
  type: :runtime
52
58
  prerelease: false
53
59
  version_requirements: !ruby/object:Gem::Requirement
54
60
  requirements:
55
- - - ">="
61
+ - - '>='
56
62
  - !ruby/object:Gem::Version
57
63
  version: 0.0.17
58
- - - "~>"
64
+ - - ~>
59
65
  - !ruby/object:Gem::Version
60
66
  version: '0.0'
61
67
  - !ruby/object:Gem::Dependency
62
68
  name: cabin
63
69
  requirement: !ruby/object:Gem::Requirement
64
70
  requirements:
65
- - - "~>"
71
+ - - ~>
66
72
  - !ruby/object:Gem::Version
67
73
  version: '0.6'
68
74
  type: :runtime
69
75
  prerelease: false
70
76
  version_requirements: !ruby/object:Gem::Requirement
71
77
  requirements:
72
- - - "~>"
78
+ - - ~>
73
79
  - !ruby/object:Gem::Version
74
80
  version: '0.6'
75
81
  - !ruby/object:Gem::Dependency
76
- name: logstash-core
82
+ name: logstash-core-plugin-api
77
83
  requirement: !ruby/object:Gem::Requirement
78
84
  requirements:
79
- - - ">="
85
+ - - '>='
80
86
  - !ruby/object:Gem::Version
81
- version: 1.4.0
82
- - - "<"
87
+ version: '1.60'
88
+ - - <=
83
89
  - !ruby/object:Gem::Version
84
- version: 3.0.0
90
+ version: '2.99'
85
91
  type: :runtime
86
92
  prerelease: false
87
93
  version_requirements: !ruby/object:Gem::Requirement
88
94
  requirements:
89
- - - ">="
95
+ - - '>='
90
96
  - !ruby/object:Gem::Version
91
- version: 1.4.0
92
- - - "<"
97
+ version: '1.60'
98
+ - - <=
93
99
  - !ruby/object:Gem::Version
94
- version: 3.0.0
100
+ version: '2.99'
95
101
  - !ruby/object:Gem::Dependency
96
102
  name: aws-sdk
97
103
  requirement: !ruby/object:Gem::Requirement
98
104
  requirements:
99
- - - ">="
105
+ - - '>='
100
106
  - !ruby/object:Gem::Version
101
107
  version: 2.1.14
102
- - - "~>"
108
+ - - ~>
103
109
  - !ruby/object:Gem::Version
104
110
  version: '2'
105
111
  type: :runtime
106
112
  prerelease: false
107
113
  version_requirements: !ruby/object:Gem::Requirement
108
114
  requirements:
109
- - - ">="
115
+ - - '>='
110
116
  - !ruby/object:Gem::Version
111
117
  version: 2.1.14
112
- - - "~>"
118
+ - - ~>
113
119
  - !ruby/object:Gem::Version
114
120
  version: '2'
115
121
  - !ruby/object:Gem::Dependency
116
122
  name: faraday
117
123
  requirement: !ruby/object:Gem::Requirement
118
124
  requirements:
119
- - - "~>"
125
+ - - ~>
120
126
  - !ruby/object:Gem::Version
121
127
  version: 0.9.1
122
128
  type: :runtime
123
129
  prerelease: false
124
130
  version_requirements: !ruby/object:Gem::Requirement
125
131
  requirements:
126
- - - "~>"
132
+ - - ~>
127
133
  - !ruby/object:Gem::Version
128
134
  version: 0.9.1
129
135
  - !ruby/object:Gem::Dependency
130
136
  name: faraday_middleware
131
137
  requirement: !ruby/object:Gem::Requirement
132
138
  requirements:
133
- - - "~>"
139
+ - - ~>
134
140
  - !ruby/object:Gem::Version
135
141
  version: 0.10.0
136
142
  type: :runtime
137
143
  prerelease: false
138
144
  version_requirements: !ruby/object:Gem::Requirement
139
145
  requirements:
140
- - - "~>"
146
+ - - ~>
141
147
  - !ruby/object:Gem::Version
142
148
  version: 0.10.0
149
+ - !ruby/object:Gem::Dependency
150
+ name: addressable
151
+ requirement: !ruby/object:Gem::Requirement
152
+ requirements:
153
+ - - <
154
+ - !ruby/object:Gem::Version
155
+ version: 2.5.0
156
+ type: :development
157
+ prerelease: false
158
+ version_requirements: !ruby/object:Gem::Requirement
159
+ requirements:
160
+ - - <
161
+ - !ruby/object:Gem::Version
162
+ version: 2.5.0
143
163
  - !ruby/object:Gem::Dependency
144
164
  name: ftw
145
165
  requirement: !ruby/object:Gem::Requirement
146
166
  requirements:
147
- - - "~>"
167
+ - - ~>
148
168
  - !ruby/object:Gem::Version
149
169
  version: 0.0.42
150
170
  type: :development
151
171
  prerelease: false
152
172
  version_requirements: !ruby/object:Gem::Requirement
153
173
  requirements:
154
- - - "~>"
174
+ - - ~>
155
175
  - !ruby/object:Gem::Version
156
176
  version: 0.0.42
157
177
  - !ruby/object:Gem::Dependency
158
178
  name: logstash-input-generator
159
179
  requirement: !ruby/object:Gem::Requirement
160
180
  requirements:
161
- - - ">="
181
+ - - '>='
162
182
  - !ruby/object:Gem::Version
163
183
  version: '0'
164
184
  type: :development
165
185
  prerelease: false
166
186
  version_requirements: !ruby/object:Gem::Requirement
167
187
  requirements:
168
- - - ">="
188
+ - - '>='
169
189
  - !ruby/object:Gem::Version
170
190
  version: '0'
171
191
  - !ruby/object:Gem::Dependency
172
192
  name: logstash-devutils
173
193
  requirement: !ruby/object:Gem::Requirement
174
194
  requirements:
175
- - - ">="
195
+ - - '>='
176
196
  - !ruby/object:Gem::Version
177
197
  version: '0'
178
198
  type: :development
179
199
  prerelease: false
180
200
  version_requirements: !ruby/object:Gem::Requirement
181
201
  requirements:
182
- - - ">="
202
+ - - '>='
183
203
  - !ruby/object:Gem::Version
184
204
  version: '0'
185
205
  - !ruby/object:Gem::Dependency
186
206
  name: longshoreman
187
207
  requirement: !ruby/object:Gem::Requirement
188
208
  requirements:
189
- - - ">="
209
+ - - '>='
190
210
  - !ruby/object:Gem::Version
191
211
  version: '0'
192
212
  type: :development
193
213
  prerelease: false
194
214
  version_requirements: !ruby/object:Gem::Requirement
195
215
  requirements:
196
- - - ">="
216
+ - - '>='
197
217
  - !ruby/object:Gem::Version
198
218
  version: '0'
199
219
  description: Output events to Amazon Elasticsearch Service with V4 signing
@@ -202,21 +222,21 @@ executables: []
202
222
  extensions: []
203
223
  extra_rdoc_files: []
204
224
  files:
205
- - Gemfile
206
- - LICENSE
207
- - NOTICE.TXT
208
- - README.md
209
- - lib/logstash/outputs/amazon_es.rb
210
225
  - lib/logstash/outputs/amazon_es/aws_transport.rb
211
226
  - lib/logstash/outputs/amazon_es/aws_v4_signer.rb
212
227
  - lib/logstash/outputs/amazon_es/aws_v4_signer_impl.rb
213
228
  - lib/logstash/outputs/amazon_es/elasticsearch-template.json
214
229
  - lib/logstash/outputs/amazon_es/http_client.rb
215
- - logstash-output-amazon_es.gemspec
230
+ - lib/logstash/outputs/amazon_es.rb
216
231
  - spec/amazon_es_spec_helper.rb
217
232
  - spec/unit/outputs/amazon_es_spec.rb
218
233
  - spec/unit/outputs/elasticsearch/protocol_spec.rb
219
234
  - spec/unit/outputs/elasticsearch_proxy_spec.rb
235
+ - logstash-output-amazon_es.gemspec
236
+ - README.md
237
+ - Gemfile
238
+ - LICENSE
239
+ - NOTICE.TXT
220
240
  homepage: http://logstash.net/
221
241
  licenses:
222
242
  - apache-2.0
@@ -229,17 +249,17 @@ require_paths:
229
249
  - lib
230
250
  required_ruby_version: !ruby/object:Gem::Requirement
231
251
  requirements:
232
- - - ">="
252
+ - - '>='
233
253
  - !ruby/object:Gem::Version
234
254
  version: '0'
235
255
  required_rubygems_version: !ruby/object:Gem::Requirement
236
256
  requirements:
237
- - - ">="
257
+ - - '>='
238
258
  - !ruby/object:Gem::Version
239
259
  version: '0'
240
260
  requirements: []
241
261
  rubyforge_project:
242
- rubygems_version: 2.4.6
262
+ rubygems_version: 2.0.14.1
243
263
  signing_key:
244
264
  specification_version: 4
245
265
  summary: Logstash Output to Amazon Elasticsearch Service