logstash-output-amazon_es 1.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1 -0
  3. data/lib/logstash/outputs/amazon_es.rb +4 -4
  4. data/lib/logstash/outputs/amazon_es/aws_transport.rb +9 -3
  5. data/lib/logstash/outputs/amazon_es/http_client.rb +5 -4
  6. data/logstash-output-amazon_es.gemspec +5 -4
  7. metadata +66 -46
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: fbcd7505af8bdb4cf52bc2479b6f4e8d710925e8
4
- data.tar.gz: 03c45c246fb13c690c1e2bf72c223c208c0aae26
3
+ metadata.gz: 0c5f98a6e4c8fcd85d45b66ad156290bb73e6f7f
4
+ data.tar.gz: f7244a69869f93394924f217cce12daa5d3bb91f
5
5
  SHA512:
6
- metadata.gz: 9be8ee882ebbce45ec093737f60664875323ecc87535df4b9ff5971b3e23c01e9dffd9dcac1cce37a1ff010514aa5f05b0e2ad7c1bfb69961d51b5bbb25e6e00
7
- data.tar.gz: dd680f48ec1c8e1e4dbf38c8d7947e3525e70d55e4912b4ff0426ca0828c4d0f4db7ba8f8a4198341413ef346097009b3d28d43bd5617a5fc721c6caff85b3a3
6
+ metadata.gz: 72ce932d32910a609a80fbb13479dd190a812cc642951f0eedbf011a13bf0176acffb0416adf357df8880ad2c335e4566f21e673db6fcc619ff99c33635971b5
7
+ data.tar.gz: 0ae5201cf31bf696513332671646c5952f52e674c8d39731a9bcd2438f4d38533a9073639b57e0a652b78481ae4c6170c9a2067d6065fb413fdab11677d13155
data/README.md CHANGED
@@ -16,6 +16,7 @@ if above does not work, or you would like to patch code here is a workaround to
16
16
  1. Check out/clone this code from github
17
17
  2. Build plugin using - `gem build logstash-output-amazon_es.gemspec` ( this works with jruby and rubygem versions > 1.9)
18
18
  3. Install plugin using `<logstash-home>/bin/plugin install logstash-output-amazon_es-0.2.0-java.gem` (or the non java variant)
19
+ 4. For the 2.3 version , please use '<logstash-home>/bin/logstash-plugin install logstash-output-amazon_es-1.0-java.gem'
19
20
 
20
21
  ## Configuration for Amazon Elasticsearch Output plugin
21
22
 
data/lib/logstash/outputs/amazon_es.rb CHANGED
@@ -288,7 +288,7 @@ class LogStash::Outputs::AmazonES < LogStash::Outputs::Base
288
288
  @retry_queue_not_full.wait(@retry_flush_mutex) while @retry_queue.size > @retry_max_items
289
289
  }
290
290
 
291
- event['@metadata']['retry_count'] = 0
291
+ event.set('[@metadata][retry_count]', 0)
292
292
 
293
293
  # Set the 'type' value for the index.
294
294
  type = if @document_type
@@ -296,7 +296,7 @@ class LogStash::Outputs::AmazonES < LogStash::Outputs::Base
296
296
  elsif @index_type # deprecated
297
297
  event.sprintf(@index_type)
298
298
  else
299
- event["type"] || "logs"
299
+ event.get('type') || 'logs'
300
300
  end
301
301
 
302
302
  params = {
@@ -438,9 +438,9 @@ class LogStash::Outputs::AmazonES < LogStash::Outputs::Base
438
438
  unless @retry_queue.empty?
439
439
  buffer = @retry_queue.size.times.map do
440
440
  next_action, next_doc, next_event = @retry_queue.pop
441
- next_event['@metadata']['retry_count'] += 1
441
+ next_event.set('[@metadata][retry_count]', next_event.get('[@metadata][retry_count]') + 1)
442
442
 
443
- if next_event['@metadata']['retry_count'] > @max_retries
443
+ if next_event.get('[@metadata][retry_count]') > @max_retries
444
444
  @logger.error "too many attempts at sending event. dropping: #{next_event}"
445
445
  nil
446
446
  else
data/lib/logstash/outputs/amazon_es/aws_transport.rb CHANGED
@@ -33,7 +33,10 @@ module Elasticsearch
33
33
  :access_key_id,
34
34
  :secret_access_key,
35
35
  :session_token,
36
- :profile
36
+ :profile,
37
+ :instance_profile_credentials_retries,
38
+ :instance_profile_credentials_timeout,
39
+ :region
37
40
  )
38
41
 
39
42
  # Performs the request by invoking {Transport::Base#perform_request} with a block.
@@ -63,8 +66,11 @@ module Elasticsearch
63
66
  secret_access_key = options[:aws_secret_access_key] || nil
64
67
  session_token = options[:session_token] || nil
65
68
  profile = options[:profile] || 'default'
69
+ instance_cred_retries = options[:instance_profile_credentials_retries] || 0
70
+ instance_cred_timeout = options[:instance_profile_credentials_timeout] || 1
66
71
 
67
- credential_config = CredentialConfig.new(access_key_id, secret_access_key, session_token, profile)
72
+ credential_config = CredentialConfig.new(access_key_id, secret_access_key, session_token, profile,
73
+ instance_cred_retries, instance_cred_timeout, region)
68
74
  credentials = Aws::CredentialProviderChain.new(credential_config).resolve
69
75
 
70
76
  Connections::Collection.new \
@@ -100,4 +106,4 @@ module Elasticsearch
100
106
  end
101
107
  end
102
108
  end
103
- end
109
+ end
data/lib/logstash/outputs/amazon_es/http_client.rb CHANGED
@@ -69,21 +69,22 @@ module LogStash::Outputs::AES
69
69
  @client_options = {
70
70
  :hosts => uris,
71
71
  :region => options[:region],
72
- :aws_access_key_id => options[:aws_access_key_id],
73
- :aws_secret_access_key => options[:aws_secret_access_key],
74
72
  :transport_options => {
75
73
  :request => {:open_timeout => 0, :timeout => 60}, # ELB timeouts are set at 60
76
74
  :proxy => client_settings[:proxy],
77
75
  },
78
76
  :transport_class => Elasticsearch::Transport::Transport::HTTP::AWS
79
77
  }
78
+ internal_options = @client_options.clone
79
+ internal_options[:aws_access_key_id] = options[:aws_access_key_id]
80
+ internal_options[:aws_secret_access_key] = options[:aws_secret_access_key]
80
81
 
81
82
  if options[:user] && options[:password] then
82
83
  token = Base64.strict_encode64(options[:user] + ":" + options[:password])
83
- @client_options[:headers] = { "Authorization" => "Basic #{token}" }
84
+ internal_options[:headers] = { "Authorization" => "Basic #{token}" }
84
85
  end
85
86
 
86
- Elasticsearch::Client.new(client_options)
87
+ Elasticsearch::Client.new(internal_options)
87
88
  end
88
89
 
89
90
  def self.normalize_bulk_response(bulk_response)
data/logstash-output-amazon_es.gemspec CHANGED
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-output-amazon_es'
4
- s.version = '1.0'
4
+ s.version = '1.1.0'
5
5
  s.licenses = ['apache-2.0']
6
6
  s.summary = "Logstash Output to Amazon Elasticsearch Service"
7
7
  s.description = "Output events to Amazon Elasticsearch Service with V4 signing"
@@ -21,14 +21,15 @@ Gem::Specification.new do |s|
21
21
 
22
22
  # Gem dependencies
23
23
  s.add_runtime_dependency 'concurrent-ruby'
24
- s.add_runtime_dependency 'elasticsearch', ['>= 1.0.10']
24
+ s.add_runtime_dependency 'elasticsearch', '~> 1.0', '>= 1.0.10'
25
25
  s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
26
26
  s.add_runtime_dependency 'cabin', ['~> 0.6']
27
- s.add_runtime_dependency "logstash-core", ">= 1.4.0", "< 3.0.0"
28
- s.add_runtime_dependency "aws-sdk", ['>= 2.1.14', '~> 2']
27
+ s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99'
28
+ s.add_runtime_dependency 'aws-sdk', '>= 2.1.14', '~> 2'
29
29
  s.add_runtime_dependency "faraday", '~> 0.9.1'
30
30
  s.add_runtime_dependency "faraday_middleware", '~> 0.10.0'
31
31
 
32
+ s.add_development_dependency 'addressable', '< 2.5.0'
32
33
  s.add_development_dependency 'ftw', '~> 0.0.42'
33
34
  s.add_development_dependency 'logstash-input-generator'
34
35
 
metadata CHANGED
@@ -1,199 +1,219 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-amazon_es
3
3
  version: !ruby/object:Gem::Version
4
- version: '1.0'
4
+ version: 1.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Amazon
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-06-16 00:00:00.000000000 Z
11
+ date: 2017-01-11 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: concurrent-ruby
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - ">="
17
+ - - '>='
18
18
  - !ruby/object:Gem::Version
19
19
  version: '0'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - ">="
24
+ - - '>='
25
25
  - !ruby/object:Gem::Version
26
26
  version: '0'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: elasticsearch
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - ">="
31
+ - - ~>
32
+ - !ruby/object:Gem::Version
33
+ version: '1.0'
34
+ - - '>='
32
35
  - !ruby/object:Gem::Version
33
36
  version: 1.0.10
34
37
  type: :runtime
35
38
  prerelease: false
36
39
  version_requirements: !ruby/object:Gem::Requirement
37
40
  requirements:
38
- - - ">="
41
+ - - ~>
42
+ - !ruby/object:Gem::Version
43
+ version: '1.0'
44
+ - - '>='
39
45
  - !ruby/object:Gem::Version
40
46
  version: 1.0.10
41
47
  - !ruby/object:Gem::Dependency
42
48
  name: stud
43
49
  requirement: !ruby/object:Gem::Requirement
44
50
  requirements:
45
- - - ">="
51
+ - - '>='
46
52
  - !ruby/object:Gem::Version
47
53
  version: 0.0.17
48
- - - "~>"
54
+ - - ~>
49
55
  - !ruby/object:Gem::Version
50
56
  version: '0.0'
51
57
  type: :runtime
52
58
  prerelease: false
53
59
  version_requirements: !ruby/object:Gem::Requirement
54
60
  requirements:
55
- - - ">="
61
+ - - '>='
56
62
  - !ruby/object:Gem::Version
57
63
  version: 0.0.17
58
- - - "~>"
64
+ - - ~>
59
65
  - !ruby/object:Gem::Version
60
66
  version: '0.0'
61
67
  - !ruby/object:Gem::Dependency
62
68
  name: cabin
63
69
  requirement: !ruby/object:Gem::Requirement
64
70
  requirements:
65
- - - "~>"
71
+ - - ~>
66
72
  - !ruby/object:Gem::Version
67
73
  version: '0.6'
68
74
  type: :runtime
69
75
  prerelease: false
70
76
  version_requirements: !ruby/object:Gem::Requirement
71
77
  requirements:
72
- - - "~>"
78
+ - - ~>
73
79
  - !ruby/object:Gem::Version
74
80
  version: '0.6'
75
81
  - !ruby/object:Gem::Dependency
76
- name: logstash-core
82
+ name: logstash-core-plugin-api
77
83
  requirement: !ruby/object:Gem::Requirement
78
84
  requirements:
79
- - - ">="
85
+ - - '>='
80
86
  - !ruby/object:Gem::Version
81
- version: 1.4.0
82
- - - "<"
87
+ version: '1.60'
88
+ - - <=
83
89
  - !ruby/object:Gem::Version
84
- version: 3.0.0
90
+ version: '2.99'
85
91
  type: :runtime
86
92
  prerelease: false
87
93
  version_requirements: !ruby/object:Gem::Requirement
88
94
  requirements:
89
- - - ">="
95
+ - - '>='
90
96
  - !ruby/object:Gem::Version
91
- version: 1.4.0
92
- - - "<"
97
+ version: '1.60'
98
+ - - <=
93
99
  - !ruby/object:Gem::Version
94
- version: 3.0.0
100
+ version: '2.99'
95
101
  - !ruby/object:Gem::Dependency
96
102
  name: aws-sdk
97
103
  requirement: !ruby/object:Gem::Requirement
98
104
  requirements:
99
- - - ">="
105
+ - - '>='
100
106
  - !ruby/object:Gem::Version
101
107
  version: 2.1.14
102
- - - "~>"
108
+ - - ~>
103
109
  - !ruby/object:Gem::Version
104
110
  version: '2'
105
111
  type: :runtime
106
112
  prerelease: false
107
113
  version_requirements: !ruby/object:Gem::Requirement
108
114
  requirements:
109
- - - ">="
115
+ - - '>='
110
116
  - !ruby/object:Gem::Version
111
117
  version: 2.1.14
112
- - - "~>"
118
+ - - ~>
113
119
  - !ruby/object:Gem::Version
114
120
  version: '2'
115
121
  - !ruby/object:Gem::Dependency
116
122
  name: faraday
117
123
  requirement: !ruby/object:Gem::Requirement
118
124
  requirements:
119
- - - "~>"
125
+ - - ~>
120
126
  - !ruby/object:Gem::Version
121
127
  version: 0.9.1
122
128
  type: :runtime
123
129
  prerelease: false
124
130
  version_requirements: !ruby/object:Gem::Requirement
125
131
  requirements:
126
- - - "~>"
132
+ - - ~>
127
133
  - !ruby/object:Gem::Version
128
134
  version: 0.9.1
129
135
  - !ruby/object:Gem::Dependency
130
136
  name: faraday_middleware
131
137
  requirement: !ruby/object:Gem::Requirement
132
138
  requirements:
133
- - - "~>"
139
+ - - ~>
134
140
  - !ruby/object:Gem::Version
135
141
  version: 0.10.0
136
142
  type: :runtime
137
143
  prerelease: false
138
144
  version_requirements: !ruby/object:Gem::Requirement
139
145
  requirements:
140
- - - "~>"
146
+ - - ~>
141
147
  - !ruby/object:Gem::Version
142
148
  version: 0.10.0
149
+ - !ruby/object:Gem::Dependency
150
+ name: addressable
151
+ requirement: !ruby/object:Gem::Requirement
152
+ requirements:
153
+ - - <
154
+ - !ruby/object:Gem::Version
155
+ version: 2.5.0
156
+ type: :development
157
+ prerelease: false
158
+ version_requirements: !ruby/object:Gem::Requirement
159
+ requirements:
160
+ - - <
161
+ - !ruby/object:Gem::Version
162
+ version: 2.5.0
143
163
  - !ruby/object:Gem::Dependency
144
164
  name: ftw
145
165
  requirement: !ruby/object:Gem::Requirement
146
166
  requirements:
147
- - - "~>"
167
+ - - ~>
148
168
  - !ruby/object:Gem::Version
149
169
  version: 0.0.42
150
170
  type: :development
151
171
  prerelease: false
152
172
  version_requirements: !ruby/object:Gem::Requirement
153
173
  requirements:
154
- - - "~>"
174
+ - - ~>
155
175
  - !ruby/object:Gem::Version
156
176
  version: 0.0.42
157
177
  - !ruby/object:Gem::Dependency
158
178
  name: logstash-input-generator
159
179
  requirement: !ruby/object:Gem::Requirement
160
180
  requirements:
161
- - - ">="
181
+ - - '>='
162
182
  - !ruby/object:Gem::Version
163
183
  version: '0'
164
184
  type: :development
165
185
  prerelease: false
166
186
  version_requirements: !ruby/object:Gem::Requirement
167
187
  requirements:
168
- - - ">="
188
+ - - '>='
169
189
  - !ruby/object:Gem::Version
170
190
  version: '0'
171
191
  - !ruby/object:Gem::Dependency
172
192
  name: logstash-devutils
173
193
  requirement: !ruby/object:Gem::Requirement
174
194
  requirements:
175
- - - ">="
195
+ - - '>='
176
196
  - !ruby/object:Gem::Version
177
197
  version: '0'
178
198
  type: :development
179
199
  prerelease: false
180
200
  version_requirements: !ruby/object:Gem::Requirement
181
201
  requirements:
182
- - - ">="
202
+ - - '>='
183
203
  - !ruby/object:Gem::Version
184
204
  version: '0'
185
205
  - !ruby/object:Gem::Dependency
186
206
  name: longshoreman
187
207
  requirement: !ruby/object:Gem::Requirement
188
208
  requirements:
189
- - - ">="
209
+ - - '>='
190
210
  - !ruby/object:Gem::Version
191
211
  version: '0'
192
212
  type: :development
193
213
  prerelease: false
194
214
  version_requirements: !ruby/object:Gem::Requirement
195
215
  requirements:
196
- - - ">="
216
+ - - '>='
197
217
  - !ruby/object:Gem::Version
198
218
  version: '0'
199
219
  description: Output events to Amazon Elasticsearch Service with V4 signing
@@ -202,21 +222,21 @@ executables: []
202
222
  extensions: []
203
223
  extra_rdoc_files: []
204
224
  files:
205
- - Gemfile
206
- - LICENSE
207
- - NOTICE.TXT
208
- - README.md
209
- - lib/logstash/outputs/amazon_es.rb
210
225
  - lib/logstash/outputs/amazon_es/aws_transport.rb
211
226
  - lib/logstash/outputs/amazon_es/aws_v4_signer.rb
212
227
  - lib/logstash/outputs/amazon_es/aws_v4_signer_impl.rb
213
228
  - lib/logstash/outputs/amazon_es/elasticsearch-template.json
214
229
  - lib/logstash/outputs/amazon_es/http_client.rb
215
- - logstash-output-amazon_es.gemspec
230
+ - lib/logstash/outputs/amazon_es.rb
216
231
  - spec/amazon_es_spec_helper.rb
217
232
  - spec/unit/outputs/amazon_es_spec.rb
218
233
  - spec/unit/outputs/elasticsearch/protocol_spec.rb
219
234
  - spec/unit/outputs/elasticsearch_proxy_spec.rb
235
+ - logstash-output-amazon_es.gemspec
236
+ - README.md
237
+ - Gemfile
238
+ - LICENSE
239
+ - NOTICE.TXT
220
240
  homepage: http://logstash.net/
221
241
  licenses:
222
242
  - apache-2.0
@@ -229,17 +249,17 @@ require_paths:
229
249
  - lib
230
250
  required_ruby_version: !ruby/object:Gem::Requirement
231
251
  requirements:
232
- - - ">="
252
+ - - '>='
233
253
  - !ruby/object:Gem::Version
234
254
  version: '0'
235
255
  required_rubygems_version: !ruby/object:Gem::Requirement
236
256
  requirements:
237
- - - ">="
257
+ - - '>='
238
258
  - !ruby/object:Gem::Version
239
259
  version: '0'
240
260
  requirements: []
241
261
  rubyforge_project:
242
- rubygems_version: 2.4.6
262
+ rubygems_version: 2.0.14.1
243
263
  signing_key:
244
264
  specification_version: 4
245
265
  summary: Logstash Output to Amazon Elasticsearch Service