logstash-input-s3 3.3.0 → 3.3.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/lib/logstash/inputs/s3.rb +19 -13
- data/logstash-input-s3.gemspec +1 -1
- data/spec/inputs/s3_spec.rb +56 -0
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 40d44b6d4f759ae776a74c6d2e316ec74f62dba65dc2d75702749913471feda8
|
4
|
+
data.tar.gz: cc518e42d35b2ef3c39045f9700a7f970bccb0e3cf145d7cebb0dd51860bcd13
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 450923e8e0d886fe972ea82365a7b38eda37cef594f79fa3839615ab64c3fbcdad6fa12d724a80e05c57136126168c16ada1d5587e0e3152f04540846a145bdb
|
7
|
+
data.tar.gz: 70c2526ec6b8acf37d3e82af2a30bed30864e41aae8182cc3f336d68623604c3a1b3380118f4ec6cfecc87e317304fb961d198b9383d70d5f74b5d3c7513e894
|
data/CHANGELOG.md
CHANGED
data/lib/logstash/inputs/s3.rb
CHANGED
@@ -110,18 +110,21 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
|
|
110
110
|
def list_new_files
|
111
111
|
objects = {}
|
112
112
|
|
113
|
-
|
114
|
-
@
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
113
|
+
begin
|
114
|
+
@s3bucket.objects(:prefix => @prefix).each do |log|
|
115
|
+
@logger.debug("S3 input: Found key", :key => log.key)
|
116
|
+
unless ignore_filename?(log.key)
|
117
|
+
if sincedb.newer?(log.last_modified) && log.content_length > 0
|
118
|
+
objects[log.key] = log.last_modified
|
119
|
+
@logger.debug("S3 input: Adding to objects[]", :key => log.key)
|
120
|
+
@logger.debug("objects[] length is: ", :length => objects.length)
|
121
|
+
end
|
121
122
|
end
|
122
123
|
end
|
124
|
+
rescue Aws::Errors::ServiceError => e
|
125
|
+
@logger.error("S3 input: Unable to list objects in bucket", :prefix => prefix, :message => e.message)
|
123
126
|
end
|
124
|
-
|
127
|
+
objects.keys.sort {|a,b| objects[a] <=> objects[b]}
|
125
128
|
end # def fetch_new_files
|
126
129
|
|
127
130
|
public
|
@@ -372,11 +375,14 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
|
|
372
375
|
@logger.debug("S3 input: Download remote file", :remote_key => remote_object.key, :local_filename => local_filename)
|
373
376
|
File.open(local_filename, 'wb') do |s3file|
|
374
377
|
return completed if stop?
|
375
|
-
|
378
|
+
begin
|
379
|
+
remote_object.get(:response_target => s3file)
|
380
|
+
completed = true
|
381
|
+
rescue Aws::Errors::ServiceError => e
|
382
|
+
@logger.warn("S3 input: Unable to download remote file", :remote_key => remote_object.key, :message => e.message)
|
383
|
+
end
|
376
384
|
end
|
377
|
-
completed
|
378
|
-
|
379
|
-
return completed
|
385
|
+
completed
|
380
386
|
end
|
381
387
|
|
382
388
|
private
|
data/logstash-input-s3.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-input-s3'
|
4
|
-
s.version = '3.3.
|
4
|
+
s.version = '3.3.1'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Streams events from files in a S3 bucket"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
data/spec/inputs/s3_spec.rb
CHANGED
@@ -256,6 +256,62 @@ describe LogStash::Inputs::S3 do
|
|
256
256
|
end
|
257
257
|
end
|
258
258
|
|
259
|
+
context 'while communicating with s3' do
|
260
|
+
let(:config) {
|
261
|
+
{
|
262
|
+
"access_key_id" => "1234",
|
263
|
+
"secret_access_key" => "secret",
|
264
|
+
"bucket" => "logstash-test",
|
265
|
+
"codec" => "json",
|
266
|
+
}
|
267
|
+
}
|
268
|
+
%w(AccessDenied NotFound).each do |error|
|
269
|
+
context "while listing bucket contents, #{error} is returned" do
|
270
|
+
before do
|
271
|
+
Aws.config[:s3] = {
|
272
|
+
stub_responses: {
|
273
|
+
list_objects: error
|
274
|
+
}
|
275
|
+
}
|
276
|
+
end
|
277
|
+
|
278
|
+
it 'should not crash the plugin' do
|
279
|
+
events = fetch_events(config)
|
280
|
+
expect(events.size).to eq(0)
|
281
|
+
end
|
282
|
+
end
|
283
|
+
end
|
284
|
+
|
285
|
+
%w(AccessDenied NoSuchKey).each do |error|
|
286
|
+
context "when retrieving an object, #{error} is returned" do
|
287
|
+
let(:objects) { [log] }
|
288
|
+
let(:log) { double(:key => 'uncompressed.log', :last_modified => Time.now - 2 * day, :content_length => 5) }
|
289
|
+
|
290
|
+
let(:config) {
|
291
|
+
{
|
292
|
+
"access_key_id" => "1234",
|
293
|
+
"secret_access_key" => "secret",
|
294
|
+
"bucket" => "logstash-test",
|
295
|
+
"codec" => "json",
|
296
|
+
}
|
297
|
+
}
|
298
|
+
before do
|
299
|
+
Aws.config[:s3] = {
|
300
|
+
stub_responses: {
|
301
|
+
get_object: error
|
302
|
+
}
|
303
|
+
}
|
304
|
+
allow_any_instance_of(Aws::S3::Bucket).to receive(:objects) { objects }
|
305
|
+
end
|
306
|
+
|
307
|
+
it 'should not crash the plugin' do
|
308
|
+
events = fetch_events(config)
|
309
|
+
expect(events.size).to eq(0)
|
310
|
+
end
|
311
|
+
end
|
312
|
+
end
|
313
|
+
end
|
314
|
+
|
259
315
|
context 'when working with logs' do
|
260
316
|
let(:objects) { [log] }
|
261
317
|
let(:log) { double(:key => 'uncompressed.log', :last_modified => Time.now - 2 * day, :content_length => 5) }
|