logstash-output-s3 3.1.1 → 3.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -6
- data/lib/logstash/outputs/s3.rb +20 -8
- data/logstash-output-s3.gemspec +1 -1
- data/spec/outputs/s3_spec.rb +27 -0
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 35c6cb316aa0ec036121f892c821dbdb11de3c8d
|
4
|
+
data.tar.gz: faa1102b573616d9b02496a32b16eca96d151ba6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e56c266affad3d06f0e04d324a9cd8ccd1827a92c0eb6a70965b70ff40d60f5761314f0d4689275bcfc4510ce3ee26fa8de5f8f54d18230bcb04872320055d99
|
7
|
+
data.tar.gz: 8483fed9096096e0d0842a5c720642b7382ec4a4b311d57f160ebc0f5b53a1ddc8af369adc0a67fa69511717302858fdf329551cb99a2993835a59985b74c395
|
data/CHANGELOG.md
CHANGED
@@ -1,9 +1,6 @@
|
|
1
|
-
## 3.1.
|
2
|
-
-
|
3
|
-
|
4
|
-
## 3.1.0
|
5
|
-
- breaking,config: Remove deprecated config `endpoint_region`. Please use `region` instead.
|
6
|
-
|
1
|
+
## 3.1.2
|
2
|
+
- Fix improper shutdown of output worker threads
|
3
|
+
- improve exception handling
|
7
4
|
## 3.0.1
|
8
5
|
- Republish all the gems under jruby.
|
9
6
|
|
data/lib/logstash/outputs/s3.rb
CHANGED
@@ -135,7 +135,7 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
|
|
135
135
|
|
136
136
|
# Exposed attributes for testing purpose.
|
137
137
|
attr_accessor :tempfile
|
138
|
-
attr_reader :page_counter
|
138
|
+
attr_reader :page_counter, :upload_workers
|
139
139
|
attr_reader :s3
|
140
140
|
|
141
141
|
def aws_s3_config
|
@@ -370,7 +370,7 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
|
|
370
370
|
private
|
371
371
|
def shutdown_upload_workers
|
372
372
|
@logger.debug("S3: Gracefully shutdown the upload workers")
|
373
|
-
@upload_queue << LogStash::
|
373
|
+
@upload_queue << LogStash::SHUTDOWN
|
374
374
|
end
|
375
375
|
|
376
376
|
private
|
@@ -421,10 +421,11 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
|
|
421
421
|
Stud::Task.new do
|
422
422
|
LogStash::Util::set_thread_name("<S3 upload worker #{worker_id}")
|
423
423
|
|
424
|
-
|
424
|
+
continue = true
|
425
|
+
while continue do
|
425
426
|
@logger.debug("S3: upload worker is waiting for a new file to upload.", :worker_id => worker_id)
|
426
427
|
|
427
|
-
upload_worker
|
428
|
+
continue = upload_worker
|
428
429
|
end
|
429
430
|
end
|
430
431
|
end
|
@@ -432,15 +433,26 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
|
|
432
433
|
|
433
434
|
private
|
434
435
|
def upload_worker
|
435
|
-
file =
|
436
|
+
file = nil
|
437
|
+
begin
|
438
|
+
file = @upload_queue.deq
|
436
439
|
|
437
|
-
|
438
|
-
when LogStash::ShutdownEvent
|
440
|
+
if file == LogStash::SHUTDOWN
|
439
441
|
@logger.debug("S3: upload worker is shutting down gracefuly")
|
440
|
-
@upload_queue.enq(LogStash::
|
442
|
+
@upload_queue.enq(LogStash::SHUTDOWN)
|
443
|
+
false
|
441
444
|
else
|
442
445
|
@logger.debug("S3: upload working is uploading a new file", :filename => File.basename(file))
|
443
446
|
move_file_to_bucket(file)
|
447
|
+
true
|
448
|
+
end
|
449
|
+
rescue Exception => ex
|
450
|
+
@logger.error("failed to upload, will re-enqueue #{file} for upload",
|
451
|
+
:ex => ex, :backtrace => ex.backtrace)
|
452
|
+
unless file.nil? # Rare case if the first line of the begin doesn't execute
|
453
|
+
@upload_queue.enq(file)
|
454
|
+
end
|
455
|
+
true
|
444
456
|
end
|
445
457
|
end
|
446
458
|
|
data/logstash-output-s3.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-output-s3'
|
4
|
-
s.version = '3.1.
|
4
|
+
s.version = '3.1.2'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "This plugin was created for store the logstash's events into Amazon Simple Storage Service (Amazon S3)"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
data/spec/outputs/s3_spec.rb
CHANGED
@@ -302,6 +302,33 @@ describe LogStash::Outputs::S3 do
|
|
302
302
|
end
|
303
303
|
end
|
304
304
|
|
305
|
+
describe "closing" do
|
306
|
+
let(:options) do
|
307
|
+
{
|
308
|
+
"access_key_id" => 1234,
|
309
|
+
"secret_access_key" => "secret",
|
310
|
+
"bucket" => "mahbucket"
|
311
|
+
}
|
312
|
+
end
|
313
|
+
subject do
|
314
|
+
::LogStash::Outputs::S3.new(options)
|
315
|
+
end
|
316
|
+
|
317
|
+
before do
|
318
|
+
subject.register
|
319
|
+
end
|
320
|
+
|
321
|
+
it "should be clean" do
|
322
|
+
subject.do_close
|
323
|
+
end
|
324
|
+
|
325
|
+
it "should remove all worker threads" do
|
326
|
+
subject.do_close
|
327
|
+
sleep 1
|
328
|
+
expect(subject.upload_workers.map(&:thread).any?(&:alive?)).to be false
|
329
|
+
end
|
330
|
+
end
|
331
|
+
|
305
332
|
it "doesn't skip events if using the time_file option", :tag => :slow do
|
306
333
|
Stud::Temporary.directory do |temporary_directory|
|
307
334
|
time_file = rand(1..2)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-s3
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.1.
|
4
|
+
version: 3.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-08-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -140,7 +140,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
140
140
|
version: '0'
|
141
141
|
requirements: []
|
142
142
|
rubyforge_project:
|
143
|
-
rubygems_version: 2.
|
143
|
+
rubygems_version: 2.4.8
|
144
144
|
signing_key:
|
145
145
|
specification_version: 4
|
146
146
|
summary: This plugin was created for store the logstash's events into Amazon Simple Storage Service (Amazon S3)
|