logstash-output-s3 3.1.1 → 3.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 9008635509ce94ad1d6e2242f256c67e0e2ed269
4
- data.tar.gz: 5f05f5f226976a70c3df9342dd6b4db0e3760274
3
+ metadata.gz: 35c6cb316aa0ec036121f892c821dbdb11de3c8d
4
+ data.tar.gz: faa1102b573616d9b02496a32b16eca96d151ba6
5
5
  SHA512:
6
- metadata.gz: f6a4a44f0b6c9da9f8ec46c230b9aa7d01c2508d01fbb68632debceb23223bc252c9d80e7eb2a241764718e67eee3ba289f7100677cad2ffa73db0e4b4a5892e
7
- data.tar.gz: ad0adb0d2a1125a427e013fdcc1d1c2884c6d215523df590081e3b670440656c1d92cebb5765dfe7c898220596d4f6553e74ba336929247ee1f606440dbf654f
6
+ metadata.gz: e56c266affad3d06f0e04d324a9cd8ccd1827a92c0eb6a70965b70ff40d60f5761314f0d4689275bcfc4510ce3ee26fa8de5f8f54d18230bcb04872320055d99
7
+ data.tar.gz: 8483fed9096096e0d0842a5c720642b7382ec4a4b311d57f160ebc0f5b53a1ddc8af369adc0a67fa69511717302858fdf329551cb99a2993835a59985b74c395
data/CHANGELOG.md CHANGED
@@ -1,9 +1,6 @@
1
- ## 3.1.1
2
- - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
3
-
4
- ## 3.1.0
5
- - breaking,config: Remove deprecated config `endpoint_region`. Please use `region` instead.
6
-
1
+ ## 3.1.2
2
+ - Fix improper shutdown of output worker threads
3
+ - improve exception handling
7
4
  ## 3.0.1
8
5
  - Republish all the gems under jruby.
9
6
 
@@ -135,7 +135,7 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
135
135
 
136
136
  # Exposed attributes for testing purpose.
137
137
  attr_accessor :tempfile
138
- attr_reader :page_counter
138
+ attr_reader :page_counter, :upload_workers
139
139
  attr_reader :s3
140
140
 
141
141
  def aws_s3_config
@@ -370,7 +370,7 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
370
370
  private
371
371
  def shutdown_upload_workers
372
372
  @logger.debug("S3: Gracefully shutdown the upload workers")
373
- @upload_queue << LogStash::ShutdownEvent
373
+ @upload_queue << LogStash::SHUTDOWN
374
374
  end
375
375
 
376
376
  private
@@ -421,10 +421,11 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
421
421
  Stud::Task.new do
422
422
  LogStash::Util::set_thread_name("<S3 upload worker #{worker_id}")
423
423
 
424
- while true do
424
+ continue = true
425
+ while continue do
425
426
  @logger.debug("S3: upload worker is waiting for a new file to upload.", :worker_id => worker_id)
426
427
 
427
- upload_worker
428
+ continue = upload_worker
428
429
  end
429
430
  end
430
431
  end
@@ -432,15 +433,26 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
432
433
 
433
434
  private
434
435
  def upload_worker
435
- file = @upload_queue.deq
436
+ file = nil
437
+ begin
438
+ file = @upload_queue.deq
436
439
 
437
- case file
438
- when LogStash::ShutdownEvent
440
+ if file == LogStash::SHUTDOWN
439
441
  @logger.debug("S3: upload worker is shutting down gracefuly")
440
- @upload_queue.enq(LogStash::ShutdownEvent)
442
+ @upload_queue.enq(LogStash::SHUTDOWN)
443
+ false
441
444
  else
442
445
  @logger.debug("S3: upload working is uploading a new file", :filename => File.basename(file))
443
446
  move_file_to_bucket(file)
447
+ true
448
+ end
449
+ rescue Exception => ex
450
+ @logger.error("failed to upload, will re-enqueue #{file} for upload",
451
+ :ex => ex, :backtrace => ex.backtrace)
452
+ unless file.nil? # Rare case if the first line of the begin doesn't execute
453
+ @upload_queue.enq(file)
454
+ end
455
+ true
444
456
  end
445
457
  end
446
458
 
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-output-s3'
4
- s.version = '3.1.1'
4
+ s.version = '3.1.2'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "This plugin was created for store the logstash's events into Amazon Simple Storage Service (Amazon S3)"
7
7
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -302,6 +302,33 @@ describe LogStash::Outputs::S3 do
302
302
  end
303
303
  end
304
304
 
305
+ describe "closing" do
306
+ let(:options) do
307
+ {
308
+ "access_key_id" => 1234,
309
+ "secret_access_key" => "secret",
310
+ "bucket" => "mahbucket"
311
+ }
312
+ end
313
+ subject do
314
+ ::LogStash::Outputs::S3.new(options)
315
+ end
316
+
317
+ before do
318
+ subject.register
319
+ end
320
+
321
+ it "should be clean" do
322
+ subject.do_close
323
+ end
324
+
325
+ it "should remove all worker threads" do
326
+ subject.do_close
327
+ sleep 1
328
+ expect(subject.upload_workers.map(&:thread).any?(&:alive?)).to be false
329
+ end
330
+ end
331
+
305
332
  it "doesn't skip events if using the time_file option", :tag => :slow do
306
333
  Stud::Temporary.directory do |temporary_directory|
307
334
  time_file = rand(1..2)
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-s3
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.1.1
4
+ version: 3.1.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-07-14 00:00:00.000000000 Z
11
+ date: 2016-08-08 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -140,7 +140,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
140
140
  version: '0'
141
141
  requirements: []
142
142
  rubyforge_project:
143
- rubygems_version: 2.6.3
143
+ rubygems_version: 2.4.8
144
144
  signing_key:
145
145
  specification_version: 4
146
146
  summary: This plugin was created for store the logstash's events into Amazon Simple Storage Service (Amazon S3)