logstash-output-google_bigquery 4.1.1-java → 4.1.2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1bea90d55eb25689d86a79c1adf326d382d8f2d7c6045a027935b5f19c4e005f
-  data.tar.gz: fd315b0e136a61429a63489a2d3ea21e294d474814d809911b382268037a8ce8
+  metadata.gz: 9361650ebd1559f1d2e7237028993cb9d86ae9cd0c9e86473a4d9f5910418736
+  data.tar.gz: b82fb5507e89bd687de2fda556937817164bb3b25aaaf40cda8fe7beca51a5cf
 SHA512:
-  metadata.gz: d178dea332da37a9f4762624a409415b90b55b74d3c2279cb521c79621d74e2fb70686b1256249d368da22a08c742a10714b8623eb632c423d9d8be4f90e8d2f
-  data.tar.gz: a83de7d1395056256d36889a4ecb8ee30b0d79cfc81ae2b193363a86cc188728218670ad5250a4cf24796eeff0af9d8f4a2aa4972e8c9d169c4a486f1c5df5ff
+  metadata.gz: c6d0dfb7489d814b9a0b10d73270165e6842720c317d692f8d49e0308abf8c6773ce929eac14e02d43537f8b308845c94d4cdc31eb8fe81dc437c043e4ee95b9
+  data.tar.gz: 40c022608e5c861990fc710ae220caab8cb3aef5a968b1ba59120d1cb77d2c639dcb6d5aeb16a03fd7eb5736ff8778c2370de26e7c47271ce72aa431d3e4254f

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 4.1.2
+ - Fixed issue where Logstash shutdown could cause data loss due to not flushing buffers on close [#52](https://github.com/logstash-plugins/logstash-output-google_bigquery/pull/52)
+
 ## 4.1.1
  - Fixed inaccuracies in documentation [#46](https://github.com/logstash-plugins/logstash-output-google_bigquery/pull/46) 
 

data/lib/logstash/outputs/google_bigquery.rb

@@ -7,6 +7,7 @@ require 'logstash/outputs/bigquery/schema'
 
 require 'time'
 require 'fileutils'
+require 'concurrent'
 
 #
 # === Summary
@@ -181,6 +182,7 @@ class LogStash::Outputs::GoogleBigQuery < LogStash::Outputs::Base
     @schema = LogStash::Outputs::BigQuery::Schema.parse_csv_or_json @csv_schema, @json_schema
     @bq_client = LogStash::Outputs::BigQuery::StreamingClient.new @json_key_file, @project_id, @logger
     @batcher = LogStash::Outputs::BigQuery::Batcher.new @batch_size, @batch_size_bytes
+    @stopping = Concurrent::AtomicBoolean.new(false)
 
     init_batcher_flush_thread
   end
@@ -274,11 +276,24 @@ class LogStash::Outputs::GoogleBigQuery < LogStash::Outputs::Base
 
   def init_batcher_flush_thread
     @flush_thread = Thread.new do
-      loop do
-        sleep @flush_interval_secs
+      until stopping?
+        Stud.stoppable_sleep(@flush_interval_secs) { stopping? }
 
         @batcher.enqueue(nil) { |batch| publish(batch) }
       end
     end
   end
+
+  def stopping?
+    @stopping.value
+  end
+
+  def close
+    @stopping.make_true
+    @flush_thread.wakeup
+    @flush_thread.join
+    # Final flush to publish any events published if a pipeline receives a shutdown signal after flush thread
+    # has begun flushing.
+    @batcher.enqueue(nil) { |batch| publish(batch) }
+  end
 end

data/logstash-output-google_bigquery.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-google_bigquery'
-  s.version         = '4.1.1'
+  s.version         = '4.1.2'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Writes events to Google BigQuery"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-google_bigquery
 version: !ruby/object:Gem::Version
-  version: 4.1.1
+  version: 4.1.2
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-10-25 00:00:00.000000000 Z
+date: 2019-07-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement