logstash-output-google_cloud_storage 3.1.0 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a9181dddb40215cd78a1b07d3b327df10e7dd678b3eab3d34f511a84f11a92a6
-  data.tar.gz: 4696f52738d5e20466b8639565a7eeee646b19e27e0421fc39c7cc283359a07b
+  metadata.gz: ef57485cd166eb205939da40bb4db73428955388af8e5c13d313852eb8c297c7
+  data.tar.gz: 8d3e0f581f611a9c7148ecc9f871b54e8c7ebf356ea3da39a3fa0a187d710184
 SHA512:
-  metadata.gz: 93b09c005fc9cd4afdca9678b14949f35ad5d07ec92cb5f1ffb2ab20d8228f6e46e72e197846d1704cc2575f9bbf8c603e03ebf355b0423f6de6e42f6cb698dc
-  data.tar.gz: 05e5bd961335f99458a4c46d5b271bcfd1279c6349817a483c9c65c1c62b0f1efeac48221b5c3c0e8869e0603caedcb5ece6b1d6da21a21641341fd306769549
+  metadata.gz: 022b0a599c17c5a9dc062093662556f2cfd15c7d2345527b92a4b9bda4fb2fa3756e837bbe471a60bf2034ce2a6b059f76b2ec01c1d27ba6b54575f628e46bc8
+  data.tar.gz: b603dea8edb673e1a1e0c72a5431d5d915ade37d1f6164fb3b40882bdecb936a6959ec16e6724d239ca26e87b651c6a8602cfbd439d82d824bb14781611a8bd0

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 3.2.0
+  - Change uploads to use a job pool for better performance
+    - Fixes [#22](https://github.com/logstash-plugins/logstash-output-google_cloud_storage/issues/22) - Refactor Job Queue Architecture
+    - Fixes [#5](https://github.com/logstash-plugins/logstash-output-google_cloud_storage/issues/5) - Major Performance Issues
+  - Wait for files to upload before Logstash quits
+    - Fixes [#15](https://github.com/logstash-plugins/logstash-output-google_cloud_storage/issues/15) - Fails to upload files when Logstash exits
+
 ## 3.1.0
   - Add support for disabling hostname in the log file names
   - Add support for adding a UUID to the log file names

data/docs/index.asciidoc

@@ -88,6 +88,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-key_password>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-key_path>> |<<string,string>>|Yes
 | <<plugins-{type}s-{plugin}-log_file_prefix>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-max_concurrent_uploads>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-max_file_size_kbytes>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-output_format>> |<<string,string>>, one of `["json", "plain"]`|No
 | <<plugins-{type}s-{plugin}-service_account>> |<<string,string>>|Yes
@@ -187,6 +188,17 @@ GCS path to private key file.
 Log file prefix. Log file will follow the format:
 <prefix>_hostname_date<.part?>.log
 
+[id="plugins-{type}s-{plugin}-max_concurrent_uploads"]
+===== `max_concurrent_uploads`
+
+  * Value type is <<number,number>>
+  * Default value is `5`
+
+Sets the maximum number of concurrent uploads to Cloud Storage at a time.
+Uploads are I/O bound so it makes sense to tune this paramater with regards
+to the network bandwidth available and the latency between your server and
+Cloud Storage.
+
 [id="plugins-{type}s-{plugin}-max_file_size_kbytes"]
 ===== `max_file_size_kbytes` 
 

data/lib/logstash/outputs/gcs/worker_pool.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+require 'thread'
+require 'concurrent'
+
+module LogStash
+  module Outputs
+    module Gcs
+      # WorkerPool creates a pool of workers that can handle jobs.
+      class WorkerPool
+        attr_reader :workers
+
+        def initialize(max_threads, synchronous=false)
+          @synchronous = synchronous
+
+          # set queue depth to the be the same as the number of threads so
+          # there's at most one pending job each when the plugin quits
+          @workers = Concurrent::ThreadPoolExecutor.new(
+            min_threads: 1,
+            max_threads: max_threads,
+            max_queue: max_threads,
+            fallback_policy: :caller_runs
+          )
+        end
+
+        # Submits a job to the worker pool, raises an error if the pool has
+        # already been stopped.
+        def post(&block)
+          raise 'Pool already stopped' unless @workers.running?
+
+          if @synchronous
+            block.call
+          else
+            @workers.post do
+              block.call
+            end
+          end
+        end
+
+        # Stops the worker pool
+        def stop!
+          @workers.shutdown
+          @workers.wait_for_termination
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/google_cloud_storage.rb

@@ -20,8 +20,11 @@
 # -----
 require "logstash/outputs/base"
 require "logstash/outputs/gcs/path_factory"
+require "logstash/outputs/gcs/worker_pool"
 require "logstash/namespace"
 require "logstash/json"
+require "stud/interval"
+require "thread"
 require "zlib"
 
 # Summary: plugin to upload log events to Google Cloud Storage (GCS), rolling
@@ -71,8 +74,6 @@ require "zlib"
 # * There's no recover method, so if logstash/plugin crashes, files may not
 # be uploaded to GCS.
 # * Allow user to configure file name.
-# * Allow parallel uploads for heavier loads (+ connection configuration if
-# exposed by Ruby API client)
 class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
   config_name "google_cloud_storage"
 
@@ -137,28 +138,22 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
   # When this feature is enabled, the uploader_interval_secs option has no effect.
   config :upload_synchronous, :validate => :boolean, :default => false
 
+  config :max_concurrent_uploads, :validate  => :number, :default => 5
+
   public
   def register
     require "fileutils"
-    require "thread"
-
     @logger.debug("GCS: register plugin")
-
     @last_flush_cycle = Time.now
 
-    unless upload_synchronous
-      initialize_upload_queue()
-    end
-
+    @workers = LogStash::Outputs::Gcs::WorkerPool.new(@max_concurrent_uploads, @upload_synchronous)
     initialize_temp_directory()
     initialize_path_factory
     open_current_file
 
     initialize_google_client()
 
-    unless upload_synchronous
-      @uploader = start_uploader
-    end
+    start_uploader
 
     if @gzip
       @content_type = 'application/gzip'
@@ -180,22 +175,7 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
     end
 
     # Time to roll file based on the date pattern? Or is it over the size limit?
-    if (@path_factory.should_rotate? || (@max_file_size_kbytes > 0 && @temp_file.size >= @max_file_size_kbytes * 1024))
-      @logger.debug("GCS: log file will be closed and uploaded",
-                    :filename => File.basename(@temp_file.to_path),
-                    :size => @temp_file.size.to_s,
-                    :max_size => @max_file_size_kbytes.to_s)
-      # Close does not guarantee that data is physically written to disk.
-      @temp_file.fsync()
-      @temp_file.close()
-
-      if upload_synchronous
-        upload_object(@temp_file.to_path)
-        File.delete(@temp_file.to_path)
-      end
-
-      initialize_next_log()
-    end
+    initialize_next_log if ready_to_rotate?
 
     @temp_file.write(message)
     @temp_file.write("\n")
@@ -208,21 +188,24 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
 
   public
   def close
-    @logger.debug("GCS: close method called")
+    @logger.debug('Stopping the plugin, uploading the remaining files.')
 
-    @temp_file.fsync()
-    filename = @temp_file.to_path
-    size = @temp_file.size
-    @temp_file.close()
+    Stud.stop!(@registration_thread) unless @registration_thread.nil?
 
-    if upload_synchronous && size > 0
-      @logger.debug("GCS: uploading last file of #{size.to_s}b")
-      upload_object(filename)
-      File.delete(filename)
-    end
+    close_and_upload_current
+    @workers.stop!
   end
 
   private
+
+
+  def ready_to_rotate?
+    path_changed = @path_factory.should_rotate?
+    too_big = @max_file_size_kbytes > 0 && @temp_file.size >= @max_file_size_kbytes * 1024
+
+    path_changed || too_big
+  end
+
   ##
   # Flushes temporary log file every flush_interval_secs seconds or so.
   # This is triggered by events, but if there are no events there's no point
@@ -276,54 +259,17 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
 
   def start_uploader
     Thread.new do
-      @logger.debug("GCS: starting uploader")
-      while true
-        upload_from_queue()
-      end
-    end
-  end
-  ##
-  # Uploads log files.
-  #
-  # Uploader is done in a separate thread, not holding the receive method above.
-  def upload_from_queue
-    filename = @upload_queue.pop
-
-    # Reenqueue if it is still the current file.
-    if filename == @temp_file.to_path
-      if !@path_factory.should_rotate?
-        @logger.debug("GCS: reenqueue as log file is being currently appended to.",
-                      :filename => filename)
-        @upload_queue << filename
-        # If we got here, it means that older files were uploaded, so let's
-        # wait another minute before checking on this file again.
-        sleep @uploader_interval_secs
-        return
-      else
-        @logger.debug("GCS: flush and close file to be uploaded.",
-                      :filename => filename)
-        @temp_file.fsync()
-        @temp_file.close()
-        initialize_next_log()
+      @registration_thread = Thread.current
+      Stud.interval(@uploader_interval_secs) do
+        initialize_next_log if ready_to_rotate?
       end
     end
-
-    if File.stat(filename).size > 0
-      upload_object(filename)
-    else
-      @logger.debug("GCS: file size is zero, skip upload.",
-                     :filename => filename,
-                     :filesize => File.stat(filename).size)
-    end
-    @logger.debug("GCS: delete local temporary file ",
-                  :filename => filename)
-    File.delete(filename)
   end
 
   ##
   # Opens current log file and updates @temp_file with an instance of IOWriter.
   # This method also adds file to the upload queue.
-  def open_current_file()
+  def open_current_file
     path = @path_factory.current_path
 
     stat = File.stat(path) rescue nil
@@ -336,9 +282,6 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
       fd = Zlib::GzipWriter.new(fd)
     end
     @temp_file = GCSIOWriter.new(fd)
-    unless upload_synchronous
-      @upload_queue << @temp_file.to_path
-    end
   end
 
   ##
@@ -347,6 +290,7 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
   # the base log file name is the same (e.g. log file was not rolled given the
   # date pattern).
   def initialize_next_log
+    close_and_upload_current
     @path_factory.rotate_path!
     open_current_file()
   end
@@ -369,15 +313,6 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
     @client.authorization = service_account.authorize
   end
 
-  # Initialize the queue that harbors files to be uploaded
-  def initialize_upload_queue
-    @upload_queue = new_upload_queue()
-  end
-
-  def new_upload_queue
-    Queue.new
-  end
-
   ##
   # Uploads a local file to the configured bucket.
   def upload_object(filename)
@@ -404,6 +339,32 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
       retry
     end
   end
+
+  def close_and_upload_current
+    return if @temp_file.nil?
+
+    filename = @temp_file.to_path
+    @temp_file.fsync
+    @temp_file.close
+    @logger.info("Uploading file: #{filename}")
+
+    @workers.post do
+      upload_and_delete(filename)
+    end
+  end
+
+  def upload_and_delete(filename)
+    file_size = File.stat(filename).size
+
+    if file_size > 0
+      upload_object(filename)
+    else
+      @logger.debug('File size is zero, skip upload.', :filename => filename)
+    end
+
+    @logger.debug('Delete local temporary file', :filename => filename)
+    File.delete(filename)
+  end
 end
 
 ##

data/logstash-output-google_cloud_storage.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-google_cloud_storage'
-  s.version         = '3.1.0'
+  s.version         = '3.2.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "plugin to upload log events to Google Cloud Storage (GCS)"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -25,7 +25,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'google-api-client', '~> 0.8.7' # version 0.9.x works only with ruby 2.x
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'mime-types', '~> 2' # last version compatible with ruby 2.x
-
+  s.add_runtime_dependency 'concurrent-ruby', '1.0.5'
   s.add_development_dependency 'logstash-devutils'
 end
 

data/spec/outputs/gcs/worker_pool_spec.rb

@@ -0,0 +1,27 @@
+# encoding: utf-8
+require 'logstash/outputs/gcs/worker_pool'
+
+describe LogStash::Outputs::Gcs::WorkerPool do
+  describe '#post' do
+    it 'runs the task in the same thread if synchronous' do
+      pool = LogStash::Outputs::Gcs::WorkerPool.new(5, true)
+      expect(pool.workers).to_not receive(:post)
+
+      pool.post { 1 + 2 }
+    end
+
+    it 'runs the task in a different thread if asynchronous' do
+      pool = LogStash::Outputs::Gcs::WorkerPool.new(5, false)
+      expect(pool.workers).to receive(:post)
+
+      pool.post { 1 + 2 }
+    end
+
+    it 'raises an error if the pool is already stopped' do
+      pool = LogStash::Outputs::Gcs::WorkerPool.new(5, true)
+      pool.stop!
+
+      expect{ pool.post{} }.to raise_error(RuntimeError)
+    end
+  end
+end

data/spec/outputs/google_cloud_storage_spec.rb

@@ -4,13 +4,13 @@ require "google/api_client"
 require "tempfile"
 
 describe LogStash::Outputs::GoogleCloudStorage do
-  
+
   let(:client) { double("google-client") }
   let(:service_account) { double("service-account") }
   let(:key)    { "key" }
 
   subject { described_class.new(config) }
-  let(:config) { {"bucket" => "", "key_path" => "", "service_account" => "", "uploader_interval_secs" => 0.1 } }
+  let(:config) { {"bucket" => "", "key_path" => "", "service_account" => "", "uploader_interval_secs" => 0.1, "upload_synchronous" => true} }
 
   before(:each) do
     allow(Google::APIClient).to receive(:new).and_return(client)
@@ -24,36 +24,4 @@ describe LogStash::Outputs::GoogleCloudStorage do
   it "should register without errors" do
     expect { subject.register }.to_not raise_error
   end
-
-  describe "file size based decider for uploading" do
-    let(:upload_queue) { Queue.new }
-    let(:content) { }
-    before(:each) do
-      allow(subject).to receive(:new_upload_queue).and_return(upload_queue)
-      subject.send(:initialize_upload_queue)
-      subject.send(:initialize_temp_directory)
-      subject.send(:initialize_path_factory)
-      subject.send(:open_current_file)
-      current_file = upload_queue.pop
-      File.write(current_file, content) if content
-      upload_queue.push(current_file)
-      subject.send(:initialize_next_log)
-    end
-
-    context "when spooled file is empty" do
-      let(:content) { nil }
-      it "doesn't get uploaded" do
-        expect(subject).to_not receive(:upload_object)
-        subject.send(:upload_from_queue)
-      end
-    end
-
-    context "when spooled file has content" do
-      let(:content) { "hello" }
-      it "gets uploaded" do
-        expect(subject).to receive(:upload_object)
-        subject.send(:upload_from_queue)
-      end
-    end
-  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-google_cloud_storage
 version: !ruby/object:Gem::Version
-  version: 3.1.0
+  version: 3.2.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-05-16 00:00:00.000000000 Z
+date: 2018-06-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -86,6 +86,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.5
+  name: concurrent-ruby
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.5
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -116,9 +130,11 @@ files:
 - README.md
 - docs/index.asciidoc
 - lib/logstash/outputs/gcs/path_factory.rb
+- lib/logstash/outputs/gcs/worker_pool.rb
 - lib/logstash/outputs/google_cloud_storage.rb
 - logstash-output-google_cloud_storage.gemspec
 - spec/outputs/gcs/path_factory_spec.rb
+- spec/outputs/gcs/worker_pool_spec.rb
 - spec/outputs/google_cloud_storage_spec.rb
 - spec/spec_helper.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
@@ -149,5 +165,6 @@ specification_version: 4
 summary: plugin to upload log events to Google Cloud Storage (GCS)
 test_files:
 - spec/outputs/gcs/path_factory_spec.rb
+- spec/outputs/gcs/worker_pool_spec.rb
 - spec/outputs/google_cloud_storage_spec.rb
 - spec/spec_helper.rb