logstash-output-google_cloud_storage 3.0.4 → 3.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 0cf8af9589beeacbd3a73a889dfe8d0397e25204
-  data.tar.gz: be2f8a97aa3858984669a37119803c79d213ae70
+SHA256:
+  metadata.gz: b52d957da3f5bba4f2d7ce3a947d82c6f29e17050aca2d7b9c5b5e6d75f5c519
+  data.tar.gz: f2724f44723ce7c75533ba73b4965f0d59f639960c5ffd7e597dfc9e214539a1
 SHA512:
-  metadata.gz: 780e33a16e0df8a81e1554c9403c8bcfcbf65e2fc0a1327f4ef750e6b80fe694bd62265aec915438f8f71c5cea1a67e1609c08885c8a3dda6a3aa78f8866be35
-  data.tar.gz: af43a2d1f1f1f145ae638ff9df5fe122da4e8d8fa574170e4f5d11c9038f1246519e4e089a4665ab98a628a841be5e7c6e42e2080bc42cde27755e33aecbdb7c
+  metadata.gz: 86427f0d948eaf48b1fa34fe6dab17f7ab972a91b41cddaaf58a706ca20351a15cd13028ab37387486c72d4c1de25bee1a76009c08ea7dd0309a998e2855cee3
+  data.tar.gz: f8d8e6104679fba6713b53024469e150d01243016134022eaebb948c6247da050cdd2adc19c1c3fd53350f4a13e2de3145b7090a6c3b3d9878c5cf8c5743c837

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.0.5
+  - Docs: Set the default_codec doc attribute.
+
 ## 3.0.4
   - Fix some documentation issues
 

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012–2016 Elasticsearch <http://www.elastic.co>
+Copyright (c) 2012-2018 Elasticsearch <http://www.elastic.co>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/docs/index.asciidoc

@@ -1,5 +1,6 @@
 :plugin: google_cloud_storage
 :type: output
+:default_codec: plain
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!
@@ -203,4 +204,6 @@ around one hour).
 
 
 [id="plugins-{type}s-{plugin}-common-options"]
-include::{include_path}/{type}.asciidoc[]
+include::{include_path}/{type}.asciidoc[]
+
+:default_codec!:

data/lib/logstash/outputs/google_cloud_storage.rb

@@ -59,6 +59,7 @@ require "zlib"
 #      flush_interval_secs => 2                                  (optional)
 #      gzip => false                                             (optional)
 #      uploader_interval_secs => 60                              (optional)
+#      upload_synchronous => false                               (optional)
 #    }
 # }
 # --------------------------
@@ -118,6 +119,17 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
   # around one hour).
   config :uploader_interval_secs, :validate => :number, :default => 60
 
+  # When true, files are uploaded by the event processing thread as soon as a file is ready.
+  # When false, (the default behaviour), files will be uploaded in a dedicated thread.
+  #
+  # Enabling this option provides greater likelihood that all generated files will be
+  # to GCS, especially in the event of a graceful shutdown of logstash, such as when an
+  # input plugin reaches the end of events. This comes at the price of introducing delays
+  # in the event processing pipeline as files are uploaded.
+  #
+  # When this feature is enabled, the uploader_interval_secs option has no effect.
+  config :upload_synchronous, :validate => :boolean, :default => false
+
   public
   def register
     require "fileutils"
@@ -125,12 +137,19 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
 
     @logger.debug("GCS: register plugin")
 
-    @upload_queue = Queue.new
     @last_flush_cycle = Time.now
+
+    unless upload_synchronous
+      initialize_upload_queue()
+    end
+
     initialize_temp_directory()
     initialize_current_log()
     initialize_google_client()
-    initialize_uploader()
+
+    unless upload_synchronous
+      @uploader = start_uploader
+    end
 
     if @gzip
       @content_type = 'application/gzip'
@@ -162,6 +181,12 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
       # Close does not guarantee that data is physically written to disk.
       @temp_file.fsync()
       @temp_file.close()
+
+      if upload_synchronous
+        upload_object(@temp_file.to_path)
+        File.delete(@temp_file.to_path)
+      end
+
       initialize_next_log()
     end
 
@@ -179,7 +204,15 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
     @logger.debug("GCS: close method called")
 
     @temp_file.fsync()
+    filename = @temp_file.to_path
+    size = @temp_file.size
     @temp_file.close()
+
+    if upload_synchronous && size > 0
+      @logger.debug("GCS: uploading last file of #{size.to_s}b")
+      upload_object(filename)
+      File.delete(filename)
+    end
   end
 
   private
@@ -222,42 +255,51 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
     end
   end
 
+  def start_uploader
+    Thread.new do
+      @logger.debug("GCS: starting uploader")
+      while true
+        upload_from_queue()
+      end
+    end
+  end
   ##
-  # Starts thread to upload log files.
+  # Uploads log files.
   #
   # Uploader is done in a separate thread, not holding the receive method above.
-  def initialize_uploader
-    @uploader = Thread.new do
-      @logger.debug("GCS: starting uploader")
-      while true
-        filename = @upload_queue.pop
-
-        # Reenqueue if it is still the current file.
-        if filename == @temp_file.to_path
-          if @current_base_path == get_base_path()
-            @logger.debug("GCS: reenqueue as log file is being currently appended to.",
-                          :filename => filename)
-            @upload_queue << filename
-            # If we got here, it means that older files were uploaded, so let's
-            # wait another minute before checking on this file again.
-            sleep @uploader_interval_secs
-            next
-          else
-            @logger.debug("GCS: flush and close file to be uploaded.",
-                          :filename => filename)
-            @temp_file.fsync()
-            @temp_file.close()
-            initialize_next_log()
-          end
-        end
-
-        upload_object(filename)
-        @logger.debug("GCS: delete local temporary file ",
+  def upload_from_queue
+    filename = @upload_queue.pop
+
+    # Reenqueue if it is still the current file.
+    if filename == @temp_file.to_path
+      if @current_base_path == get_base_path()
+        @logger.debug("GCS: reenqueue as log file is being currently appended to.",
                       :filename => filename)
-        File.delete(filename)
+        @upload_queue << filename
+        # If we got here, it means that older files were uploaded, so let's
+        # wait another minute before checking on this file again.
         sleep @uploader_interval_secs
+        return
+      else
+        @logger.debug("GCS: flush and close file to be uploaded.",
+                      :filename => filename)
+        @temp_file.fsync()
+        @temp_file.close()
+        initialize_next_log()
       end
     end
+
+    if File.stat(filename).size > 0
+      upload_object(filename)
+    else
+      @logger.debug("GCS: file size is zero, skip upload.",
+                     :filename => filename,
+                     :filesize => File.stat(filename).size)
+    end
+    @logger.debug("GCS: delete local temporary file ",
+                  :filename => filename)
+    File.delete(filename)
+    sleep @uploader_interval_secs
   end
 
   ##
@@ -319,7 +361,9 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
       fd = Zlib::GzipWriter.new(fd)
     end
     @temp_file = GCSIOWriter.new(fd)
-    @upload_queue << @temp_file.to_path
+    unless upload_synchronous
+      @upload_queue << @temp_file.to_path
+    end
   end
 
   ##
@@ -374,6 +418,15 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
     @client.authorization = service_account.authorize
   end
 
+  # Initialize the queue that harbors files to be uploaded
+  def initialize_upload_queue
+    @upload_queue = new_upload_queue()
+  end
+
+  def new_upload_queue
+    Queue.new
+  end
+
   ##
   # Uploads a local file to the configured bucket.
   def upload_object(filename)

data/logstash-output-google_cloud_storage.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-google_cloud_storage'
-  s.version         = '3.0.4'
+  s.version         = '3.0.5'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "plugin to upload log events to Google Cloud Storage (GCS)"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/outputs/google_cloud_storage_spec.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 require_relative "../spec_helper"
 require "google/api_client"
+require "tempfile"
 
 describe LogStash::Outputs::GoogleCloudStorage do
   
@@ -8,6 +9,9 @@ describe LogStash::Outputs::GoogleCloudStorage do
   let(:service_account) { double("service-account") }
   let(:key)    { "key" }
 
+  subject { described_class.new(config) }
+  let(:config) { {"bucket" => "", "key_path" => "", "service_account" => "", "uploader_interval_secs" => 0.1 } }
+
   before(:each) do
     allow(Google::APIClient).to receive(:new).and_return(client)
     allow(client).to receive(:discovered_api).with("storage", "v1")
@@ -18,7 +22,37 @@ describe LogStash::Outputs::GoogleCloudStorage do
   end
 
   it "should register without errors" do
-    plugin = LogStash::Plugin.lookup("output", "google_cloud_storage").new({"bucket" => "", "key_path" => "", "service_account" => ""})
-    expect { plugin.register }.to_not raise_error
+    expect { subject.register }.to_not raise_error
+  end
+
+  describe "file size based decider for uploading" do
+    let(:upload_queue) { Queue.new }
+    let(:content) { }
+    before(:each) do
+      allow(subject).to receive(:new_upload_queue).and_return(upload_queue)
+      subject.send(:initialize_upload_queue)
+      subject.send(:initialize_temp_directory)
+      subject.send(:initialize_current_log)
+      current_file = upload_queue.pop
+      File.write(current_file, content) if content
+      upload_queue.push(current_file)
+      subject.send(:initialize_next_log)
+    end
+
+    context "when spooled file is empty" do
+      let(:content) { nil }
+      it "doesn't get uploaded" do
+        expect(subject).to_not receive(:upload_object)
+        subject.send(:upload_from_queue)
+      end
+    end
+
+    context "when spooled file has content" do
+      let(:content) { "hello" }
+      it "gets uploaded" do
+        expect(subject).to receive(:upload_object)
+        subject.send(:upload_from_queue)
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-google_cloud_storage
 version: !ruby/object:Gem::Version
-  version: 3.0.4
+  version: 3.0.5
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-08-16 00:00:00.000000000 Z
+date: 2018-04-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -100,7 +100,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
@@ -139,7 +141,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.8
+rubygems_version: 2.6.11
 signing_key:
 specification_version: 4
 summary: plugin to upload log events to Google Cloud Storage (GCS)