logstash-output-google_cloud_storage 3.0.5 → 3.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b52d957da3f5bba4f2d7ce3a947d82c6f29e17050aca2d7b9c5b5e6d75f5c519
-  data.tar.gz: f2724f44723ce7c75533ba73b4965f0d59f639960c5ffd7e597dfc9e214539a1
+  metadata.gz: a9181dddb40215cd78a1b07d3b327df10e7dd678b3eab3d34f511a84f11a92a6
+  data.tar.gz: 4696f52738d5e20466b8639565a7eeee646b19e27e0421fc39c7cc283359a07b
 SHA512:
-  metadata.gz: 86427f0d948eaf48b1fa34fe6dab17f7ab972a91b41cddaaf58a706ca20351a15cd13028ab37387486c72d4c1de25bee1a76009c08ea7dd0309a998e2855cee3
-  data.tar.gz: f8d8e6104679fba6713b53024469e150d01243016134022eaebb948c6247da050cdd2adc19c1c3fd53350f4a13e2de3145b7090a6c3b3d9878c5cf8c5743c837
+  metadata.gz: 93b09c005fc9cd4afdca9678b14949f35ad5d07ec92cb5f1ffb2ab20d8228f6e46e72e197846d1704cc2575f9bbf8c603e03ebf355b0423f6de6e42f6cb698dc
+  data.tar.gz: 05e5bd961335f99458a4c46d5b271bcfd1279c6349817a483c9c65c1c62b0f1efeac48221b5c3c0e8869e0603caedcb5ece6b1d6da21a21641341fd306769549

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 3.1.0
+  - Add support for disabling hostname in the log file names
+  - Add support for adding a UUID to the log file names
+
 ## 3.0.5
   - Docs: Set the default_codec doc attribute.
 

data/docs/index.asciidoc

@@ -57,6 +57,8 @@ output {
      flush_interval_secs => 2                                  (optional)
      gzip => false                                             (optional)
      uploader_interval_secs => 60                              (optional)
+     include_uuid => true                                      (optional)
+     include_hostname => true                                  (optional)
    }
 }
 --------------------------
@@ -81,6 +83,8 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-date_pattern>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-flush_interval_secs>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-gzip>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-include_hostname>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-include_uuid>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-key_password>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-key_path>> |<<string,string>>|Yes
 | <<plugins-{type}s-{plugin}-log_file_prefix>> |<<string,string>>|No
@@ -131,6 +135,32 @@ on every message.
 
 Gzip output stream when writing events to log files.
 
+[id="plugins-{type}s-{plugin}-include_hostname"]
+===== `include_hostname`
+
+added[3.1.0]
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `true`
+
+Should the hostname be included in the file name?
+You may want to turn this off for privacy reasons or if you are running multiple
+instances of Logstash and need to match the files you create with a simple glob
+such as if you wanted to import files to BigQuery.
+
+
+[id="plugins-{type}s-{plugin}-include_uuid"]
+===== `include_uuid`
+
+added[3.1.0]
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+Adds a UUID to the end of a file name.
+You may want to enable this feature so files don't clobber one another if you're
+running multiple instances of Logstash or if you expect frequent node restarts.
+
 [id="plugins-{type}s-{plugin}-key_password"]
 ===== `key_password` 
 

data/lib/logstash/outputs/gcs/path_factory.rb

@@ -0,0 +1,116 @@
+# encoding: utf-8
+require 'thread'
+
+module LogStash
+  module Outputs
+    module Gcs
+      # PathFactory creates paths for rotating files.
+      class PathFactory
+        def initialize(directory, prefix, include_host, date_pattern, include_part, include_uuid, is_gzipped)
+          @path_lock = Mutex.new
+
+          pattern = '%{prefix}'
+          pattern += '_%{host}' if include_host
+          pattern += '_%{date}'
+          @base_pattern = pattern
+
+          pattern += '.part%{partf}' if include_part
+          pattern += '.%{uuid}' if include_uuid
+          pattern += '.log'
+          pattern += '.gz' if is_gzipped
+          @pattern = pattern
+
+          @prefix = prefix
+          @directory = directory
+          @date_pattern = date_pattern
+
+          @part_number = starting_part
+          @current = template_variables
+        end
+
+        # Rotates the path to the next one in sequence. If the path has a part number
+        # and the base path (date/hostname) haven't changed the part number is incremented.
+        def rotate_path!
+          @path_lock.synchronize {
+            @part_number = (next_base == current_base) ? @part_number + 1 : 0
+            @current = template_variables
+          }
+
+          current_path
+        end
+
+        # Checks if the file is ready to rotate because the timestamp changed.
+        def should_rotate?
+          @path_lock.synchronize {
+            next_base != current_base
+          }
+        end
+
+        # Returns the full path to the current file including parent directory.
+        def current_path(vars=nil)
+          @path_lock.synchronize {
+            filename = @pattern % (vars || @current)
+            ::File.join(@directory, filename)
+          }
+        end
+
+        private
+
+        # search through the directory for a file with the same base, and if it exists,
+        # set our part to be the max + 1 so we don't clobber existing files.
+        def starting_part
+          return 0 unless ::File.directory? @directory
+
+          base_path = ::File.join(@directory, next_base)
+
+          part_numbers = Dir.glob(base_path + '.part*').map do |item|
+            match = /^.*\.part(?<part_num>\d+).*$/.match(item)
+            next if match.nil?
+            match[:part_num].to_i
+          end
+
+          part_numbers.any? ? part_numbers.max + 1 : 0
+        end
+
+        def template_variables
+          {
+              prefix: @prefix,
+              host: Socket.gethostname,
+              date: Time.now.strftime(@date_pattern),
+              partf: '%03d' % @part_number,
+              uuid: SecureRandom.uuid
+          }
+        end
+
+        def next_base
+          @base_pattern % template_variables
+        end
+
+        def current_base
+          @base_pattern % @current
+        end
+      end
+
+      # PathFactoryBuilder makes the long PathFactory constructor chain more readable.
+      class PathFactoryBuilder
+        def self.build
+          builder = new
+          yield builder
+          builder.build_path_factory
+        end
+
+        def self.builder_setter(*names)
+          names.each do |name|
+            define_method("set_#{name}") {|arg| instance_variable_set("@#{name}", arg)}
+          end
+        end
+
+        builder_setter :directory, :prefix, :include_host, :date_pattern, :include_part, :include_uuid, :is_gzipped
+
+        def build_path_factory
+          PathFactory.new(@directory, @prefix, @include_host, @date_pattern, @include_part, @include_uuid, @is_gzipped)
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/google_cloud_storage.rb

@@ -19,6 +19,7 @@
 # limitations under the License.
 # -----
 require "logstash/outputs/base"
+require "logstash/outputs/gcs/path_factory"
 require "logstash/namespace"
 require "logstash/json"
 require "zlib"
@@ -119,6 +120,12 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
   # around one hour).
   config :uploader_interval_secs, :validate => :number, :default => 60
 
+  # Should the hostname be included in the file name?
+  config :include_hostname, :validate => :boolean, :default => true
+
+  # Should a UUID be included in the file name?
+  config :include_uuid, :validate => :boolean, :default => false
+
   # When true, files are uploaded by the event processing thread as soon as a file is ready.
   # When false, (the default behaviour), files will be uploaded in a dedicated thread.
   #
@@ -144,7 +151,9 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
     end
 
     initialize_temp_directory()
-    initialize_current_log()
+    initialize_path_factory
+    open_current_file
+
     initialize_google_client()
 
     unless upload_synchronous
@@ -170,10 +179,8 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
       message = event.to_s
     end
 
-    new_base_path = get_base_path()
-
     # Time to roll file based on the date pattern? Or is it over the size limit?
-    if (@current_base_path != new_base_path || (@max_file_size_kbytes > 0 && @temp_file.size >= @max_file_size_kbytes * 1024))
+    if (@path_factory.should_rotate? || (@max_file_size_kbytes > 0 && @temp_file.size >= @max_file_size_kbytes * 1024))
       @logger.debug("GCS: log file will be closed and uploaded",
                     :filename => File.basename(@temp_file.to_path),
                     :size => @temp_file.size.to_s,
@@ -255,6 +262,18 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
     end
   end
 
+  def initialize_path_factory
+    @path_factory = LogStash::Outputs::Gcs::PathFactoryBuilder.build do |builder|
+      builder.set_directory @temp_directory
+      builder.set_prefix @log_file_prefix
+      builder.set_include_host @include_hostname
+      builder.set_date_pattern @date_pattern
+      builder.set_include_part(@max_file_size_kbytes > 0)
+      builder.set_include_uuid @include_uuid
+      builder.set_is_gzipped @gzip
+    end
+  end
+
   def start_uploader
     Thread.new do
       @logger.debug("GCS: starting uploader")
@@ -272,7 +291,7 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
 
     # Reenqueue if it is still the current file.
     if filename == @temp_file.to_path
-      if @current_base_path == get_base_path()
+      if !@path_factory.should_rotate?
         @logger.debug("GCS: reenqueue as log file is being currently appended to.",
                       :filename => filename)
         @upload_queue << filename
@@ -299,58 +318,14 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
     @logger.debug("GCS: delete local temporary file ",
                   :filename => filename)
     File.delete(filename)
-    sleep @uploader_interval_secs
-  end
-
-  ##
-  # Returns base path to log file that is invariant regardless of whether
-  # max file or gzip options.
-  def get_base_path
-    return @temp_directory + File::SEPARATOR + @log_file_prefix + "_" +
-      Socket.gethostname() + "_" + Time.now.strftime(@date_pattern)
-  end
-
-  ##
-  # Returns log file suffix, which will vary depending on whether gzip is
-  # enabled.
-  def get_suffix
-    return @gzip ? ".log.gz" : ".log"
-  end
-
-  ##
-  # Returns full path to the log file based on global variables (like
-  # current_base_path) and configuration options (max file size and gzip
-  # enabled).
-  def get_full_path
-    if @max_file_size_kbytes > 0
-      return @current_base_path + ".part" + ("%03d" % @size_counter) + get_suffix()
-    else
-      return @current_base_path + get_suffix()
-    end
-  end
-
-  ##
-  # Returns latest part number for a base path. This method checks all existing
-  # log files in order to find the highest part number, so this file can be used
-  # for appending log events.
-  #
-  # Only applicable if max file size is enabled.
-  def get_latest_part_number(base_path)
-    part_numbers = Dir.glob(base_path + ".part*" + get_suffix()).map do |item|
-      match = /^.*\.part(?<part_num>\d+)#{get_suffix()}$/.match(item)
-      next if match.nil?
-      match[:part_num].to_i
-    end
-
-    return part_numbers.max if part_numbers.any?
-    0
   end
 
   ##
   # Opens current log file and updates @temp_file with an instance of IOWriter.
   # This method also adds file to the upload queue.
   def open_current_file()
-    path = get_full_path()
+    path = @path_factory.current_path
+
     stat = File.stat(path) rescue nil
     if stat and stat.ftype == "fifo" and RUBY_PLATFORM == "java"
       fd = java.io.FileWriter.new(java.io.File.new(path))
@@ -366,37 +341,13 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
     end
   end
 
-  ##
-  # Opens log file on plugin initialization, trying to resume from an existing
-  # file. If max file size is enabled, find the highest part number and resume
-  # from it.
-  def initialize_current_log
-    @current_base_path = get_base_path
-    if @max_file_size_kbytes > 0
-      @size_counter = get_latest_part_number(@current_base_path)
-      @logger.debug("GCS: resuming from latest part.",
-                    :part => @size_counter)
-    end
-    open_current_file()
-  end
-
   ##
   # Generates new log file name based on configuration options and opens log
   # file. If max file size is enabled, part number if incremented in case the
   # the base log file name is the same (e.g. log file was not rolled given the
   # date pattern).
   def initialize_next_log
-    new_base_path = get_base_path
-    if @max_file_size_kbytes > 0
-      @size_counter = @current_base_path == new_base_path ? @size_counter + 1 : 0
-      @logger.debug("GCS: opening next log file.",
-                    :filename => @current_base_path,
-                    :part => @size_counter)
-    else
-      @logger.debug("GCS: opening next log file.",
-                    :filename => @current_base_path)
-    end
-    @current_base_path = new_base_path
+    @path_factory.rotate_path!
     open_current_file()
   end
 

data/logstash-output-google_cloud_storage.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-google_cloud_storage'
-  s.version         = '3.0.5'
-  s.licenses        = ['Apache License (2.0)']
+  s.version         = '3.1.0'
+  s.licenses        = ['Apache-2.0']
   s.summary         = "plugin to upload log events to Google Cloud Storage (GCS)"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors         = ["Elastic"]

data/spec/outputs/gcs/path_factory_spec.rb

@@ -0,0 +1,188 @@
+# encoding: utf-8
+require 'logstash/outputs/gcs/path_factory'
+
+describe LogStash::Outputs::Gcs::PathFactory do
+  describe '#initialize' do
+    it 'includes optional fields if requested' do
+      pf = LogStash::Outputs::Gcs::PathFactoryBuilder.build do |builder|
+        builder.set_directory 'path/to/directory'
+        builder.set_prefix 'prefix'
+        builder.set_include_host true
+        builder.set_date_pattern ''
+        builder.set_include_part true
+        builder.set_include_uuid true
+        builder.set_is_gzipped true
+      end
+
+      vars = {
+          prefix: 'prefix',
+          host: 'hostname',
+          date: '2018-01-01',
+          uuid: '00000000-0000-0000-0000-000000000000',
+          partf: '333'
+      }
+
+      expected = 'prefix_hostname_2018-01-01.part333.00000000-0000-0000-0000-000000000000.log.gz'
+      expected = File.join('path/to/directory', expected)
+
+      actual = pf.current_path(vars)
+
+      expect(actual).to eq(expected)
+    end
+
+    it 'excludes optional fields if not requested' do
+      pf = LogStash::Outputs::Gcs::PathFactoryBuilder.build do |builder|
+        builder.set_directory 'path/to/directory'
+        builder.set_prefix 'prefix'
+        builder.set_include_host false
+        builder.set_date_pattern ''
+        builder.set_include_part false
+        builder.set_include_uuid false
+        builder.set_is_gzipped false
+      end
+
+      vars = {
+          prefix: 'prefix',
+          host: 'hostname',
+          date: '2018-01-01',
+          uuid: '00000000-0000-0000-0000-000000000000',
+          partf: '333'
+      }
+
+      expected = 'prefix_2018-01-01.log'
+      expected = File.join('path/to/directory', expected)
+
+      actual = pf.current_path(vars)
+
+      expect(actual).to eq(expected)
+    end
+
+    it 'loads a path immediately' do
+      pf = LogStash::Outputs::Gcs::PathFactoryBuilder.build do |builder|
+        builder.set_directory ''
+        builder.set_prefix ''
+        builder.set_include_host false
+        builder.set_date_pattern ''
+        builder.set_include_part false
+        builder.set_include_uuid false
+        builder.set_is_gzipped false
+      end
+
+      expect(pf.current_path).to_not eq(nil)
+    end
+
+    it 'recovers the starting part number' do
+      contents = ['pre_date.part009.log.gz', 'pre_date.part091.log.gz', 'pre_date.part000.log.gz']
+
+      allow(::File).to receive(:directory?).with('dir').and_return(true)
+      allow(Dir).to receive(:glob).and_return(contents)
+
+      pf = LogStash::Outputs::Gcs::PathFactoryBuilder.build do |builder|
+        builder.set_directory 'dir'
+        builder.set_prefix 'pre'
+        builder.set_include_host false
+        builder.set_date_pattern 'date'
+        builder.set_include_part true
+        builder.set_include_uuid false
+        builder.set_is_gzipped false
+      end
+
+      expect(pf.current_path).to include('part092')
+    end
+  end
+
+  describe 'rotate_path!' do
+    it 'increments the part number if the base has not changed' do
+      pf = LogStash::Outputs::Gcs::PathFactoryBuilder.build do |builder|
+        builder.set_directory 'dir'
+        builder.set_prefix 'pre'
+        builder.set_include_host false
+        builder.set_date_pattern 'date'
+        builder.set_include_part true
+        builder.set_include_uuid false
+        builder.set_is_gzipped false
+      end
+
+      expect(pf.current_path).to eq(File.join('dir', 'pre_date.part000.log'))
+
+      pf.rotate_path!
+      expect(pf.current_path).to eq(File.join('dir', 'pre_date.part001.log'))
+    end
+
+    it 'resets the part number if the base has changed' do
+      pf = LogStash::Outputs::Gcs::PathFactoryBuilder.build do |builder|
+        builder.set_directory 'dir'
+        builder.set_prefix 'pre'
+        builder.set_include_host false
+        builder.set_date_pattern '%N'
+        builder.set_include_part true
+        builder.set_include_uuid false
+        builder.set_is_gzipped false
+      end
+      expect(pf.current_path).to include('part000')
+
+      pf.rotate_path!
+      expect(pf.current_path).to include('part000')
+    end
+
+    it 'returns the current_path' do
+      pf = LogStash::Outputs::Gcs::PathFactoryBuilder.build do |builder|
+        builder.set_directory 'dir'
+        builder.set_prefix 'pre'
+        builder.set_include_host false
+        builder.set_date_pattern 'date'
+        builder.set_include_part true
+        builder.set_include_uuid false
+        builder.set_is_gzipped false
+      end
+      after = pf.rotate_path!
+      expect(after).to eq(File.join('dir', 'pre_date.part001.log'))
+    end
+  end
+
+  describe 'should_rotate?' do
+    it 'returns false when the times in the bases are the same' do
+      pf = LogStash::Outputs::Gcs::PathFactoryBuilder.build do |builder|
+        builder.set_directory ''
+        builder.set_prefix ''
+        builder.set_include_host false
+        builder.set_date_pattern ''
+        builder.set_include_part false
+        builder.set_include_uuid false
+        builder.set_is_gzipped false
+      end
+      sleep 0.1
+      expect(pf.should_rotate?).to eq(false)
+    end
+
+    it 'returns true when the times in the bases are different' do
+      pf = LogStash::Outputs::Gcs::PathFactoryBuilder.build do |builder|
+        builder.set_directory ''
+        builder.set_prefix ''
+        builder.set_include_host false
+        builder.set_date_pattern '%N'
+        builder.set_include_part false
+        builder.set_include_uuid false
+        builder.set_is_gzipped false
+      end
+      sleep 0.1
+      expect(pf.should_rotate?).to eq(true)
+    end
+  end
+
+  describe 'current_path' do
+    it 'joins the directory and filename' do
+      pf = LogStash::Outputs::Gcs::PathFactoryBuilder.build do |builder|
+        builder.set_directory 'dir'
+        builder.set_prefix 'pre'
+        builder.set_include_host false
+        builder.set_date_pattern 'date'
+        builder.set_include_part false
+        builder.set_include_uuid false
+        builder.set_is_gzipped false
+      end
+
+      expect(pf.current_path).to eq(File.join('dir', 'pre_date.log'))
+    end
+  end
+end

data/spec/outputs/google_cloud_storage_spec.rb

@@ -32,7 +32,8 @@ describe LogStash::Outputs::GoogleCloudStorage do
       allow(subject).to receive(:new_upload_queue).and_return(upload_queue)
       subject.send(:initialize_upload_queue)
       subject.send(:initialize_temp_directory)
-      subject.send(:initialize_current_log)
+      subject.send(:initialize_path_factory)
+      subject.send(:open_current_file)
       current_file = upload_queue.pop
       File.write(current_file, content) if content
       upload_queue.push(current_file)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-google_cloud_storage
 version: !ruby/object:Gem::Version
-  version: 3.0.5
+  version: 3.1.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-04-06 00:00:00.000000000 Z
+date: 2018-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -115,13 +115,15 @@ files:
 - NOTICE.TXT
 - README.md
 - docs/index.asciidoc
+- lib/logstash/outputs/gcs/path_factory.rb
 - lib/logstash/outputs/google_cloud_storage.rb
 - logstash-output-google_cloud_storage.gemspec
+- spec/outputs/gcs/path_factory_spec.rb
 - spec/outputs/google_cloud_storage_spec.rb
 - spec/spec_helper.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
-- Apache License (2.0)
+- Apache-2.0
 metadata:
   logstash_plugin: 'true'
   logstash_group: output
@@ -141,10 +143,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: plugin to upload log events to Google Cloud Storage (GCS)
 test_files:
+- spec/outputs/gcs/path_factory_spec.rb
 - spec/outputs/google_cloud_storage_spec.rb
 - spec/spec_helper.rb