logstash-output-google_cloud_storage 4.0.0-java

data/lib/logstash-output-google_cloud_storage_jars.rb

@@ -0,0 +1,35 @@
+# AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
+
+require 'jar_dependencies'
+require_jar('com.google.cloud', 'google-cloud-storage', '1.39.0')
+require_jar('com.google.api', 'api-common', '1.7.0')
+require_jar('com.google.api-client', 'google-api-client', '1.24.1')
+require_jar('com.google.api', 'gax', '1.30.0')
+require_jar('com.google.api', 'gax-httpjson', '0.47.0')
+require_jar('com.google.api.grpc', 'proto-google-common-protos', '1.12.0')
+require_jar('com.google.api.grpc', 'proto-google-iam-v1', '0.12.0')
+require_jar('com.google.apis', 'google-api-services-storage', 'v1-rev135-1.24.1')
+require_jar('com.google.auth', 'google-auth-library-credentials', '0.10.0')
+require_jar('com.google.auth', 'google-auth-library-oauth2-http', '0.10.0')
+require_jar('com.google.cloud', 'google-cloud-core', '1.39.0')
+require_jar('com.google.cloud', 'google-cloud-core-http', '1.39.0')
+require_jar('com.google.code.findbugs', 'jsr305', '3.0.2')
+require_jar('com.google.code.gson', 'gson', '2.7')
+require_jar('com.google.guava', 'guava', '20.0')
+require_jar('com.google.http-client', 'google-http-client', '1.24.1')
+require_jar('com.google.http-client', 'google-http-client-appengine', '1.24.1')
+require_jar('com.google.http-client', 'google-http-client-jackson', '1.24.1')
+require_jar('com.google.http-client', 'google-http-client-jackson2', '1.24.1')
+require_jar('com.google.oauth-client', 'google-oauth-client', '1.24.1')
+require_jar('com.google.protobuf', 'protobuf-java', '3.6.0')
+require_jar('com.google.protobuf', 'protobuf-java-util', '3.6.0')
+require_jar('commons-codec', 'commons-codec', '1.9')
+require_jar('commons-logging', 'commons-logging', '1.2')
+require_jar('io.grpc', 'grpc-context', '1.12.0')
+require_jar('io.opencensus', 'opencensus-api', '0.15.0')
+require_jar('io.opencensus', 'opencensus-contrib-http-util', '0.15.0')
+require_jar('joda-time', 'joda-time', '2.9.2')
+require_jar('org.apache.httpcomponents', 'httpclient', '4.5.3')
+require_jar('org.apache.httpcomponents', 'httpcore', '4.4.6')
+require_jar('org.codehaus.jackson', 'jackson-core-asl', '1.9.11')
+require_jar('org.threeten', 'threetenbp', '1.3.3')

data/lib/logstash/outputs/gcs/client.rb

@@ -0,0 +1,88 @@
+require 'thread'
+require 'java'
+require 'logstash-output-google_cloud_storage_jars.rb'
+
+java_import 'com.google.api.gax.rpc.FixedHeaderProvider'
+java_import 'com.google.api.gax.retrying.RetrySettings'
+java_import 'com.google.auth.oauth2.GoogleCredentials'
+java_import 'com.google.cloud.storage.BlobInfo'
+java_import 'com.google.cloud.storage.StorageOptions'
+java_import 'java.io.FileInputStream'
+java_import 'org.threeten.bp.Duration'
+
+module LogStash
+  module Outputs
+    module Gcs
+      class Client
+        def initialize(bucket, json_key_path, logger)
+          @logger = logger
+          @bucket = bucket
+
+          # create client
+          @storage = initialize_storage(json_key_path)
+        end
+
+        def upload_object(file_path, content_encoding, content_type)
+          input = FileInputStream.new(file_path)
+
+          blob_name = ::File.basename(file_path)
+          blob_info = com.google.cloud.storage.BlobInfo.newBuilder(@bucket, blob_name)
+                          .setContentEncoding(content_encoding)
+                          .setContentType(content_type)
+                          .build
+
+          @logger.info("Uploading file to #{@bucket}/#{blob_name}")
+          @storage.create(blob_info, input)
+
+          input.close
+          @logger.info("Uploaded file to #{@bucket}/#{blob_name}")
+        end
+
+        def initialize_storage(json_key_path)
+          @logger.info("Initializing Google API client, key: #{json_key_path}")
+
+          StorageOptions.newBuilder
+              .setCredentials(credentials(json_key_path))
+              .setHeaderProvider(http_headers)
+              .setRetrySettings(retry_settings)
+              .build
+              .getService
+        end
+
+        private
+
+        def credentials(json_key_path)
+          return GoogleCredentials.getApplicationDefault() if nil_or_empty?(json_key_path)
+
+          key_file = FileInputStream.new(json_key_path)
+          GoogleCredentials.fromStream(key_file)
+        end
+
+        def http_headers
+          FixedHeaderProvider.create({ 'User-Agent' => 'Elastic/logstash-output-google_cloud_storage' })
+        end
+
+        def retry_settings
+          # backoff values taken from com.google.api.client.util.ExponentialBackOff
+          RetrySettings.newBuilder()
+              .setInitialRetryDelay(Duration.ofMillis(500))
+              .setRetryDelayMultiplier(1.5)
+              .setMaxRetryDelay(Duration.ofSeconds(60))
+              .setInitialRpcTimeout(Duration.ofSeconds(20))
+              .setRpcTimeoutMultiplier(1.5)
+              .setMaxRpcTimeout(Duration.ofSeconds(20))
+              .setTotalTimeout(Duration.ofMinutes(15))
+              .build
+        end
+
+        def api_debug(message, dataset, table)
+          @logger.debug(message, dataset: dataset, table: table)
+        end
+
+        def nil_or_empty?(param)
+          param.nil? || param.empty?
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/gcs/log_rotate.rb

@@ -0,0 +1,77 @@
+# encoding: utf-8
+require 'logstash/outputs/gcs/temp_log_file'
+require 'concurrent'
+
+module LogStash
+  module Outputs
+    module Gcs
+      class LogRotate
+        def initialize(path_factory, max_file_size_bytes, gzip, flush_interval_secs, gzip_encoded=false)
+          @path_factory = path_factory
+          @max_file_size_bytes = max_file_size_bytes
+          @gzip = gzip
+          @flush_interval_secs = flush_interval_secs
+          @gzip_encoded = gzip_encoded
+
+          @lock = Concurrent::ReentrantReadWriteLock.new
+          @rotate_callback = nil
+
+          rotate_log!
+        end
+
+        # writeln writes a message and carriage-return character to the open
+        # log file, rotating and syncing it if necessary.
+        #
+        # nil messages do not get written, but may cause the log to rotate
+        def writeln(message=nil)
+          @lock.with_write_lock do
+            rotate_log! if should_rotate?
+
+            @temp_file.write(message, "\n") unless message.nil?
+
+            @temp_file.fsync if @temp_file.time_since_sync >= @flush_interval_secs
+          end
+        end
+
+        # rotate_log! closes the current log (if it exists), notifies the
+        # handler, rolls the path over and opens a new log.
+        #
+        # Invariant: the old log will ALWAYS be closed and a new one will
+        # ALWAYS be open at the completion of this function.
+        def rotate_log!
+          @lock.with_write_lock do
+            unless @temp_file.nil?
+              @temp_file.close!
+              @rotate_callback.call(@temp_file.path) unless @rotate_callback.nil?
+            end
+
+            @path_factory.rotate_path!
+
+            path = @path_factory.current_path
+            @temp_file = LogStash::Outputs::Gcs::LogFileFactory.create(path, @gzip, true, @gzip_encoded)
+          end
+        end
+
+        # on_rotate sets a handler to be called when the log gets rotated.
+        # The handler receives the path to the rotated out log as a string.
+        def on_rotate(&block)
+          @lock.with_write_lock do
+            @rotate_callback = block
+          end
+        end
+
+        private
+
+        def should_rotate?
+          @lock.with_read_lock do
+            path_changed = @path_factory.should_rotate?
+            rotate_on_size = @max_file_size_bytes > 0
+            too_big = @temp_file.size >= @max_file_size_bytes
+
+            path_changed || (rotate_on_size && too_big)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/gcs/path_factory.rb

@@ -0,0 +1,119 @@
+# encoding: utf-8
+require 'thread'
+
+module LogStash
+  module Outputs
+    module Gcs
+      # PathFactory creates paths for rotating files.
+      class PathFactory
+        def initialize(directory, prefix, include_host, date_pattern, include_part, include_uuid, is_gzipped)
+          @path_lock = Mutex.new
+
+          pattern = '%{prefix}'
+          pattern += '_%{host}' if include_host
+          pattern += '_%{date}'
+          @base_pattern = pattern
+
+          pattern += '.part%{partf}' if include_part
+          pattern += '.%{uuid}' if include_uuid
+          pattern += '.log'
+          pattern += '.gz' if is_gzipped
+          @pattern = pattern
+
+          @prefix = prefix
+          @directory = directory
+          @date_pattern = date_pattern
+
+          @part_number = starting_part
+          @current = template_variables
+        end
+
+        # Rotates the path to the next one in sequence. If the path has a part number
+        # and the base path (date/hostname) haven't changed the part number is incremented.
+        # Returns the path that was rotated out
+        def rotate_path!
+          last_path = current_path
+
+          @path_lock.synchronize {
+            @part_number = (next_base == current_base) ? @part_number + 1 : 0
+            @current = template_variables
+          }
+
+          last_path
+        end
+
+        # Checks if the file is ready to rotate because the timestamp changed.
+        def should_rotate?
+          @path_lock.synchronize {
+            next_base != current_base
+          }
+        end
+
+        # Returns the full path to the current file including parent directory.
+        def current_path(vars=nil)
+          @path_lock.synchronize {
+            filename = @pattern % (vars || @current)
+            ::File.join(@directory, filename)
+          }
+        end
+
+        private
+
+        # search through the directory for a file with the same base, and if it exists,
+        # set our part to be the max + 1 so we don't clobber existing files.
+        def starting_part
+          return 0 unless ::File.directory? @directory
+
+          base_path = ::File.join(@directory, next_base)
+
+          part_numbers = Dir.glob(base_path + '.part*').map do |item|
+            match = /^.*\.part(?<part_num>\d+).*$/.match(item)
+            next if match.nil?
+            match[:part_num].to_i
+          end
+
+          part_numbers.any? ? part_numbers.max + 1 : 0
+        end
+
+        def template_variables
+          {
+              prefix: @prefix,
+              host: Socket.gethostname,
+              date: Time.now.strftime(@date_pattern),
+              partf: '%03d' % @part_number,
+              uuid: SecureRandom.uuid
+          }
+        end
+
+        def next_base
+          @base_pattern % template_variables
+        end
+
+        def current_base
+          @base_pattern % @current
+        end
+      end
+
+      # PathFactoryBuilder makes the long PathFactory constructor chain more readable.
+      class PathFactoryBuilder
+        def self.build
+          builder = new
+          yield builder
+          builder.build_path_factory
+        end
+
+        def self.builder_setter(*names)
+          names.each do |name|
+            define_method("set_#{name}") {|arg| instance_variable_set("@#{name}", arg)}
+          end
+        end
+
+        builder_setter :directory, :prefix, :include_host, :date_pattern, :include_part, :include_uuid, :is_gzipped
+
+        def build_path_factory
+          PathFactory.new(@directory, @prefix, @include_host, @date_pattern, @include_part, @include_uuid, @is_gzipped)
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/gcs/temp_log_file.rb

@@ -0,0 +1,111 @@
+# encoding: utf-8
+require 'zlib'
+require 'concurrent'
+require 'time'
+
+module LogStash
+  module Outputs
+    module Gcs
+      # LogFileFactory creates a LogFile according to user specification
+      # optionally gzipping it and creating mutexes around modification
+      # points.
+      class LogFileFactory
+        def self.create(path, gzip, synchronize=true, gzip_encoded=false)
+          lf = LogStash::Outputs::Gcs::PlainLogFile.new(path)
+          lf = LogStash::Outputs::Gcs::GzipLogFile.new(lf) if gzip
+          lf = LogStash::Outputs::Gcs::GzipLogFile.new(lf) if gzip_encoded
+          lf = LogStash::Outputs::Gcs::SynchronizedLogFile.new(lf) if synchronize
+
+          lf
+        end
+      end
+
+      # PlainLogFile writes events to a plain text file.
+      class PlainLogFile
+        attr_reader :path, :fd
+
+        def initialize(path)
+          @path = path
+          @fd = ::File.new(path, 'a+')
+          @last_sync = Time.now
+        end
+
+        def write(*contents)
+          contents.each { |c| @fd.write(c) }
+        end
+
+        def fsync
+          @fd.fsync
+          @last_sync = Time.now
+        end
+
+        def close!
+          @fd.fsync
+          @fd.close
+        end
+
+        def size
+          ::File.stat(@path).size
+        end
+
+        def time_since_sync
+          Time.now - @last_sync
+        end
+      end
+
+      # GzipLogFile wraps another log file and writes events through it.
+      class GzipLogFile
+        attr_reader :fd
+
+        def initialize(child)
+          @child = child
+          @fd = Zlib::GzipWriter.new(child.fd)
+        end
+
+        def write(*contents)
+          contents.each { |c| @fd.write(c) }
+        end
+
+        def fsync
+          @fd.flush
+          @child.fsync
+        end
+
+        def close!
+          fsync
+          # The Gzip writer closes the underlying IO after
+          # appending the Gzip footer.
+          @fd.close
+        end
+
+        def method_missing(method_name, *args, &block)
+          @child.send(method_name, *args, &block)
+        end
+      end
+
+      # SynchronizedLogFile wraps another log file and uses reentrant locks
+      # around its methods to prevent concurrent modification.
+      class SynchronizedLogFile
+        def initialize(child)
+          @child = child
+          @lock = Concurrent::ReentrantReadWriteLock.new
+        end
+
+        def time_since_sync
+          @lock.with_read_lock { @child.time_since_sync }
+        end
+
+        def path
+          @lock.with_read_lock { @child.path }
+        end
+
+        def method_missing(method_name, *args, &block)
+          # unless otherwise specified, get a write lock
+          @lock.with_write_lock do
+            @child.send(method_name, *args, &block)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/gcs/worker_pool.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+require 'thread'
+require 'concurrent'
+
+module LogStash
+  module Outputs
+    module Gcs
+      # WorkerPool creates a pool of workers that can handle jobs.
+      class WorkerPool
+        attr_reader :workers
+
+        def initialize(max_threads, synchronous=false)
+          @synchronous = synchronous
+
+          # set queue depth to the be the same as the number of threads so
+          # there's at most one pending job each when the plugin quits
+          @workers = Concurrent::ThreadPoolExecutor.new(
+            min_threads: 1,
+            max_threads: max_threads,
+            max_queue: max_threads,
+            fallback_policy: :caller_runs
+          )
+        end
+
+        # Submits a job to the worker pool, raises an error if the pool has
+        # already been stopped.
+        def post(&block)
+          raise 'Pool already stopped' unless @workers.running?
+
+          if @synchronous
+            block.call
+          else
+            @workers.post do
+              block.call
+            end
+          end
+        end
+
+        # Stops the worker pool
+        def stop!
+          @workers.shutdown
+          @workers.wait_for_termination
+        end
+      end
+    end
+  end
+end