logstash-output-googlecloudstorage 0.1.0-java

data/lib/logstash/outputs/gcs/log_rotate.rb

@@ -0,0 +1,77 @@
+# encoding: utf-8
+require 'logstash/outputs/gcs/temp_log_file'
+require 'concurrent'
+
+module LogStash
+  module Outputs
+    module Gcs
+      class LogRotate
+        def initialize(path_factory, max_file_size_bytes, gzip, flush_interval_secs, gzip_encoded=false)
+          @path_factory = path_factory
+          @max_file_size_bytes = max_file_size_bytes
+          @gzip = gzip
+          @flush_interval_secs = flush_interval_secs
+          @gzip_encoded = gzip_encoded
+
+          @lock = Concurrent::ReentrantReadWriteLock.new
+          @rotate_callback = nil
+
+          rotate_log!
+        end
+
+        # writeln writes a message and carriage-return character to the open
+        # log file, rotating and syncing it if necessary.
+        #
+        # nil messages do not get written, but may cause the log to rotate
+        def writeln(message=nil)
+          @lock.with_write_lock do
+            rotate_log! if should_rotate?
+
+            @temp_file.write(message, "\n") unless message.nil?
+
+            @temp_file.fsync if @temp_file.time_since_sync >= @flush_interval_secs
+          end
+        end
+
+        # rotate_log! closes the current log (if it exists), notifies the
+        # handler, rolls the path over and opens a new log.
+        #
+        # Invariant: the old log will ALWAYS be closed and a new one will
+        # ALWAYS be open at the completion of this function.
+        def rotate_log!
+          @lock.with_write_lock do
+            unless @temp_file.nil?
+              @temp_file.close!
+              @rotate_callback.call(@temp_file.path) unless @rotate_callback.nil?
+            end
+
+            @path_factory.rotate_path!
+
+            path = @path_factory.current_path
+            @temp_file = LogStash::Outputs::Gcs::LogFileFactory.create(path, @gzip, true, @gzip_encoded)
+          end
+        end
+
+        # on_rotate sets a handler to be called when the log gets rotated.
+        # The handler receives the path to the rotated out log as a string.
+        def on_rotate(&block)
+          @lock.with_write_lock do
+            @rotate_callback = block
+          end
+        end
+
+        private
+
+        def should_rotate?
+          @lock.with_read_lock do
+            path_changed = @path_factory.should_rotate?
+            rotate_on_size = @max_file_size_bytes > 0
+            too_big = @temp_file.size >= @max_file_size_bytes
+
+            path_changed || (rotate_on_size && too_big)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/gcs/path_factory.rb

@@ -0,0 +1,119 @@
+# encoding: utf-8
+require 'thread'
+
+module LogStash
+  module Outputs
+    module Gcs
+      # PathFactory creates paths for rotating files.
+      class PathFactory
+        def initialize(directory, prefix, include_host, date_pattern, include_part, include_uuid, is_gzipped)
+          @path_lock = Mutex.new
+
+          pattern = prefix
+          pattern += '_%{host}' if include_host
+          pattern += '_%{date}'
+          @base_pattern = pattern
+
+          pattern += '.part%{partf}' if include_part
+          pattern += '.%{uuid}' if include_uuid
+          pattern += '.log'
+          pattern += '.gz' if is_gzipped
+          @pattern = pattern
+
+          @prefix = prefix
+          @directory = directory
+          @date_pattern = date_pattern
+
+          @part_number = starting_part
+          @current = template_variables
+        end
+
+        # Rotates the path to the next one in sequence. If the path has a part number
+        # and the base path (date/hostname) haven't changed the part number is incremented.
+        # Returns the path that was rotated out
+        def rotate_path!
+          last_path = current_path
+
+          @path_lock.synchronize {
+            @part_number = (next_base == current_base) ? @part_number + 1 : 0
+            @current = template_variables
+          }
+
+          last_path
+        end
+
+        # Checks if the file is ready to rotate because the timestamp changed.
+        def should_rotate?
+          @path_lock.synchronize {
+            next_base != current_base
+          }
+        end
+
+        # Returns the full path to the current file including parent directory.
+        def current_path(vars=nil)
+          @path_lock.synchronize {
+            filename = @pattern % (vars || @current)
+            ::File.join(@directory, filename)
+          }
+        end
+
+        private
+
+        # search through the directory for a file with the same base, and if it exists,
+        # set our part to be the max + 1 so we don't clobber existing files.
+        def starting_part
+          return 0 unless ::File.directory? @directory
+
+          base_path = ::File.join(@directory, next_base)
+
+          part_numbers = Dir.glob(base_path + '.part*').map do |item|
+            match = /^.*\.part(?<part_num>\d+).*$/.match(item)
+            next if match.nil?
+            match[:part_num].to_i
+          end
+
+          part_numbers.any? ? part_numbers.max + 1 : 0
+        end
+
+        def template_variables
+          {
+              prefix: @prefix,
+              host: Socket.gethostname,
+              date: Time.now.strftime(@date_pattern),
+              partf: '%03d' % @part_number,
+              uuid: SecureRandom.uuid
+          }
+        end
+
+        def next_base
+          @base_pattern % template_variables
+        end
+
+        def current_base
+          @base_pattern % @current
+        end
+      end
+
+      # PathFactoryBuilder makes the long PathFactory constructor chain more readable.
+      class PathFactoryBuilder
+        def self.build
+          builder = new
+          yield builder
+          builder.build_path_factory
+        end
+
+        def self.builder_setter(*names)
+          names.each do |name|
+            define_method("set_#{name}") {|arg| instance_variable_set("@#{name}", arg)}
+          end
+        end
+
+        builder_setter :directory, :prefix, :include_host, :date_pattern, :include_part, :include_uuid, :is_gzipped
+
+        def build_path_factory
+          PathFactory.new(@directory, @prefix, @include_host, @date_pattern, @include_part, @include_uuid, @is_gzipped)
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/gcs/temp_log_file.rb

@@ -0,0 +1,111 @@
+# encoding: utf-8
+require 'zlib'
+require 'concurrent'
+require 'time'
+
+module LogStash
+  module Outputs
+    module Gcs
+      # LogFileFactory creates a LogFile according to user specification
+      # optionally gzipping it and creating mutexes around modification
+      # points.
+      class LogFileFactory
+        def self.create(path, gzip, synchronize=true, gzip_encoded=false)
+          lf = LogStash::Outputs::Gcs::PlainLogFile.new(path)
+          lf = LogStash::Outputs::Gcs::GzipLogFile.new(lf) if gzip
+          lf = LogStash::Outputs::Gcs::GzipLogFile.new(lf) if gzip_encoded
+          lf = LogStash::Outputs::Gcs::SynchronizedLogFile.new(lf) if synchronize
+
+          lf
+        end
+      end
+
+      # PlainLogFile writes events to a plain text file.
+      class PlainLogFile
+        attr_reader :path, :fd
+
+        def initialize(path)
+          @path = path
+          @fd = ::File.new(path, 'a+')
+          @last_sync = Time.now
+        end
+
+        def write(*contents)
+          contents.each { |c| @fd.write(c) }
+        end
+
+        def fsync
+          @fd.fsync
+          @last_sync = Time.now
+        end
+
+        def close!
+          @fd.fsync
+          @fd.close
+        end
+
+        def size
+          ::File.stat(@path).size
+        end
+
+        def time_since_sync
+          Time.now - @last_sync
+        end
+      end
+
+      # GzipLogFile wraps another log file and writes events through it.
+      class GzipLogFile
+        attr_reader :fd
+
+        def initialize(child)
+          @child = child
+          @fd = Zlib::GzipWriter.new(child.fd)
+        end
+
+        def write(*contents)
+          contents.each { |c| @fd.write(c) }
+        end
+
+        def fsync
+          @fd.flush
+          @child.fsync
+        end
+
+        def close!
+          fsync
+          # The Gzip writer closes the underlying IO after
+          # appending the Gzip footer.
+          @fd.close
+        end
+
+        def method_missing(method_name, *args, &block)
+          @child.send(method_name, *args, &block)
+        end
+      end
+
+      # SynchronizedLogFile wraps another log file and uses reentrant locks
+      # around its methods to prevent concurrent modification.
+      class SynchronizedLogFile
+        def initialize(child)
+          @child = child
+          @lock = Concurrent::ReentrantReadWriteLock.new
+        end
+
+        def time_since_sync
+          @lock.with_read_lock { @child.time_since_sync }
+        end
+
+        def path
+          @lock.with_read_lock { @child.path }
+        end
+
+        def method_missing(method_name, *args, &block)
+          # unless otherwise specified, get a write lock
+          @lock.with_write_lock do
+            @child.send(method_name, *args, &block)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/gcs/worker_pool.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+require 'thread'
+require 'concurrent'
+
+module LogStash
+  module Outputs
+    module Gcs
+      # WorkerPool creates a pool of workers that can handle jobs.
+      class WorkerPool
+        attr_reader :workers
+
+        def initialize(max_threads, synchronous=false)
+          @synchronous = synchronous
+
+          # set queue depth to the be the same as the number of threads so
+          # there's at most one pending job each when the plugin quits
+          @workers = Concurrent::ThreadPoolExecutor.new(
+            min_threads: 1,
+            max_threads: max_threads,
+            max_queue: max_threads,
+            fallback_policy: :caller_runs
+          )
+        end
+
+        # Submits a job to the worker pool, raises an error if the pool has
+        # already been stopped.
+        def post(&block)
+          raise 'Pool already stopped' unless @workers.running?
+
+          if @synchronous
+            block.call
+          else
+            @workers.post do
+              block.call
+            end
+          end
+        end
+
+        # Stops the worker pool
+        def stop!
+          @workers.shutdown
+          @workers.wait_for_termination
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/google_cloud_storage.rb

@@ -0,0 +1,304 @@
+# [source,txt]
+# -----
+# encoding: utf-8
+# Author: Rodrigo De Castro <rdc@google.com>
+# Date: 2013-09-20
+#
+# Copyright 2013 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# -----
+require "logstash/outputs/base"
+require "logstash/outputs/gcs/path_factory"
+require "logstash/outputs/gcs/worker_pool"
+require "logstash/outputs/gcs/log_rotate"
+require "logstash/namespace"
+require "logstash/json"
+require "stud/interval"
+require "thread"
+require "zlib"
+
+# Summary: plugin to upload log events to Google Cloud Storage (GCS), rolling
+# files based on the date pattern provided as a configuration setting. Events
+# are written to files locally and, once file is closed, this plugin uploads
+# it to the configured bucket.
+#
+# For more info on Google Cloud Storage, please go to:
+# https://cloud.google.com/products/cloud-storage
+#
+# In order to use this plugin, a Google service account must be used. For
+# more information, please refer to:
+# https://developers.google.com/storage/docs/authentication#service_accounts
+#
+# Recommendation: experiment with the settings depending on how much log
+# data you generate, so the uploader can keep up with the generated logs.
+# Using gzip output can be a good option to reduce network traffic when
+# uploading the log files and in terms of storage costs as well.
+#
+# USAGE:
+# This is an example of logstash config:
+#
+# [source,json]
+# --------------------------
+# output {
+#    google_cloud_storage {
+#      bucket => "my_bucket"                                     (required)
+#      key_path => "/path/to/privatekey.p12"                     (required)
+#      key_password => "notasecret"                              (optional)
+#      service_account => "1234@developer.gserviceaccount.com"   (required)
+#      temp_directory => "/tmp/logstash-gcs"                     (optional)
+#      log_file_prefix => "logstash_gcs"                         (optional)
+#      max_file_size_kbytes => 1024                              (optional)
+#      output_format => "plain"                                  (optional)
+#      date_pattern => "%Y-%m-%dT%H:00"                          (optional)
+#      flush_interval_secs => 2                                  (optional)
+#      gzip => false                                             (optional)
+#      gzip_content_encoding => false                            (optional)
+#      uploader_interval_secs => 60                              (optional)
+#      upload_synchronous => false                               (optional)
+#    }
+# }
+# --------------------------
+#
+# Improvements TODO list:
+# * Support logstash event variables to determine filename.
+# * Turn Google API code into a Plugin Mixin (like AwsConfig).
+# * There's no recover method, so if logstash/plugin crashes, files may not
+# be uploaded to GCS.
+# * Allow user to configure file name.
+class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
+  config_name "google_cloud_storage"
+
+  concurrency :single
+
+  # GCS bucket name, without "gs://" or any other prefix.
+  config :bucket, :validate => :string, :required => true
+
+  # GCS path to private key file.
+  config :key_path, :validate => :string, :required => true
+
+  # GCS private key password.
+  config :key_password, :validate => :string, :default => "notasecret"
+
+  # GCS service account.
+  config :service_account, :validate => :string, :required => true
+
+  # Directory where temporary files are stored.
+  # Defaults to /tmp/logstash-gcs-<random-suffix>
+  config :temp_directory, :validate => :string, :default => ""
+
+  # Log file prefix. Log file will follow the format:
+  # <prefix>_hostname_date<.part?>.log
+  config :log_file_prefix, :validate => :string, :default => "logstash_gcs"
+
+  # Sets max file size in kbytes. 0 disable max file check.
+  config :max_file_size_kbytes, :validate => :number, :default => 10000
+
+  # The event format you want to store in files. Defaults to plain text.
+  config :output_format, :validate => [ "json", "plain" ], :default => "plain"
+
+  # Time pattern for log file, defaults to hourly files.
+  # Must Time.strftime patterns: www.ruby-doc.org/core-2.0/Time.html#method-i-strftime
+  config :date_pattern, :validate => :string, :default => "%Y-%m-%dT%H:00"
+
+  # Flush interval in seconds for flushing writes to log files. 0 will flush
+  # on every message.
+  config :flush_interval_secs, :validate => :number, :default => 2
+
+  # Gzip output stream when writing events to log files, set
+  # `Content-Type` to `application/gzip` instead of `text/plain`, and
+  # use file suffix `.log.gz` instead of `.log`.
+  config :gzip, :validate => :boolean, :default => false
+
+  # Gzip output stream when writing events to log files and set
+  # `Content-Encoding` to `gzip`.
+  config :gzip_content_encoding, :validate => :boolean, :default => false
+
+  # Uploader interval when uploading new files to GCS. Adjust time based
+  # on your time pattern (for example, for hourly files, this interval can be
+  # around one hour).
+  config :uploader_interval_secs, :validate => :number, :default => 60
+
+  # Should the hostname be included in the file name?
+  config :include_hostname, :validate => :boolean, :default => true
+
+  # Should a UUID be included in the file name?
+  config :include_uuid, :validate => :boolean, :default => false
+
+  # When true, files are uploaded by the event processing thread as soon as a file is ready.
+  # When false, (the default behaviour), files will be uploaded in a dedicated thread.
+  #
+  # Enabling this option provides greater likelihood that all generated files will be
+  # to GCS, especially in the event of a graceful shutdown of logstash, such as when an
+  # input plugin reaches the end of events. This comes at the price of introducing delays
+  # in the event processing pipeline as files are uploaded.
+  #
+  # When this feature is enabled, the uploader_interval_secs option has no effect.
+  config :upload_synchronous, :validate => :boolean, :default => false
+
+  config :max_concurrent_uploads, :validate  => :number, :default => 5
+
+  public
+  def register
+    @logger.debug('Registering Google Cloud Storage plugin')
+
+    @workers = LogStash::Outputs::Gcs::WorkerPool.new(@max_concurrent_uploads, @upload_synchronous)
+    initialize_temp_directory
+    initialize_path_factory
+    initialize_log_rotater
+
+    initialize_google_client
+
+    start_uploader
+
+    @content_type = @gzip ? 'application/gzip' : 'text/plain'
+    @content_encoding = @gzip_content_encoding ? 'gzip' : 'identity'
+  end
+
+  # Method called for each log event. It writes the event to the current output
+  # file, flushing depending on flush interval configuration.
+  public
+  def receive(event)
+    @logger.debug('Received event', :event => event)
+
+    if @output_format == 'json'
+      message = LogStash::Json.dump(event.to_hash)
+    else
+      message = event.to_s
+    end
+
+    @log_rotater.writeln(message)
+  end
+
+  public
+  def close
+    @logger.debug('Stopping the plugin, uploading the remaining files.')
+    Thread.kill(@uploader_thread) unless @uploader_thread.nil?
+
+    # Force rotate the log. If it contains data it will be submitted
+    # to the work pool and will be uploaded before the plugin stops.
+    @log_rotater.rotate_log!
+    @workers.stop!
+  end
+
+  private
+
+  ##
+  # Creates temporary directory, if it does not exist.
+  #
+  # A random suffix is appended to the temporary directory
+  def initialize_temp_directory
+    require "stud/temporary"
+
+    if @temp_directory.empty?
+      @temp_directory = Stud::Temporary.directory('logstash-gcs')
+    end
+
+    FileUtils.mkdir_p(@temp_directory) unless File.directory?(@temp_directory)
+
+    @logger.info("Using temporary directory: #{@temp_directory}")
+  end
+
+  def initialize_path_factory
+    @path_factory = LogStash::Outputs::Gcs::PathFactoryBuilder.build do |builder|
+      builder.set_directory @temp_directory
+      builder.set_prefix @log_file_prefix
+      builder.set_include_host @include_hostname
+      builder.set_date_pattern @date_pattern
+      builder.set_include_part(@max_file_size_kbytes > 0)
+      builder.set_include_uuid @include_uuid
+      builder.set_is_gzipped @gzip
+    end
+  end
+
+  # start_uploader periodically sends flush events through the log rotater
+  def start_uploader
+    @uploader_thread = Thread.new do
+      Stud.interval(@uploader_interval_secs) do
+        @log_rotater.writeln(nil)
+      end
+    end
+  end
+
+  ##
+  # Initializes Google Client instantiating client and authorizing access.
+  def initialize_google_client
+    require "google/api_client"
+    require "openssl"
+
+    @client = Google::APIClient.new(:application_name =>
+                                    'Logstash Google Cloud Storage output plugin',
+                                    :application_version => '0.1')
+    @storage = @client.discovered_api('storage', 'v1')
+
+    key = Google::APIClient::PKCS12.load_key(@key_path, @key_password)
+    service_account = Google::APIClient::JWTAsserter.new(@service_account,
+                                                         'https://www.googleapis.com/auth/devstorage.read_write',
+                                                         key)
+    @client.authorization = service_account.authorize
+  end
+
+  ##
+  # Uploads a local file to the configured bucket.
+  def upload_object(filename)
+    begin
+      @logger.debug("GCS: upload object.", :filename => filename)
+
+      media = Google::APIClient::UploadIO.new(filename, @content_type)
+      metadata_insert_result = @client.execute(:api_method => @storage.objects.insert,
+                                               :parameters => {
+                                                 'uploadType' => 'multipart',
+                                                 'bucket' => @bucket,
+                                                 'contentEncoding' => @content_encoding,
+                                                 'name' => File.basename(filename)
+                                               },
+                                               :body_object => {contentType: @content_type},
+                                               :media => media)
+      contents = metadata_insert_result.data
+      @logger.debug("GCS: multipart insert",
+                    :object => contents.name,
+                    :self_link => contents.self_link)
+    rescue => e
+      @logger.error("GCS: failed to upload file", :exception => e)
+      # TODO(rdc): limit retries?
+      sleep 1
+      retry
+    end
+  end
+
+  def upload_and_delete(filename)
+    file_size = File.stat(filename).size
+
+    if file_size > 0
+      upload_object(filename)
+    else
+      @logger.debug('File size is zero, skip upload.', :filename => filename)
+    end
+
+    @logger.debug('Delete local temporary file', :filename => filename)
+    File.delete(filename)
+  end
+
+  def initialize_log_rotater
+    max_file_size = @max_file_size_kbytes * 1024
+    @log_rotater = LogStash::Outputs::Gcs::LogRotate.new(@path_factory, max_file_size, @gzip, @flush_interval_secs, @gzip_content_encoding)
+
+    @log_rotater.on_rotate do |filename|
+      @logger.info("Rotated out file: #{filename}")
+      @workers.post do
+        upload_and_delete(filename)
+      end
+    end
+  end
+  attr_accessor :active
+end