logstash-output-google_cloud_storage 4.0.0-java

data/lib/logstash/outputs/google_cloud_storage.rb

@@ -0,0 +1,276 @@
+# [source,txt]
+# -----
+# encoding: utf-8
+# Author: Rodrigo De Castro <rdc@google.com>
+# Date: 2013-09-20
+#
+# Copyright 2013 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# -----
+require 'logstash/outputs/gcs/client'
+require "logstash/outputs/base"
+require "logstash/outputs/gcs/path_factory"
+require "logstash/outputs/gcs/worker_pool"
+require "logstash/outputs/gcs/log_rotate"
+require "logstash/namespace"
+require "logstash/json"
+require "stud/interval"
+require "thread"
+require "zlib"
+
+# Summary: plugin to upload log events to Google Cloud Storage (GCS), rolling
+# files based on the date pattern provided as a configuration setting. Events
+# are written to files locally and, once file is closed, this plugin uploads
+# it to the configured bucket.
+#
+# For more info on Google Cloud Storage, please go to:
+# https://cloud.google.com/products/cloud-storage
+#
+# In order to use this plugin, a Google service account must be used. For
+# more information, please refer to:
+# https://developers.google.com/storage/docs/authentication#service_accounts
+#
+# Recommendation: experiment with the settings depending on how much log
+# data you generate, so the uploader can keep up with the generated logs.
+# Using gzip output can be a good option to reduce network traffic when
+# uploading the log files and in terms of storage costs as well.
+#
+# USAGE:
+# This is an example of logstash config:
+#
+# [source,json]
+# --------------------------
+# output {
+#    google_cloud_storage {
+#      bucket => "my_bucket"                                     (required)
+#      json_key_file => "/path/to/privatekey.json"               (optional)
+#      temp_directory => "/tmp/logstash-gcs"                     (optional)
+#      log_file_prefix => "logstash_gcs"                         (optional)
+#      max_file_size_kbytes => 1024                              (optional)
+#      output_format => "plain"                                  (optional)
+#      date_pattern => "%Y-%m-%dT%H:00"                          (optional)
+#      flush_interval_secs => 2                                  (optional)
+#      gzip => false                                             (optional)
+#      gzip_content_encoding => false                            (optional)
+#      uploader_interval_secs => 60                              (optional)
+#      upload_synchronous => false                               (optional)
+#    }
+# }
+# --------------------------
+#
+# Improvements TODO list:
+# * Support logstash event variables to determine filename.
+# * Turn Google API code into a Plugin Mixin (like AwsConfig).
+# * There's no recover method, so if logstash/plugin crashes, files may not
+# be uploaded to GCS.
+# * Allow user to configure file name.
+class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
+  config_name "google_cloud_storage"
+
+  concurrency :single
+
+  # GCS bucket name, without "gs://" or any other prefix.
+  config :bucket, :validate => :string, :required => true
+
+  # GCS path to private key file.
+  config :key_path, :validate => :string, :obsolete => 'Use json_key_file or ADC instead.'
+
+  # GCS private key password.
+  config :key_password, :validate => :string, :deprecated => 'Use json_key_file or ADC instead.'
+
+  # GCS service account.
+  config :service_account, :validate => :string, :deprecated => 'Use json_key_file or ADC instead.'
+
+  # Directory where temporary files are stored.
+  # Defaults to /tmp/logstash-gcs-<random-suffix>
+  config :temp_directory, :validate => :string, :default => ""
+
+  # Log file prefix. Log file will follow the format:
+  # <prefix>_hostname_date<.part?>.log
+  config :log_file_prefix, :validate => :string, :default => "logstash_gcs"
+
+  # Sets max file size in kbytes. 0 disable max file check.
+  config :max_file_size_kbytes, :validate => :number, :default => 10000
+
+  # The event format you want to store in files. Defaults to plain text.
+  config :output_format, :validate => [ "json", "plain" ], :default => "plain"
+
+  # Time pattern for log file, defaults to hourly files.
+  # Must Time.strftime patterns: www.ruby-doc.org/core-2.0/Time.html#method-i-strftime
+  config :date_pattern, :validate => :string, :default => "%Y-%m-%dT%H:00"
+
+  # Flush interval in seconds for flushing writes to log files. 0 will flush
+  # on every message.
+  config :flush_interval_secs, :validate => :number, :default => 2
+
+  # Gzip output stream when writing events to log files, set
+  # `Content-Type` to `application/gzip` instead of `text/plain`, and
+  # use file suffix `.log.gz` instead of `.log`.
+  config :gzip, :validate => :boolean, :default => false
+
+  # Gzip output stream when writing events to log files and set
+  # `Content-Encoding` to `gzip`.
+  config :gzip_content_encoding, :validate => :boolean, :default => false
+
+  # Uploader interval when uploading new files to GCS. Adjust time based
+  # on your time pattern (for example, for hourly files, this interval can be
+  # around one hour).
+  config :uploader_interval_secs, :validate => :number, :default => 60
+
+  # Should the hostname be included in the file name?
+  config :include_hostname, :validate => :boolean, :default => true
+
+  # Should a UUID be included in the file name?
+  config :include_uuid, :validate => :boolean, :default => false
+
+  # The path to the service account's JSON credentials file.
+  # Application Default Credentials (ADC) are used if the path is blank.
+  # See: https://cloud.google.com/docs/authentication/production
+  #
+  # You must run on GCP for ADC to work.
+  config :json_key_file, :validate => :string, :default => ""
+
+  # When true, files are uploaded by the event processing thread as soon as a file is ready.
+  # When false, (the default behaviour), files will be uploaded in a dedicated thread.
+  #
+  # Enabling this option provides greater likelihood that all generated files will be
+  # to GCS, especially in the event of a graceful shutdown of logstash, such as when an
+  # input plugin reaches the end of events. This comes at the price of introducing delays
+  # in the event processing pipeline as files are uploaded.
+  #
+  # When this feature is enabled, the uploader_interval_secs option has no effect.
+  config :upload_synchronous, :validate => :boolean, :default => false
+
+  config :max_concurrent_uploads, :validate  => :number, :default => 5
+
+  public
+  def register
+    @logger.debug('Registering Google Cloud Storage plugin')
+
+    @workers = LogStash::Outputs::Gcs::WorkerPool.new(@max_concurrent_uploads, @upload_synchronous)
+    initialize_temp_directory
+    initialize_path_factory
+    initialize_log_rotater
+
+    initialize_google_client
+
+    start_uploader
+
+    @content_type = @gzip ? 'application/gzip' : 'text/plain'
+    @content_encoding = @gzip_content_encoding ? 'gzip' : 'identity'
+  end
+
+  # Method called for each log event. It writes the event to the current output
+  # file, flushing depending on flush interval configuration.
+  public
+  def receive(event)
+    @logger.debug('Received event', :event => event)
+
+    if @output_format == 'json'
+      message = LogStash::Json.dump(event.to_hash)
+    else
+      message = event.to_s
+    end
+
+    @log_rotater.writeln(message)
+  end
+
+  public
+  def close
+    @logger.debug('Stopping the plugin, uploading the remaining files.')
+    Stud.stop!(@registration_thread) unless @registration_thread.nil?
+
+    # Force rotate the log. If it contains data it will be submitted
+    # to the work pool and will be uploaded before the plugin stops.
+    @log_rotater.rotate_log!
+    @workers.stop!
+  end
+
+  private
+
+  ##
+  # Creates temporary directory, if it does not exist.
+  #
+  # A random suffix is appended to the temporary directory
+  def initialize_temp_directory
+    require "stud/temporary"
+
+    if @temp_directory.empty?
+      @temp_directory = Stud::Temporary.directory('logstash-gcs')
+    end
+
+    FileUtils.mkdir_p(@temp_directory) unless File.directory?(@temp_directory)
+
+    @logger.info("Using temporary directory: #{@temp_directory}")
+  end
+
+  def initialize_path_factory
+    @path_factory = LogStash::Outputs::Gcs::PathFactoryBuilder.build do |builder|
+      builder.set_directory(@temp_directory)
+      builder.set_prefix(@log_file_prefix)
+      builder.set_include_host(@include_hostname)
+      builder.set_date_pattern(@date_pattern)
+      builder.set_include_part(@max_file_size_kbytes > 0)
+      builder.set_include_uuid(@include_uuid)
+      builder.set_is_gzipped(@gzip)
+    end
+  end
+
+  # start_uploader periodically sends flush events through the log rotater
+  def start_uploader
+    @registration_thread = Thread.new do
+      Stud.interval(@uploader_interval_secs) do
+        @log_rotater.writeln(nil)
+      end
+    end
+  end
+
+  ##
+  # Initializes Google Client instantiating client and authorizing access.
+  def initialize_google_client
+    @client = LogStash::Outputs::Gcs::Client.new(@bucket, @json_key_file, @logger)
+  end
+
+  ##
+  # Uploads a local file to the configured bucket.
+  def upload_object(filename)
+    @client.upload_object(filename, @content_encoding, @content_type)
+  end
+
+  def upload_and_delete(filename)
+    file_size = File.stat(filename).size
+
+    if file_size > 0
+      upload_object(filename)
+    else
+      @logger.debug('File size is zero, skip upload.', :filename => filename)
+    end
+
+    @logger.debug('Delete local temporary file', :filename => filename)
+    File.delete(filename)
+  end
+
+  def initialize_log_rotater
+    max_file_size = @max_file_size_kbytes * 1024
+    @log_rotater = LogStash::Outputs::Gcs::LogRotate.new(@path_factory, max_file_size, @gzip, @flush_interval_secs, @gzip_content_encoding)
+
+    @log_rotater.on_rotate do |filename|
+      @logger.info("Rotated out file: #{filename}")
+      @workers.post do
+        upload_and_delete(filename)
+      end
+    end
+  end
+  attr_accessor :active
+end

data/logstash-output-google_cloud_storage.gemspec

@@ -0,0 +1,33 @@
+Gem::Specification.new do |s|
+  s.name            = 'logstash-output-google_cloud_storage'
+  s.version         = '4.0.0'
+  s.licenses        = ['Apache-2.0']
+  s.summary         = "plugin to upload log events to Google Cloud Storage (GCS)"
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ["Elastic"]
+  s.email           = 'info@elastic.co'
+  s.homepage        = "http://www.elastic.co/guide/en/logstash/current/index.html"
+  s.require_paths = ["lib", "vendor/jar-dependencies"]
+
+  # Files
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+
+  s.add_runtime_dependency 'stud'
+  s.add_runtime_dependency 'logstash-codec-plain'
+  s.add_runtime_dependency 'mime-types', '~> 2' # last version compatible with ruby 2.x
+  s.add_runtime_dependency 'concurrent-ruby' # use version bundled with Logstash to avoid platform mismatch on plugin install
+  s.add_development_dependency 'logstash-devutils'
+
+  # JARs
+  s.platform = 'java'
+end
+

data/spec/fixtures/credentials.json

@@ -0,0 +1,8 @@
+{
+  "//": "Dummy account from https://github.com/GoogleCloudPlatform/google-cloud-java/google-cloud-clients/google-cloud-core/src/test/java/com/google/cloud/ServiceOptionsTest.java",
+  "private_key_id": "somekeyid",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC+K2hSuFpAdrJI\nnCgcDz2M7t7bjdlsadsasad+fvRSW6TjNQZ3p5LLQY1kSZRqBqylRkzteMOyHgaR\n0Pmxh3ILCND5men43j3h4eDbrhQBuxfEMalkG92sL+PNQSETY2tnvXryOvmBRwa/\nQP/9dJfIkIDJ9Fw9N4Bhhhp6mCcRpdQjV38H7JsyJ7lih/oNjECgYAt\nknddadwkwewcVxHFhcZJO+XWf6ofLUXpRwiTZakGMn8EE1uVa2LgczOjwWHGi99MFjxSer5m9\n1tCa3/KEGKiS/YL71JvjwX3mb+cewlkcmweBKZHM2JPTk0ZednFSpVZMtycjkbLa\ndYOS8V85AgMBewECggEBAKksaldajfDZDV6nGqbFjMiizAKJolr/M3OQw16K6o3/\n0S31xIe3sSlgW0+UbYlF4U8KifhManD1apVSC3csafaspP4RZUHFhtBywLO9pR5c\nr6S5aLp+gPWFyIp1pfXbWGvc5VY/v9x7ya1VEa6rXvLsKupSeWAW4tMj3eo/64ge\nsdaceaLYw52KeBYiT6+vpsnYrEkAHO1fF/LavbLLOFJmFTMxmsNaG0tuiJHgjshB\n82DpMCbXG9YcCgI/DbzuIjsdj2JC1cascSP//3PmefWysucBQe7Jryb6NQtASmnv\nCdDw/0jmZTEjpe4S1lxfHplAhHFtdgYTvyYtaLZiVVkCgYEA8eVpof2rceecw/I6\n5ng1q3Hl2usdWV/4mZMvR0fOemacLLfocX6IYxT1zA1FFJlbXSRsJMf/Qq39mOR2\nSpW+hr4jCoHeRVYLgsbggtrevGmILAlNoqCMpGZ6vDmJpq6ECV9olliDvpPgWOP+\nmYPDreFBGxWvQrADNbRt2dmGsrsCgYEAyUHqB2wvJHFqdmeBsaacewzV8x9WgmeX\ngUIi9REwXlGDW0Mz50dxpxcKCAYn65+7TCnY5O/jmL0VRxU1J2mSWyWTo1C+17L0\n3fUqjxL1pkefwecxwecvC+gFFYdJ4CQ/MHHXU81Lwl1iWdFCd2UoGddYaOF+KNeM\nHC7cmqra+JsCgYEAlUNywzq8nUg7282E+uICfCB0LfwejuymR93CtsFgb7cRd6ak\nECR8FGfCpH8ruWJINllbQfcHVCX47ndLZwqv3oVFKh6pAS/vVI4dpOepP8++7y1u\ncoOvtreXCX6XqfrWDtKIvv0vjlHBhhhp6mCcRpdQjV38H7JsyJ7lih/oNjECgYAt\nkndj5uNl5SiuVxHFhcZJO+XWf6ofLUregtevZakGMn8EE1uVa2AY7eafmoU/nZPT\n00YB0TBATdCbn/nBSuKDESkhSg9s2GEKQZG5hBmL5uCMfo09z3SfxZIhJdlerreP\nJ7gSidI12N+EZxYd4xIJh/HFDgp7RRO87f+WJkofMQKBgGTnClK1VMaCRbJZPriw\nEfeFCoOX75MxKwXs6xgrw4W//AYGGUjDt83lD6AZP6tws7gJ2IwY/qP7+lyhjEqN\nHtfPZRGFkGZsdaksdlaksd323423d+15/UvrlRSFPNj1tWQmNKkXyRDW4IG1Oa2p\nrALStNBx5Y9t0/LQnFI4w3aG\n-----END PRIVATE KEY-----\n",
+  "client_email": "someclientid@developer.gserviceaccount.com",
+  "client_id": "someclientid.apps.googleusercontent.com",
+  "type": "service_account"
+}

data/spec/outputs/gcs/client_spec.rb

@@ -0,0 +1,18 @@
+# encoding: utf-8
+
+require 'logstash/outputs/gcs/client'
+
+describe LogStash::Outputs::Gcs::Client do
+
+  # This test is mostly to make sure the Java types, signatures and classes
+  # haven't changed being that JRuby is very relaxed.
+  describe '#initialize' do
+    let(:logger) { spy('logger') }
+
+    it 'does not throw an error when initializing' do
+      key_file = ::File.join('spec', 'fixtures', 'credentials.json')
+      key_file = ::File.absolute_path(key_file)
+      LogStash::Outputs::Gcs::Client.new('my-bucket', key_file, logger)
+    end
+  end
+end

data/spec/outputs/gcs/log_rotate_spec.rb

@@ -0,0 +1,129 @@
+# encoding: utf-8
+require 'logstash/outputs/gcs/log_rotate'
+require 'logstash/outputs/gcs/path_factory'
+require 'logstash/outputs/gcs/temp_log_file'
+
+describe LogStash::Outputs::Gcs::LogRotate do
+  let(:tempdir){ Stud::Temporary.directory }
+  let(:path_factory) do
+    LogStash::Outputs::Gcs::PathFactoryBuilder.build do |builder|
+      builder.set_directory tempdir
+      builder.set_prefix 'prefix'
+      builder.set_include_host true
+      builder.set_date_pattern ''
+      builder.set_include_part true
+      builder.set_include_uuid true
+      builder.set_is_gzipped true
+    end
+  end
+  let(:open_file_1) { double('open-temp-1', :size => 5, :path => 'one', :close! => true, :time_since_sync => 10, :fsync => true)}
+  let(:open_file_2) { double('open-temp-2', :size => 5, :path => 'two', :close! => true, :time_since_sync => 60, :fsync => true)}
+
+  describe '#initialize' do
+    it 'opens the first file' do
+      expect(LogStash::Outputs::Gcs::LogFileFactory).to receive(:create).and_return(open_file_1)
+
+      LogStash::Outputs::Gcs::LogRotate.new(path_factory, 10, false, 30)
+    end
+  end
+
+  describe '#writeln' do
+    subject { LogStash::Outputs::Gcs::LogRotate.new(path_factory, 10, false, 30) }
+
+    it 'does not rotate if size is small and path is the same' do
+      expect(path_factory).to receive(:should_rotate?).and_return(false)
+      # once for init
+      expect(path_factory).to receive(:rotate_path!).once
+
+      subject.writeln('foo')
+    end
+
+    it 'rotates the file if the size is too big' do
+      # once for init, once for writeln
+      expect(path_factory).to receive(:rotate_path!).twice
+
+      subject.writeln('this line is longer than ten characters' * 1000)
+      subject.writeln('flush')
+    end
+
+    it 'rotates the file if the path changed' do
+      expect(path_factory).to receive(:should_rotate?).and_return(true)
+      # once for init, once for writeln
+      expect(path_factory).to receive(:rotate_path!).twice
+
+      subject.writeln('foo')
+    end
+
+    it 'writes the message' do
+      expect(LogStash::Outputs::Gcs::LogFileFactory).to receive(:create).and_return(open_file_1)
+      expect(open_file_1).to receive(:write).with('foo', "\n")
+
+      subject.writeln('foo')
+    end
+
+    it 'does not write nil messages' do
+      expect(LogStash::Outputs::Gcs::LogFileFactory).to receive(:create).and_return(open_file_1)
+      expect(open_file_1).not_to receive(:write)
+
+      subject.writeln(nil)
+    end
+
+    it 'does not fsync if delta less than limit' do
+      expect(LogStash::Outputs::Gcs::LogFileFactory).to receive(:create).and_return(open_file_1)
+      expect(open_file_1).not_to receive(:fsync)
+
+      subject.writeln(nil)
+    end
+
+    it 'fsyncs if delta greater than limit' do
+      expect(LogStash::Outputs::Gcs::LogFileFactory).to receive(:create).and_return(open_file_2)
+      expect(open_file_2).to receive(:fsync)
+
+      subject.writeln(nil)
+    end
+  end
+
+  describe '#rotate_log!' do
+    subject { LogStash::Outputs::Gcs::LogRotate.new(path_factory, 10, false, 30) }
+
+    before :each do
+      allow(LogStash::Outputs::Gcs::LogFileFactory).to receive(:create).and_return(open_file_1, open_file_2)
+    end
+
+    it 'closes the old file' do
+      expect(open_file_1).to receive(:close!)
+
+      subject.rotate_log!
+    end
+
+    it 'calls the callback with the old file name' do
+      value = nil
+      subject.on_rotate { |old_path| value = old_path }
+
+      subject.rotate_log!
+      expect(value).to eq(open_file_1.path)
+    end
+
+    it 'opens a new file based on the new path' do
+      expect(LogStash::Outputs::Gcs::LogFileFactory).to receive(:create).and_return(open_file_1, open_file_2)
+      expect(open_file_2).to receive(:write).with('foo', "\n")
+
+      subject.rotate_log!
+      subject.writeln('foo')
+    end
+  end
+
+  describe '#on_rotate' do
+    subject { LogStash::Outputs::Gcs::LogRotate.new(path_factory, 10, false, 30) }
+
+    it 'replaces an existing callback' do
+      value = :none
+
+      subject.on_rotate { value = :first }
+      subject.on_rotate { value = :second }
+
+      subject.rotate_log!
+      expect(value).to eq(:second)
+    end
+  end
+end