logstash-input-google_cloud_storage 0.9.0-java

data/lib/logstash/inputs/cloud_storage/blob_adapter.rb

@@ -0,0 +1,92 @@
+# encoding: utf-8
+
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+require 'java'
+require 'logstash-input-google_cloud_storage_jars.rb'
+
+module LogStash
+  module Inputs
+    module CloudStorage
+      # BlobAdapter exposes parts of `com.google.cloud.storage.Blob` for use in
+      # the plugin for easier mocking and future-proofing.
+      class BlobAdapter
+        def initialize(blob)
+          @blob = blob
+        end
+
+        def name
+          @blob.getName()
+        end
+
+        def attributes
+          {
+            'bucket' => @blob.getBucket(),
+            'metadata' => @blob.getMetadata(),
+            'name' => @blob.getName(),
+            'md5' => @blob.getMd5(),
+            'crc32c' => @blob.getCrc32c(),
+            'generation' => @blob.getGeneration()
+          }
+        end
+
+        java_import 'com.google.cloud.storage.Blob$BlobSourceOption'
+        def delete!
+          @blob.delete(BlobSourceOption.generationMatch())
+        end
+
+        def set_metadata!(key, value)
+          new_metadata = { key => value }
+          @blob.toBuilder().setMetadata(new_metadata).build().update()
+        end
+
+        def metadata
+          @blob.getMetadata()
+        end
+
+        def generation
+          @blob.getGeneration()
+        end
+
+        def line_attributes(line_number)
+          attrs = attributes
+
+          attrs['line'] = line_number
+          attrs['line_id'] = "#{uri}:#{line_number}@#{generation}"
+
+          attrs
+        end
+
+        def uri
+          "gs://#{@blob.getBucket()}/#{name}"
+        end
+
+        java_import 'java.nio.file.Paths'
+        def download_to(path)
+          temp_path = Paths.get(path)
+          @blob.downloadTo(temp_path)
+        end
+
+        def with_downloaded(temp_directory)
+          temp_file = ::File.join(temp_directory, SecureRandom.uuid)
+          download_to(temp_file)
+
+          yield temp_file
+
+          FileUtils.remove_entry_secure(temp_file, true)
+        end
+      end
+    end
+  end
+end

data/lib/logstash/inputs/cloud_storage/blob_filter.rb

@@ -0,0 +1,99 @@
+# encoding: utf-8
+
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module LogStash
+  module Inputs
+    module CloudStorage
+      # BlobFilter acts as a .filter for BlobAdapters.
+      class BlobFilter
+        # Initialize the filter.
+        # Throws an exception if the regular expressions could not compile.
+        def initialize(logger, include_regex, exclude_regex, metadata_key, processed_db)
+          @logger = logger
+          @include_regex = compile_regex(include_regex)
+          @exclude_regex = compile_regex(exclude_regex)
+          @metadata_key = metadata_key
+          @processed_db = processed_db
+
+          @logger.info('Turn on debugging to explain why blobs are filtered.')
+        end
+
+        # should_process? returns true if the blob matches all the
+        # user-provided requirements to download and extract events.
+        def should_process?(blob)
+          @logger.debug("Found blob: #{blob.name}")
+
+          # Evaluate all conditions because the operations are cheap and give the
+          # user a complete idea of why a blob was included/excluded.
+          conditions = [
+            not_already_run?(blob),
+            included?(blob),
+            not_excluded?(blob),
+            metadata_does_not_exist?(blob)
+          ]
+
+          conditions.all?
+        end
+
+        private
+
+        def compile_regex(regex)
+          Regexp.new(regex)
+        rescue StandardError => e
+          raise "Could not compile regex '#{regex}': #{e}"
+        end
+
+        def not_already_run?(blob)
+          result = @processed_db.nil? || !@processed_db.already_processed?(blob)
+
+          explain('Not included in ProcessedDB', result)
+        end
+
+        def included?(blob)
+          explain('Matches include regex', name_matches(blob.name, @include_regex))
+        end
+
+        def not_excluded?(blob)
+          explain('Does not match exclude regex', !name_matches(blob.name, @exclude_regex))
+        end
+
+        def metadata_does_not_exist?(blob)
+          # an empty key means the user doesn't want to store/check the metadata
+          return true if @metadata_key.empty?
+
+          has_key = !blob.metadata.nil? && blob.metadata.key?(@metadata_key)
+
+          explain('Does not have metadata key', !has_key)
+        end
+
+        def explain(message, result)
+          pf = result ? 'pass' : 'fail'
+
+          @logger.debug(" - #{message}? (#{pf})")
+
+          result
+        end
+
+        def name_matches(name, regex)
+          match = regex.match(name)
+          return false if match.nil?
+
+          match.pre_match.empty? && match.post_match.empty?
+        end
+      end
+    end
+  end
+end

data/lib/logstash/inputs/cloud_storage/client.rb

@@ -0,0 +1,87 @@
+# encoding: utf-8
+
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+require 'thread'
+require 'java'
+require 'logstash-input-google_cloud_storage_jars.rb'
+require 'logstash/inputs/cloud_storage/blob_adapter'
+
+module LogStash
+  module Inputs
+    module CloudStorage
+      # Client provides all the required transport and authentication setup for the plugin.
+      class Client
+        def initialize(bucket, json_key_path, logger)
+          @logger = logger
+          @bucket = bucket
+
+          # create client
+          @storage = initialize_storage json_key_path
+        end
+
+        def list_blobs
+          @storage.list(@bucket).iterateAll().each do |blobname|
+            yield LogStash::Inputs::CloudStorage::BlobAdapter.new(blobname)
+          end
+        rescue Java::ComGoogleCloudStorage::StorageException => e
+          raise "Error listing bucket contents: #{e}"
+        end
+
+        private
+
+        def initialize_storage(json_key_path)
+          com.google.cloud.storage.StorageOptions.newBuilder()
+             .setCredentials(credentials(json_key_path))
+             .setHeaderProvider(http_headers)
+             .setRetrySettings(retry_settings)
+             .build()
+             .getService()
+        end
+
+        java_import 'com.google.auth.oauth2.GoogleCredentials'
+        def credentials(json_key_path)
+          return GoogleCredentials.getApplicationDefault() if json_key_path.empty?
+
+          key_file = java.io.FileInputStream.new(json_key_path)
+          GoogleCredentials.fromStream(key_file)
+        end
+
+        java_import 'com.google.api.gax.rpc.FixedHeaderProvider'
+        def http_headers
+          gem_name = 'logstash-input-google_cloud_storage'
+          gem_version = '1.0.0'
+          user_agent = "Elastic/#{gem_name} version/#{gem_version}"
+
+          FixedHeaderProvider.create({ 'User-Agent' => user_agent })
+        end
+
+        java_import 'com.google.api.gax.retrying.RetrySettings'
+        java_import 'org.threeten.bp.Duration'
+        def retry_settings
+          # backoff values taken from com.google.api.client.util.ExponentialBackOff
+          RetrySettings.newBuilder()
+                       .setInitialRetryDelay(Duration.ofMillis(500))
+                       .setRetryDelayMultiplier(1.5)
+                       .setMaxRetryDelay(Duration.ofSeconds(60))
+                       .setInitialRpcTimeout(Duration.ofSeconds(20))
+                       .setRpcTimeoutMultiplier(1.5)
+                       .setMaxRpcTimeout(Duration.ofSeconds(20))
+                       .setTotalTimeout(Duration.ofMinutes(15))
+                       .build()
+        end
+      end
+    end
+  end
+end

data/lib/logstash/inputs/cloud_storage/file_reader.rb

@@ -0,0 +1,59 @@
+# encoding: utf-8
+
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+require 'zlib'
+
+module LogStash
+  module Inputs
+    module CloudStorage
+      # FileReader provides a unified way to read different types of log files
+      # with predictable callbacks.
+      class FileReader
+        # read_lines reads lines from a file one at a time, optionally decoding
+        # the file as gzip if decode_gzip is true.
+        #
+        # Handles files with both UNIX and Windows line endings.
+        def self.read_lines(filename, decode_gzip, &block)
+          if decode_gzip && gzip?(filename)
+            read_gzip_lines(filename, &block)
+          else
+            read_plain_lines(filename, &block)
+          end
+        end
+
+        # gzip? returns true if the given filename has a gzip file extension.
+        def self.gzip?(filename)
+          filename.end_with? '.gz'
+        end
+
+        def self.read_plain_lines(filename, &block)
+          line_num = 1
+          ::File.open(filename).each do |line|
+            block.call(line.chomp, line_num)
+            line_num += 1
+          end
+        end
+
+        def self.read_gzip_lines(filename, &block)
+          line_num = 1
+          Zlib::GzipReader.open(filename).each_line do |line|
+            block.call(line.chomp, line_num)
+            line_num += 1
+          end
+        end
+      end
+    end
+  end
+end

data/lib/logstash/inputs/cloud_storage/processed_db.rb

@@ -0,0 +1,53 @@
+# encoding: utf-8
+
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+require 'fileutils'
+require 'digest'
+
+module LogStash
+  module Inputs
+    module CloudStorage
+      # ProcessedDb tracks files and generations that have already been processed.
+      # File names and generations are concatenated then SHA1 hashed.
+      # The directory structure is git-like the first 3 characters of the hash are used
+      # as a top level directory, and the rest is stored as a directory name within that.
+      # This keeps the directory count manageable.
+      class ProcessedDb
+        def initialize(db_directory)
+          @db_directory = db_directory
+        end
+
+        def already_processed?(blob)
+          path = encode_path(blob)
+          ::File.exist?(path)
+        end
+
+        def mark_processed(blob)
+          path = encode_path(blob)
+          FileUtils.mkdir_p(path)
+        end
+
+        def encode_path(blob)
+          key = "#{blob.generation}|#{blob.name}"
+          encoded = Digest::SHA1.hexdigest(key)
+          prefix = encoded.slice(0, 3)
+          suffix = encoded.slice(3..-1)
+
+          ::File.join(@db_directory, prefix, suffix)
+        end
+      end
+    end
+  end
+end

data/lib/logstash/inputs/google_cloud_storage.rb

@@ -0,0 +1,130 @@
+# encoding: utf-8
+
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+require 'logstash/inputs/base'
+require 'logstash/namespace'
+require 'stud/interval'
+require 'logstash/inputs/cloud_storage/client'
+require 'logstash/inputs/cloud_storage/processed_db'
+require 'logstash/inputs/cloud_storage/blob_filter'
+require 'logstash/inputs/cloud_storage/file_reader'
+
+# GoogleCloudStorage is an input plugin for LogStash that
+# reads blobs in Cloud Storage buckets.
+class LogStash::Inputs::GoogleCloudStorage < LogStash::Inputs::Base
+  config_name 'google_cloud_storage'
+
+  default :codec, 'plain'
+
+  # Connection Settings
+  config :bucket_id, :validate => :string, :required => true
+  config :json_key_file, :validate => :string, :default => nil
+  config :interval, :validate => :number, :default => 60
+
+  # Inclusion/Exclusion Criteria
+  config :file_matches, :validate => :string, :default => '.*\\.log(\\.gz)?'
+  config :file_exclude, :validate => :string, :default => '^$'
+  config :metadata_key, :validate => :string, :default => 'x-goog-meta-ls-gcs-input'
+  config :processed_db_path, :validate => :string, :default => nil
+
+  config :delete, :validate => :boolean, :default => false
+  config :unpack_gzip, :validate => :boolean, :default => true
+
+  # Other Criteria
+  config :temp_directory, :validate => :string, :default => File.join(Dir.tmpdir, 'ls-in-gcs')
+
+  # Accessors for testing
+  attr_accessor :event_output_queue
+  attr_reader :processed_db
+
+  def register
+    FileUtils.mkdir_p(@temp_directory) unless Dir.exist?(@temp_directory)
+
+    @client = LogStash::Inputs::CloudStorage::Client.new(@bucket_id, @json_key_file, @logger)
+
+    if @processed_db_path.nil?
+      ls_data = LogStash::SETTINGS.get_value('path.data')
+      @processed_db_path = File.join(ls_data, 'plugins', 'inputs', 'google_cloud_storage', 'db')
+    end
+
+    @logger.info("ProcessedDb created in: #{@processed_db_path}")
+
+    @processed_db = LogStash::Inputs::CloudStorage::ProcessedDb.new(@processed_db_path)
+
+    @blob_filter = LogStash::Inputs::CloudStorage::BlobFilter.new(@logger, @file_matches, @file_exclude, @metadata_key, @processed_db)
+  end
+
+  def run(queue)
+    @event_output_queue = queue
+
+    @main_plugin_thread = Thread.current
+    Stud.interval(@interval) do
+      list_download_process
+    end
+  end
+
+  # Fetches new files ready to be processed, downloads and processes them and finally
+  # runs post-processing steps.
+  def list_download_process
+    list_processable_blobs do |blob|
+      @logger.info("Found matching blob #{blob.uri}")
+      download_and_process(blob)
+      postprocess(blob)
+    end
+  end
+
+  def stop
+    # Stud events were started on the main plugin thread so stop all events relative to it.
+    Stud.stop!(@main_plugin_thread)
+  end
+
+  private
+
+  # list_processable_blobs will list blobs in the bucket and yield them if they are not filtered
+  def list_processable_blobs
+    @logger.info("Fetching blobs from #{@bucket_id}")
+    @client.list_blobs do |blob|
+      yield blob if @blob_filter.should_process?(blob)
+    end
+  end
+
+  def download_and_process(blob)
+    @logger.info("Downloading blob #{blob.uri}")
+
+    blob.with_downloaded(@temp_directory) do |path|
+      @logger.info("Reading events from #{blob.uri} (temp file: #{path})")
+
+      LogStash::Inputs::CloudStorage::FileReader.read_lines(path, @unpack_gzip) do |line, num|
+        extract_event(line, num, blob)
+      end
+    end
+  end
+
+  def postprocess(blob)
+    blob.set_metadata!(@metadata_key, 'processed') unless @metadata_key.empty?
+
+    blob.delete! if @delete
+
+    @processed_db.mark_processed(blob) unless @processed_db_path.empty?
+  end
+
+  def extract_event(line, line_num, blob)
+    @codec.decode(line) do |event|
+      decorate(event)
+      event.set('[@metadata][gcs]', blob.line_attributes(line_num))
+      @event_output_queue << event
+    end
+  end
+end