logstash-output-stackdriver_logging 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 7eeee7c50713fc116155c686aec51663bb78b62b7ec3015ab65abb2847a86d8a
+  data.tar.gz: e8ae42e6139cf1c71a70ecc6aa3efc47888e29c6614548823ff55c4dd259513c
+SHA512:
+  metadata.gz: f1f503d74a900679df982128b5b013355e4f7b9315001edc1a0d0fae808e9086fc7812010037e2eea468c089af704fd96beda645e25641486ccaa6319f8f8eae
+  data.tar.gz: 2f2df41576e922909ccef24c32b0c7d1d02a2282cb5325454c09d18e402af9d6598b953c5d8475a98859a17bd3b36d04e421566e230203c4ec3afdb97c98fd57

data/CHANGELOG.md

@@ -0,0 +1,12 @@
+## 2.0.3
+  - Docs: Set the default_codec doc attribute.
+
+## 2.0.2
+ - Docs: Add documentation template
+## 2.0.1
+ - Add encoding: utf-8 to spec files. This can help prevent issues during testing.
+## 2.0.0
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
+   instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
+ - Dependency on logstash-core update to 2.0
+

data/CONTRIBUTORS

@@ -0,0 +1,11 @@
+The following is a list of people who have contributed ideas, code, bug
+reports, or in general have helped logstash along its way.
+
+Contributors:
+* Aaron Mildenstein (untergeek)
+* Pier-Hugues Pellerin (ph)
+
+Note: If you've sent us patches, bug reports, or otherwise contributed to
+Logstash, and you aren't on the list above and want to be, please let us know
+and we'll make sure you're here. Contributions from folks like you are what make
+open source awesome.

data/DEVELOPER.md

@@ -0,0 +1,2 @@
+# logstash-output-example
+Example output plugin. This should help bootstrap your effort to write your own output plugin!

data/Gemfile

@@ -0,0 +1,8 @@
+source 'https://rubygems.org'
+
+gemspec
+
+gem 'logstash-core'
+gem 'logstash-core-plugin-api'
+gem 'google-api-client'
+gem 'googleauth'

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 Geoff Garbers
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/NOTICE.TXT

@@ -0,0 +1,5 @@
+Elasticsearch
+Copyright 2012-2015 Elasticsearch
+
+This product includes software developed by The Apache Software
+Foundation (http://www.apache.org/).

data/README.md

@@ -0,0 +1,46 @@
+# Stackdriver Logging Logstash Plugin
+
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
+
+It is fully free and fully open source. The license is MIT, meaning you are pretty much free to use it however you want in whatever way.
+
+
+## Summary
+
+This plugin sends events to Stackdriver Logging. Events are sent in batches as they are received.
+
+
+## Usage
+
+This is an example of Logstash config for this plugin:
+
+```
+output {
+    stackdriver_logging {
+        project_id => "folkloric-guru-278"    (optional) <1>
+        key_file => "/path/to/key_file.json"  (optional) <2>
+        log_name => ""                        (required) <3>
+        severity_field => "severity"          (optional)
+        timestamp_field => "@timestamp"       (optional)
+        default_severity => "default"         (optional)
+        strip_at_fields => false              (optional)
+        strip_fields => []                    (optional)
+    }
+}
+```
+
+There are some points to keep in mind when providing configuration:
+
+1. When running on Google Compute Engine and no `project_id` is provided, the `project_id` paramater will be
+   automatically retrieved from the metadata server.
+
+2. If no key is provided, defaults to using [Application Default Credentials](https://cloud.google.com/docs/authentication/production).
+
+3. The log name can be interpolated with event fields, to extract the log name from a message (ie: `log_name => "%{[log_name]}"`).
+
+## Considerations
+
+There is a cost to storing log data in Stackdriver Logging. See [the pricing guide](https://cloud.google.com/stackdriver/pricing)
+for more information on costs involved.
+
+All logs are stored under the `global` resource.

data/lib/logstash/outputs/stackdriver_logging.rb

@@ -0,0 +1,177 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+require "logstash/logging"
+require 'google/apis/logging_v2'
+require 'googleauth'
+require 'json'
+require 'faraday'
+
+# === Summary
+#
+# This plugin sends events to Stackdriver Logging. Events are sent in batches as they are received.
+#
+# === Environment configuration
+#
+# Logging should already be configured on your Google Cloud Platform project. However, you will need
+# to create a service account with permissions to write logs.
+#
+# === Usage
+#
+# This is an example of Logstash config for this plugin:
+#
+# [source,ruby]
+# --------------------------
+# output {
+#   stackdriver_logging {
+#     project_id => "folkloric-guru-278"    (optional) <1>
+#     key_file => "/path/to/key_file.json"  (optional) <2>
+#     log_name => ""                        (required) <3>
+#     severity_field => "severity"          (optional)
+#     timestamp_field => "@timestamp"       (optional)
+#     default_severity => "default"         (optional)
+#     strip_at_fields => false              (optional)
+#     strip_fields => []                    (optional)
+#   }
+# }
+# --------------------------
+#
+# <1> If running on Google Compute Engine, the project ID will be automatically retrieved from the metadata server. Required
+#     if not running within GCE.
+#
+# <2> If no key is provided, defaults to using https://cloud.google.com/docs/authentication/production[Application Default Credentials].
+#
+# <3> The log name can be interpolated with log data, to extract the log name from a message.
+#
+# === Considerations
+#
+# There is a cost to storing log data in Stackdriver Logging. See https://cloud.google.com/stackdriver/pricing[the pricing guide]
+# for more information on costs involved.
+#
+# All logs are stored under the `global` resource.
+#
+#
+class LogStash::Outputs::StackdriverLogging < LogStash::Outputs::Base
+  config_name "stackdriver_logging"
+
+  # The Google Cloud project to write the logs to. This is optional if running on GCE,
+  # and will default to the instance's project.
+  config :project_id, :validate => :string, :required => false, :default => nil
+
+  # The path to the service account JSON file that contains the credentials
+  # of the service account to write logs as.
+  config :key_file, :validate => :path, :required => false
+
+  # The name of the log to write logs to. Can be interpolated with event fields.
+  config :log_name, :validate => :string, :required => true
+
+  # The name of the field that contains the logging severity level. If no field with this name is specified, defaults
+  # to @default_severity.
+  config :severity_field, :validate => :string, :required => false, :default => "severity"
+
+  # The field name in the event that contains the timestamp of the log message.
+  config :timestamp_field, :validate => :string, :required => false, :default => "@timestamp"
+
+  # If no severity is found, the default severity level to assume.
+  config :default_severity, :validate => :string, :required => false, :default => "default"
+
+  # Boolean flag indicating whether or not to remove the fields in the event that are prefixed with "@", immediately
+  # prior to sending.
+  config :strip_at_fields, :validate => :boolean, :required => false, :default => false
+
+  # Array of field names to remove from the event immediately prior to sending.
+  config :strip_fields, :validate => :array, :required => false, :default => []
+
+  concurrency :single
+
+  public
+  def register
+    @service = Google::Apis::LoggingV2::LoggingService.new
+    scope = %w(https://www.googleapis.com/auth/logging.write)
+
+    # Always load key file if provided.
+    if @key_file
+      @service.authorization = Google::Auth::ServiceAccountCredentials.make_creds(json_key_io: File.open(@key_file),
+                                                                                  scope: scope)
+    # Fall back to getting the application default credentials.
+    else
+      @service.authorization = Google::Auth.get_application_default(scope)
+    end
+
+    # project_id is not defined. Try to extract it from teh metadata server.
+    unless @project_id
+      if Google::Auth::GCECredentials.on_gce?
+        connection = Faraday::Connection.new("http://169.254.169.254/computeMetadata/v1/", { :headers => headers })
+        connection.headers = { "Metadata-Flavor": "Google" }
+        response = connection.get "project/project-id"
+
+        if response.status
+          @project_id = response.body.to_s.strip
+        end
+
+
+      else
+        @logger.error "Unable to detect the Google Cloud project ID to which logs should be written." \
+                      "Please ensure that you specify the `project_id` config parameter if not running on the Google " \
+                      "Cloud Platform."
+        @logger.error "You will not be able to be able to write logs to Google Cloud until this is resolved."
+      end
+    end
+  end
+
+  public
+  # @param [Array] events
+  def multi_receive(events)
+    # Do nothing if no events are received.
+    if events.length < 1
+      return
+    end
+
+    entries = []
+
+    events.each do |event|
+      entry = Google::Apis::LoggingV2::LogEntry.new
+      entry.severity = event.include?(@severity_field) ? event.get(@severity_field) : @default_severity
+      entry.log_name = "projects/%{project}/logs/%{log_name}" % { :project => @project_id, :log_name => event.sprintf(@log_name) }
+      entry.timestamp = event.get(@timestamp_field)
+      entry.json_payload = event.to_hash
+
+      # Strip the "@" fields.
+      if @strip_at_fields or @strip_fields.length > 0
+        filtered = {}
+
+        entry.json_payload.each do |key, value|
+          if @strip_at_fields && key[0] == "@"
+            next
+          end
+
+          if @strip_fields.length > 0 && @strip_fields.include?(key)
+            next
+          end
+
+          filtered[key] = value
+        end
+
+        entry.json_payload = filtered
+      end
+
+      entries.push entry
+    end
+
+    resource = Google::Apis::LoggingV2::MonitoredResource.new
+    resource.type = "global"
+    resource.labels = { :project_id => @project_id }
+
+    request = Google::Apis::LoggingV2::WriteLogEntriesRequest.new(entries: entries)
+    request.resource = resource
+
+    @service.write_entry_log_entries(request) do |result, error|
+      if error
+        @logger.error "Unable to write log entries to Stackdriver Logging."
+        @logger.error "Received this error: " + error.to_s
+      else
+        @logger.debug("Wrote %{length} entries successfully." % { :length => entries.length })
+      end
+    end
+  end
+end

data/logstash-output-stackdriver_logging.gemspec

@@ -0,0 +1,27 @@
+Gem::Specification.new do |s|
+  s.name = 'logstash-output-stackdriver_logging'
+  s.version = '0.1.0'
+  s.licenses = ['MIT']
+  s.summary = "Writes payloads to Stackdriver Logging."
+  s.description  = ""
+  s.authors = ["Geoff Garbers"]
+  s.email = "geoff@garbers.co.za"
+  s.homepage = "https://github.com/garbetjie/logstash-output-stackdriver"
+  s.require_paths = %w(lib generated)
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+
+  # Gem dependencies
+  #
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency "logstash-codec-plain"
+  s.add_runtime_dependency "google-api-client"
+  s.add_runtime_dependency "googleauth"
+  s.add_development_dependency 'logstash-devutils'
+end

data/spec/outputs/stackdriver_logging_spec.rb

@@ -0,0 +1,22 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/stackdriver_logging"
+require "logstash/codecs/plain"
+require "logstash/event"
+
+describe LogStash::Outputs::StackdriverLogging do
+  let(:sample_event) { LogStash::Event.new }
+  let(:output) { LogStash::Outputs::StackdriverLogging.new }
+
+  before do
+    output.register
+  end
+
+  describe "receive message" do
+    subject { output.receive(sample_event) }
+
+    it "returns a string" do
+      expect(subject).to eq("Event received")
+    end
+  end
+end

metadata

@@ -0,0 +1,133 @@
+--- !ruby/object:Gem::Specification
+name: logstash-output-stackdriver_logging
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Geoff Garbers
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2018-11-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-plain
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: google-api-client
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: googleauth
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ''
+email: geoff@garbers.co.za
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTORS
+- DEVELOPER.md
+- Gemfile
+- LICENSE
+- NOTICE.TXT
+- README.md
+- lib/logstash/outputs/stackdriver_logging.rb
+- logstash-output-stackdriver_logging.gemspec
+- spec/outputs/stackdriver_logging_spec.rb
+homepage: https://github.com/garbetjie/logstash-output-stackdriver
+licenses:
+- MIT
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+- generated
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.6.13
+signing_key:
+specification_version: 4
+summary: Writes payloads to Stackdriver Logging.
+test_files:
+- spec/outputs/stackdriver_logging_spec.rb