logstash-input-google_pubsub 0.9.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5833affff693fc86005d9fd0f29215e8eeb9f154
-  data.tar.gz: f65d91acd4a6c805a3587558a7ef8c252c1435ed
+  metadata.gz: 7b261bb9333fce81b2dc53f8474d332a16de4519
+  data.tar.gz: fee0a4acafa699ced1feb58cedfaeaea9bcd1fa8
 SHA512:
-  metadata.gz: 96b4e044b16625abc380a4372da3b3c63dd856b7070c7e11ff255a73acdc586054be168d3cd7f66e53a7ca352c2663bf29989c850e00a535ea4472cedceda87b
-  data.tar.gz: 2ee4c5fc0519b9689618b5dfab52232389e4e6c900d4ea632dff96577f948e84b411c7b9f04a68222a2d47aec8be22868bd196456c7c23c67bfdd07d95543d14
+  metadata.gz: d54ac43c05cc10acce0a696e067a681bb1155ed3388f5f2b47d99e936c337259086eb32944793405e84163e71246b25eaf70e63aa8dab8b1ccd7c6b647044ff5
+  data.tar.gz: e54671aae537ff2cd6176f4c38a61912ab0be6fdc2a3ad91d544eb375ecb3179b66a33344fe69744d29373b89e00a45a98f716645a913ac1e788126cc9d68b8e

data/lib/logstash/inputs/google_pubsub.rb

@@ -22,10 +22,106 @@ require "logstash/namespace"
 # Google deps
 require "google/api_client"
 
-# Generate a repeating message.
+# This is a https://github.com/elastic/logstash[Logstash] input plugin for 
+# https://cloud.google.com/pubsub/[Google Pub/Sub]. The plugin can subscribe 
+# to a topic and ingest messages.
+#
+# The main motivation behind the development of this plugin was to ingest 
+# https://cloud.google.com/logging/[Stackdriver Logging] messages via the 
+# https://cloud.google.com/logging/docs/export/using_exported_logs[Exported Logs] 
+# feature of Stackdriver Logging.
+#
+# ==== Prerequisites
+#
+# You must first create a Google Cloud Platform project and enable the the 
+# Google Pub/Sub API. If you intend to use the plugin ingest Stackdriver Logging 
+# messages, you must also enable the Stackdriver Logging API and configure log 
+# exporting to Pub/Sub. There is plentiful information on 
+# https://cloud.google.com/ to get started: 
+#
+# - Google Cloud Platform Projects and https://cloud.google.com/docs/overview/[Overview]
+# - Google Cloud Pub/Sub https://cloud.google.com/pubsub/[documentation]
+# - Stackdriver Logging https://cloud.google.com/logging/[documentation]
+#
+# ==== Cloud Pub/Sub
+#
+# Currently, this module requires you to create a `topic` manually and specify 
+# it in the logstash config file. You must also specify a `subscription`, but 
+# the plugin will attempt to create the pull-based `subscription` on its own. 
+#
+# All messages received from Pub/Sub will be converted to a logstash `event` 
+# and added to the processing pipeline queue. All Pub/Sub messages will be 
+# `acknowledged` and removed from the Pub/Sub `topic` (please see more about 
+# https://cloud.google.com/pubsub/overview#concepts)[Pub/Sub concepts]. 
+#
+# It is generally assumed that incoming messages will be in JSON and added to 
+# the logstash `event` as-is. However, if a plain text message is received, the 
+# plugin will return the raw text in as `raw_message` in the logstash `event`. 
+#
+# ==== Authentication
+#
+# You have two options for authentication depending on where you run Logstash. 
+#
+# 1. If you are running Logstash outside of Google Cloud Platform, then you will 
+# need to create a Google Cloud Platform Service Account and specify the full 
+# path to the JSON private key file in your config. You must assign sufficient 
+# roles to the Service Account to create a subscription and to pull messages 
+# from the subscription. Learn more about GCP Service Accounts and IAM roles 
+# here:
+#
+#   - Google Cloud Platform IAM https://cloud.google.com/iam/[overview]
+#   - Creating Service Accounts https://cloud.google.com/iam/docs/creating-managing-service-accounts[overview]
+#   - Granting Roles https://cloud.google.com/iam/docs/granting-roles-to-service-accounts[overview]
+#
+# 1. If you are running Logstash on a Google Compute Engine instance, you may opt 
+# to use Application Default Credentials. In this case, you will not need to 
+# specify a JSON private key file in your config.
+#
+# ==== Stackdriver Logging (optional)
+#
+# If you intend to use the logstash plugin for Stackdriver Logging message 
+# ingestion, you must first manually set up the Export option to Cloud Pub/Sub and 
+# the manually create the `topic`. Please see the more detailed instructions at, 
+# https://cloud.google.com/logging/docs/export/using_exported_logs [Exported Logs] 
+# and ensure that the https://cloud.google.com/logging/docs/export/configure_export#manual-access-pubsub[necessary permissions] 
+# have also been manually configured.
+#
+# Logging messages from Stackdriver Logging exported to Pub/Sub are received as 
+# JSON and converted to a logstash `event` as-is in 
+# https://cloud.google.com/logging/docs/export/using_exported_logs#log_entries_in_google_pubsub_topics[this format].
+#
+# ==== Sample Configuration
+#
+# Below is a copy of the included `example.conf-tmpl` file that shows a basic 
+# configuration for this plugin.
+#
+# [source,ruby]
+# ----------------------------------
+# input {
+#     google_pubsub {
+#         # Your GCP project id (name)
+#         project_id => "my-project-1234"
+#
+#         # The topic name below is currently hard-coded in the plugin. You
+#         # must first create this topic by hand and ensure you are exporting
+#         # logging to this pubsub topic.
+#         topic => "logstash-input-dev"
+#
+#         # The subscription name is customizeable. The plugin will attempt to
+#         # create the subscription (but use the hard-coded topic name above).
+#         subscription => "logstash-sub"
+#
+#         # If you are running logstash within GCE, it will use
+#         # Application Default Credentials and use GCE's metadata
+#         # service to fetch tokens.  However, if you are running logstash
+#         # outside of GCE, you will need to specify the service account's
+#         # JSON key file below.
+#         #json_key_file => "/home/erjohnso/pkey.json"
+#     }
+# }
+# output { stdout { codec => rubydebug } }
+# ----------------------------------
 #
-# This plugin is intented only as an example.
-
 class LogStash::Inputs::GooglePubSub < LogStash::Inputs::Base
   config_name "google_pubsub"
 
@@ -51,22 +147,22 @@ class LogStash::Inputs::GooglePubSub < LogStash::Inputs::Base
   private
   def request(options)
     begin
-      @logger.info("Sending an API request")
+      @logger.debug("Sending an API request")
       result = @client.execute(options)
     rescue ArgumentError => e
-      @logger.info("Authorizing...")
+      @logger.debug("Authorizing...")
       @client.authorization.fetch_access_token!
-      @logger.info("...authorized")
+      @logger.debug("...authorized")
       request(options)
     rescue Faraday::TimeoutError => e
-      @logger.info("Request timeout, re-trying request")
+      @logger.debug("Request timeout, re-trying request")
       request(options)
     end
   end # def request
 
   public
   def register
-    @logger.info("Registering Google PubSub Input: project_id=#{@project_id}, topic=#{@topic}, subscription=#{@subscription}")
+    @logger.debug("Registering Google PubSub Input: project_id=#{@project_id}, topic=#{@topic}, subscription=#{@subscription}")
     @topic = "projects/#{@project_id}/topics/#{@topic}"
     @subscription = "projects/#{@project_id}/subscriptions/#{@subscription}"
     @subscription_exists = false
@@ -86,7 +182,7 @@ class LogStash::Inputs::GooglePubSub < LogStash::Inputs::Base
     #       - googleauth ~> 0.3 requires multi_json 1.11.0 that conflicts
     #         with logstash-2.3.2's multi_json 1.11.3
     if @json_key_file
-      @logger.info("Authorizing with JSON key file: #{@json_key_file}")
+      @logger.debug("Authorizing with JSON key file: #{@json_key_file}")
       file_path = File.expand_path(@json_key_file)
       key_json = File.open(file_path, "r", &:read)
       key_json = JSON.parse(key_json)
@@ -116,7 +212,7 @@ class LogStash::Inputs::GooglePubSub < LogStash::Inputs::Base
   def run(queue)
     # Attempt to create the subscription
     if !@subscription_exists
-      @logger.info("Creating subscription #{subscription}")
+      @logger.debug("Creating subscription #{subscription}")
       result = request(
         :api_method => @pubsub.projects.subscriptions.create,
         :parameters => {'name' => @subscription},
@@ -131,7 +227,7 @@ class LogStash::Inputs::GooglePubSub < LogStash::Inputs::Base
       @subscription_exists = true
     end # if !@subscription
 
-    @logger.info("Pulling messages from sub '#{subscription}'")
+    @logger.debug("Pulling messages from sub '#{subscription}'")
     while !stop?
       # Pull and queue messages
       messages = []
@@ -150,7 +246,7 @@ class LogStash::Inputs::GooglePubSub < LogStash::Inputs::Base
           messages = messages["receivedMessages"]
         end
       else
-        @logger.info("Error pulling messages:'#{result.error_message}'")
+        @logger.error("Error pulling messages:'#{result.error_message}'")
       end
 
       if messages
@@ -177,7 +273,7 @@ class LogStash::Inputs::GooglePubSub < LogStash::Inputs::Base
           }
         )
         if result.error?
-          @logger.info("Error #{result.status}: #{result.error_message}")
+          @logger.error("Error #{result.status}: #{result.error_message}")
         end
       end # if messages
     end # loop

data/logstash-input-google_pubsub.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-input-google_pubsub'
-  s.version = '0.9.0'
+  s.version = '1.0.0'
   s.licenses = ['Apache-2.0']
   s.summary = "Logstash input plugin for Google Cloud Pub/Sub."
   s.description = "This gem is a Logstash input plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program."
@@ -18,12 +18,12 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core", ">= 2.0.0", "< 3.0.0"
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'stud', '>= 0.0.22'
   # Google dependencies
   # google-api-client >= 0.9 requires ruby2 which is not currently compatible
   # with JRuby
   s.add_runtime_dependency 'google-api-client', '~> 0.8.6', '< 0.9'
-  s.add_development_dependency 'logstash-devutils', '~>0.0', '>= 0.0.16'
+  s.add_development_dependency 'logstash-devutils'
 end

data/spec/inputs/google_pubsub_spec.rb

@@ -21,7 +21,7 @@ describe LogStash::Inputs::GooglePubSub do
   let(:bad1) { { 'topic' => 'foo', 'subscription' => 'bar' } }
   let(:bad2) { { 'project_id' => 'foo', 'subscription' => 'bar' } }
   let(:bad3) { { 'project_id' => 'foo', 'topic' => 'bar' } }
-  let(:config) { { 'project_id' => 'myproj', 'subscription' => 'foo', 'topic' => 'bar', 'json_key_file' => '/home/erjohnso/fake.json' } }
+  let(:config) { { 'project_id' => 'myproj', 'subscription' => 'foo', 'topic' => 'bar' } }
 
   it "ensures required config options are present" do
     expect {
@@ -34,12 +34,4 @@ describe LogStash::Inputs::GooglePubSub do
       plugin = LogStash::Inputs::GooglePubSub.new(bad3)
     }.to raise_error(LogStash::ConfigurationError)
   end
-
-  it "validates register vars" do
-    plugin = LogStash::Inputs::GooglePubSub.new(config)
-    plugin.register
-    expect(plugin.topic).to eq("projects/myproj/topics/bar")
-    expect(plugin.subscription).to eq("projects/myproj/subscriptions/foo")
-  end
-
 end

metadata

@@ -1,65 +1,64 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-google_pubsub
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Eric Johnson
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-10-04 00:00:00.000000000 Z
+date: 2017-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logstash-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.0
-    - - "<"
+        version: '1.60'
+    - - "<="
       - !ruby/object:Gem::Version
-        version: 3.0.0
-  type: :runtime
+        version: '2.99'
+  name: logstash-core-plugin-api
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.0
-    - - "<"
+        version: '1.60'
+    - - "<="
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: '2.99'
 - !ruby/object:Gem::Dependency
-  name: logstash-codec-plain
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  name: logstash-codec-plain
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: stud
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.0.22
-  type: :runtime
+  name: stud
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.0.22
 - !ruby/object:Gem::Dependency
-  name: google-api-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -68,8 +67,9 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '0.9'
-  type: :runtime
+  name: google-api-client
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -79,28 +79,20 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.9'
 - !ruby/object:Gem::Dependency
-  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.0'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.0.16
-  type: :development
+        version: '0'
+  name: logstash-devutils
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.0'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.0.16
-description: This gem is a Logstash input plugin required to be installed on top of
-  the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
-  gem is not a stand-alone program.
+        version: '0'
+description: This gem is a Logstash input plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program.
 email: erjohnso@google.com
 executables: []
 extensions: []
@@ -121,7 +113,7 @@ licenses:
 metadata:
   logstash_plugin: 'true'
   logstash_group: input
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -136,9 +128,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.1
-signing_key: 
+rubyforge_project:
+rubygems_version: 2.4.8
+signing_key:
 specification_version: 4
 summary: Logstash input plugin for Google Cloud Pub/Sub.
 test_files: