logstash-input-google_pubsub 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 7b261bb9333fce81b2dc53f8474d332a16de4519
-  data.tar.gz: fee0a4acafa699ced1feb58cedfaeaea9bcd1fa8
+SHA256:
+  metadata.gz: e41ed92888f32a05a1d12094c6ff419be0c6949f39b71c3c084f9b522eb463bb
+  data.tar.gz: 6ade48556636b5f2c845177bb1536cf537339ff8e8619b8fb6f912b202219078
 SHA512:
-  metadata.gz: d54ac43c05cc10acce0a696e067a681bb1155ed3388f5f2b47d99e936c337259086eb32944793405e84163e71246b25eaf70e63aa8dab8b1ccd7c6b647044ff5
-  data.tar.gz: e54671aae537ff2cd6176f4c38a61912ab0be6fdc2a3ad91d544eb375ecb3179b66a33344fe69744d29373b89e00a45a98f716645a913ac1e788126cc9d68b8e
+  metadata.gz: e16e74cef0eb0a49e04727c3bbdbae707e43a23de33cd16fafa3fee3edf77853d1730fa8818b3ba73df7e73030b7122676f988479174e53071161c4a9de5e75f
+  data.tar.gz: e87a18a54dfb5a88291a419babf3db4ce01b291d746aa1ebaae9af52250ced8e0b0cdc36ce02686c43c9993de17dcd78b47ecf7ed8ed05c51494aef9c222c884

data/Gemfile

@@ -1,2 +1,11 @@
 source 'https://rubygems.org'
+
 gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
+use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+
+if Dir.exist?(logstash_path) && use_logstash_source
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/docs/index.asciidoc

@@ -0,0 +1,213 @@
+:plugin: google_pubsub
+:type: input
+
+///////////////////////////////////////////
+START - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+:version: %VERSION%
+:release_date: %RELEASE_DATE%
+:changelog_url: %CHANGELOG_URL%
+:include_path: ../../../../logstash/docs/include
+///////////////////////////////////////////
+END - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+
+[id="plugins-{type}-{plugin}"]
+
+=== Google_pubsub input plugin
+
+include::{include_path}/plugin_header.asciidoc[]
+
+==== Description
+
+Author: Eric Johnson <erjohnso@google.com>
+Date: 2016-06-01
+
+Copyright 2016 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+Google deps
+This is a https://github.com/elastic/logstash[Logstash] input plugin for 
+https://cloud.google.com/pubsub/[Google Pub/Sub]. The plugin can subscribe 
+to a topic and ingest messages.
+
+The main motivation behind the development of this plugin was to ingest 
+https://cloud.google.com/logging/[Stackdriver Logging] messages via the 
+https://cloud.google.com/logging/docs/export/using_exported_logs[Exported Logs] 
+feature of Stackdriver Logging.
+
+==== Prerequisites
+
+You must first create a Google Cloud Platform project and enable the the 
+Google Pub/Sub API. If you intend to use the plugin ingest Stackdriver Logging 
+messages, you must also enable the Stackdriver Logging API and configure log 
+exporting to Pub/Sub. There is plentiful information on 
+https://cloud.google.com/ to get started: 
+
+- Google Cloud Platform Projects and https://cloud.google.com/docs/overview/[Overview]
+- Google Cloud Pub/Sub https://cloud.google.com/pubsub/[documentation]
+- Stackdriver Logging https://cloud.google.com/logging/[documentation]
+
+==== Cloud Pub/Sub
+
+Currently, this module requires you to create a `topic` manually and specify 
+it in the logstash config file. You must also specify a `subscription`, but 
+the plugin will attempt to create the pull-based `subscription` on its own. 
+
+All messages received from Pub/Sub will be converted to a logstash `event` 
+and added to the processing pipeline queue. All Pub/Sub messages will be 
+`acknowledged` and removed from the Pub/Sub `topic` (please see more about 
+https://cloud.google.com/pubsub/overview#concepts)[Pub/Sub concepts]. 
+
+It is generally assumed that incoming messages will be in JSON and added to 
+the logstash `event` as-is. However, if a plain text message is received, the 
+plugin will return the raw text in as `raw_message` in the logstash `event`. 
+
+==== Authentication
+
+You have two options for authentication depending on where you run Logstash. 
+
+1. If you are running Logstash outside of Google Cloud Platform, then you will 
+need to create a Google Cloud Platform Service Account and specify the full 
+path to the JSON private key file in your config. You must assign sufficient 
+roles to the Service Account to create a subscription and to pull messages 
+from the subscription. Learn more about GCP Service Accounts and IAM roles 
+here:
+
+  - Google Cloud Platform IAM https://cloud.google.com/iam/[overview]
+  - Creating Service Accounts https://cloud.google.com/iam/docs/creating-managing-service-accounts[overview]
+  - Granting Roles https://cloud.google.com/iam/docs/granting-roles-to-service-accounts[overview]
+
+1. If you are running Logstash on a Google Compute Engine instance, you may opt 
+to use Application Default Credentials. In this case, you will not need to 
+specify a JSON private key file in your config.
+
+==== Stackdriver Logging (optional)
+
+If you intend to use the logstash plugin for Stackdriver Logging message 
+ingestion, you must first manually set up the Export option to Cloud Pub/Sub and 
+the manually create the `topic`. Please see the more detailed instructions at, 
+https://cloud.google.com/logging/docs/export/using_exported_logs [Exported Logs] 
+and ensure that the https://cloud.google.com/logging/docs/export/configure_export#manual-access-pubsub[necessary permissions] 
+have also been manually configured.
+
+Logging messages from Stackdriver Logging exported to Pub/Sub are received as 
+JSON and converted to a logstash `event` as-is in 
+https://cloud.google.com/logging/docs/export/using_exported_logs#log_entries_in_google_pubsub_topics[this format].
+
+==== Sample Configuration
+
+Below is a copy of the included `example.conf-tmpl` file that shows a basic 
+configuration for this plugin.
+
+[source,ruby]
+----------------------------------
+input {
+    google_pubsub {
+        # Your GCP project id (name)
+        project_id => "my-project-1234"
+
+        # The topic name below is currently hard-coded in the plugin. You
+        # must first create this topic by hand and ensure you are exporting
+        # logging to this pubsub topic.
+        topic => "logstash-input-dev"
+
+        # The subscription name is customizeable. The plugin will attempt to
+        # create the subscription (but use the hard-coded topic name above).
+        subscription => "logstash-sub"
+
+        # If you are running logstash within GCE, it will use
+        # Application Default Credentials and use GCE's metadata
+        # service to fetch tokens.  However, if you are running logstash
+        # outside of GCE, you will need to specify the service account's
+        # JSON key file below.
+        #json_key_file => "/home/erjohnso/pkey.json"
+    }
+}
+output { stdout { codec => rubydebug } }
+----------------------------------
+
+
+[id="plugins-{type}s-{plugin}-options"]
+==== Google_pubsub Input Configuration Options
+
+This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-json_key_file>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-max_messages>> |<<number,number>>|Yes
+| <<plugins-{type}s-{plugin}-project_id>> |<<string,string>>|Yes
+| <<plugins-{type}s-{plugin}-subscription>> |<<string,string>>|Yes
+| <<plugins-{type}s-{plugin}-topic>> |<<string,string>>|Yes
+|=======================================================================
+
+Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
+input plugins.
+
+&nbsp;
+
+[id="plugins-{type}s-{plugin}-json_key_file"]
+===== `json_key_file` 
+
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
+
+If logstash is running within Google Compute Engine, the plugin will use
+GCE's Application Default Credentials. Outside of GCE, you will need to
+specify a Service Account JSON key file.
+
+[id="plugins-{type}s-{plugin}-max_messages"]
+===== `max_messages` 
+
+  * This is a required setting.
+  * Value type is <<number,number>>
+  * Default value is `5`
+
+
+
+[id="plugins-{type}s-{plugin}-project_id"]
+===== `project_id` 
+
+  * This is a required setting.
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Google Cloud Project ID (name, not number)
+
+[id="plugins-{type}s-{plugin}-subscription"]
+===== `subscription` 
+
+  * This is a required setting.
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+
+
+[id="plugins-{type}s-{plugin}-topic"]
+===== `topic` 
+
+  * This is a required setting.
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Google Cloud Pub/Sub Topic and Subscription.
+Note that the topic must be created manually with Cloud Logging
+pre-configured export to PubSub configured to use the defined topic.
+The subscription will be created automatically by the plugin.
+
+
+
+[id="plugins-{type}s-{plugin}-common-options"]
+include::{include_path}/{type}.asciidoc[]

data/logstash-input-google_pubsub.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-input-google_pubsub'
-  s.version = '1.0.0'
+  s.version         = '1.0.1'
   s.licenses = ['Apache-2.0']
   s.summary = "Logstash input plugin for Google Cloud Pub/Sub."
   s.description = "This gem is a Logstash input plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program."
@@ -10,7 +10,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
    # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-google_pubsub
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Eric Johnson
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-03-28 00:00:00.000000000 Z
+date: 2017-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -92,7 +92,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: This gem is a Logstash input plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program.
+description: This gem is a Logstash input plugin required to be installed on top of
+  the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program.
 email: erjohnso@google.com
 executables: []
 extensions: []
@@ -104,6 +106,7 @@ files:
 - LICENSE
 - NOTICE.TXT
 - README.md
+- docs/index.asciidoc
 - lib/logstash/inputs/google_pubsub.rb
 - logstash-input-google_pubsub.gemspec
 - spec/inputs/google_pubsub_spec.rb
@@ -129,7 +132,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.8
+rubygems_version: 2.6.11
 signing_key:
 specification_version: 4
 summary: Logstash input plugin for Google Cloud Pub/Sub.