logstash-input-google_pubsub 1.0.6 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 59322ce3c386faf1e2ec7b3805bbce3f07b2b9439bc7086a503269178541fb08
-  data.tar.gz: 4407b4ed498ef6b5a8289978377c69f9ce0af6c10fec5f7fde04a43580e52c85
+  metadata.gz: 3f39efa6b21fafa252ad3e27a1a1a301681145c88498ac242e7b85a77a64382f
+  data.tar.gz: b92fe3cebc49c21702868a61a06a90c88b0f0372b5a7f57272add4a911359c1c
 SHA512:
-  metadata.gz: 2b90a0f77e11d12d9733591a69acf2c3842f4dc68f5ff7a9dca951f43fc7cbd337e1bf54e4d0e27d3198a434797e2fc9ea075a80badf74dad4ec83f45b49a99a
-  data.tar.gz: cce6661d8c84a268f3bb251cf953774cd6aae6db90efe37ed468ef22b5c12b9b2f6e76ca7e5e1a1a4d0c38efeadd16f5aee0ad4072674bb2ffd752b1bbb9f430
+  metadata.gz: a844cf0059e14822f1323a146b4f03b0745ab0f3c5a7c961626f35523059a632ef0169c96cf212612520b1a6ad5c6e428a08c30a82f8a8ffaab15d9c35d6b86b
+  data.tar.gz: 1ca3f6026de8645af0ea71ab648bb52fc76ac4513373d21ad6bd6e39352576b403c56e738905f0f1372182f40e08485fed375b21c7dedd117df866010783d02b

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 1.1.0
+  - Add additional attributes in the `[@metadata][pubsub_message]` field. Fixes [#7](https://github.com/logstash-plugins/logstash-input-google_pubsub/issues/7)
+
 ## 1.0.6
   - Ignore acknowledge requests with an empty array of IDs. Fixes [#14](https://github.com/logstash-plugins/logstash-input-google_pubsub/issues/14)
 

data/docs/index.asciidoc

@@ -138,6 +138,38 @@ input {
 output { stdout { codec => rubydebug } }
 ----------------------------------
 
+==== Metadata and Attributes
+
+The original Pub/Sub message is preserved in the special Logstash 
+`[@metadata][pubsub_message]` field so you can fetch:
+
+* Message attributes
+* The origiginal base64 data
+* Pub/Sub message ID for de-duplication
+* Publish time
+
+You MUST extract any fields you want in a filter prior to the data being sent
+to an output because Logstash deletes `@metadata` fields otherwise.
+
+See the PubsubMessage
+https://cloud.google.com/pubsub/docs/reference/rest/v1/PubsubMessage[documentation]
+for a full description of the fields.
+
+Example to get the message ID:
+
+[source,ruby]
+----------------------------------	
+input {google_pubsub {...}}
+
+filter {
+  mutate {
+    add_field => { "messageId" => "%{[@metadata][pubsub_message][messageId]}" }
+  }
+}
+
+output {...}
+----------------------------------
+
 
 [id="plugins-{type}s-{plugin}-options"]
 ==== Google_pubsub Input Configuration Options
@@ -152,6 +184,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-project_id>> |<<string,string>>|Yes
 | <<plugins-{type}s-{plugin}-subscription>> |<<string,string>>|Yes
 | <<plugins-{type}s-{plugin}-topic>> |<<string,string>>|Yes
+| <<plugins-{type}s-{plugin}-include_metadata>> |<<boolean,boolean>>|No
 |=======================================================================
 
 Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
@@ -209,9 +242,15 @@ Note that the topic must be created manually with Cloud Logging
 pre-configured export to PubSub configured to use the defined topic.
 The subscription will be created automatically by the plugin.
 
+[id="plugins-{type}s-{plugin}-include_metadata"]
+===== `include_metadata` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`.
 
+If set true, will include the full message data in the `[@metadata][pubsub_message]` field.
 
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]
 
-:default_codec!:
+:default_codec!:

data/lib/logstash/inputs/google_pubsub.rb

@@ -122,6 +122,38 @@ require "google/api_client"
 # output { stdout { codec => rubydebug } }
 # ----------------------------------
 #
+# ==== Metadata and Attributes
+# 
+# The original Pub/Sub message is preserved in the special Logstash 
+# `[@metadata][pubsub_message]` field so you can fetch:
+# 
+# * Message attributes
+# * The origiginal base64 data
+# * Pub/Sub message ID for de-duplication
+# * Publish time
+# 
+# You MUST extract any fields you want in a filter prior to the data being sent
+# to an output because Logstash deletes `@metadata` fields otherwise.
+# 
+# See the PubsubMessage
+# https://cloud.google.com/pubsub/docs/reference/rest/v1/PubsubMessage[documentation]
+# for a full description of the fields.
+# 
+# Example to get the message ID:
+# 
+# [source,ruby]
+# ----------------------------------	
+# input {google_pubsub {...}}
+# 
+# filter {
+#   mutate {
+#     add_field => { "messageId" => "%{[@metadata][pubsub_message][messageId]}" }
+#   }
+# }
+# 
+# output {...}
+# ----------------------------------
+#
 class LogStash::Inputs::GooglePubSub < LogStash::Inputs::Base
   config_name "google_pubsub"
 
@@ -141,6 +173,9 @@ class LogStash::Inputs::GooglePubSub < LogStash::Inputs::Base
   # specify a Service Account JSON key file.
   config :json_key_file, :validate => :path, :required => false
 
+  # If set true, will include the full message data in the `[@metadata][pubsub_message]` field.
+  config :include_metadata, :validate => :boolean, :required => false, :default => false
+
   # If undefined, Logstash will complain, even if codec is unused.
   default :codec, "plain"
 
@@ -254,6 +289,7 @@ class LogStash::Inputs::GooglePubSub < LogStash::Inputs::Base
           if msg.key?("message") and msg["message"].key?("data")
             decoded_msg = Base64.decode64(msg["message"]["data"])
             @codec.decode(decoded_msg) do |event|
+              event.set("[@metadata][pubsub_message]", msg["message"]) if @include_metadata
               decorate(event)
               queue << event
             end

data/logstash-input-google_pubsub.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-input-google_pubsub'
-  s.version         = '1.0.6'
+  s.version         = '1.1.0'
   s.licenses = ['Apache-2.0']
   s.summary = "Consume events from a Google Cloud PubSub service"
   s.description = "This gem is a Logstash input plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program."

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-google_pubsub
 version: !ruby/object:Gem::Version
-  version: 1.0.6
+  version: 1.1.0
 platform: ruby
 authors:
 - Eric Johnson
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-04-12 00:00:00.000000000 Z
+date: 2018-04-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement