logstash-input-google_pubsub 1.1.0 → 1.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3f39efa6b21fafa252ad3e27a1a1a301681145c88498ac242e7b85a77a64382f
-  data.tar.gz: b92fe3cebc49c21702868a61a06a90c88b0f0372b5a7f57272add4a911359c1c
+  metadata.gz: cc1279ce3f75158f4a960136826fc80a0a936ab358ca1d05b5d673eaaf05ce10
+  data.tar.gz: dfa594834563c91e1a482ab8ad5bcb4c64702a99dc2678751d1070bb21285858
 SHA512:
-  metadata.gz: a844cf0059e14822f1323a146b4f03b0745ab0f3c5a7c961626f35523059a632ef0169c96cf212612520b1a6ad5c6e428a08c30a82f8a8ffaab15d9c35d6b86b
-  data.tar.gz: 1ca3f6026de8645af0ea71ab648bb52fc76ac4513373d21ad6bd6e39352576b403c56e738905f0f1372182f40e08485fed375b21c7dedd117df866010783d02b
+  metadata.gz: 18ab4cc3cfd54e55a32fe89d4e0847d027ab1e7eeb7a5f9dbf3abd0591597a9b4cf56f2da5fec71a43f3a08e1458169de0301bab6d1cff04a9b6ecfbe08c88a7
+  data.tar.gz: 4a274d96d6f67f9555755c7465b9c9d97586d789f29a37853f5a782ff2e0698c760828036c3eabc002abbd322889fedb994b77cf9cbfa8bee0412117b4c6c7a5

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## 1.2.2
+ - Updated dependencies forcing this plugin to be compatible only with Logstash >= 8.2.0 [#58](https://github.com/logstash-plugins/logstash-input-google_pubsub/pull/58)
+
+## 1.2.1
+ - Fixed dependency conflicts with logstash-output-google_pubsub by using the same client library [logstash-output-google_pubsub#7](https://github.com/logstash-plugins/logstash-output-google_pubsub/issues/7)
+
+## 1.2.0
+ - Change to Java client
+ - Add `create_subscription` setting. Fixes [#9](https://github.com/logstash-plugins/logstash-input-google_pubsub/issues/9)
+
 ## 1.1.0
   - Add additional attributes in the `[@metadata][pubsub_message]` field. Fixes [#7](https://github.com/logstash-plugins/logstash-input-google_pubsub/issues/7)
 

data/CONTRIBUTORS

@@ -0,0 +1,23 @@
+The following is a list of people who have contributed ideas, code, bug
+reports, or in general have helped Logstash along its way.
+
+Contributors:
+
+* DeDe Morton (dedemorton)
+* Eric Johnson (erjohnso)
+* Google LLC
+* Jake Landis (jakelandis)
+* Jakub Sacha (jakubsacha)
+* James Bloomer (jamesbloomer)
+* João Duarte (jsvd)
+* Jordan Sissel (jordansissel)
+* Joseph Lewis III (josephlewis42)
+* Pier-Hugues Pellerin (ph)
+* Scott Brenner (ScottBrenner)
+* Suyog Rao (suyograo)
+* Sven Widén (SvenW)
+
+Note: If you've sent us patches, bug reports, or otherwise contributed to
+Logstash, and you aren't on the list above and want to be, please let us know
+and we'll make sure you're here. Contributions from folks like you are what make
+open source awesome.

data/LICENSE

@@ -1,13 +1,202 @@
-Copyright (c) 2012-2018 Elasticsearch <http://www.elastic.co>
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
 
-    http://www.apache.org/licenses/LICENSE-2.0
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
 
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2020 Elastic and contributors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -1,6 +1,6 @@
 # Logstash Plugin
 
-[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-input-google_pubsub.svg)](https://travis-ci.org/logstash-plugins/logstash-input-google_pubsub)
+[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-input-google_pubsub.svg)](https://travis-ci.com/logstash-plugins/logstash-input-google_pubsub)
 
 This is a [Logstash](https://github.com/elastic/logstash) input plugin for
 [Google Pub/Sub](https://cloud.google.com/pubsub/). The plugin can subscribe
@@ -130,6 +130,7 @@ Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/log
 - Install dependencies
 ```sh
 bundle install
+rake install_jars
 ```
 
 #### Test

data/docs/index.asciidoc

@@ -133,6 +133,11 @@ input {
         # outside of GCE, you will need to specify the service account's
         # JSON key file below.
         #json_key_file => "/home/erjohnso/pkey.json"
+
+        # Should the plugin attempt to create the subscription on startup?
+        # This is not recommended for security reasons but may be useful in
+        # some cases.
+        #create_subscription => false
     }
 }
 output { stdout { codec => rubydebug } }
@@ -185,6 +190,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-subscription>> |<<string,string>>|Yes
 | <<plugins-{type}s-{plugin}-topic>> |<<string,string>>|Yes
 | <<plugins-{type}s-{plugin}-include_metadata>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-create_subscription>> |<<boolean,boolean>>|No
 |=======================================================================
 
 Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
@@ -250,6 +256,18 @@ The subscription will be created automatically by the plugin.
 
 If set true, will include the full message data in the `[@metadata][pubsub_message]` field.
 
+[id="plugins-{type}s-{plugin}-create_subscription"]
+===== `create_subscription`
+
+added[1.2.0]
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`.
+
+If true, the plugin will try to create the subscription before publishing.
+Note: this requires additional permissions to be granted to the client and is _not_
+recommended for most use-cases.
+
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]
 

data/lib/logstash/inputs/google_pubsub.rb

@@ -19,8 +19,8 @@
 require "logstash/inputs/base"
 require "logstash/namespace"
 
-# Google deps
-require "google/api_client"
+require 'java'
+require 'logstash-input-google_pubsub_jars.rb'
 
 # This is a https://github.com/elastic/logstash[Logstash] input plugin for 
 # https://cloud.google.com/pubsub/[Google Pub/Sub]. The plugin can subscribe 
@@ -123,38 +123,70 @@ require "google/api_client"
 # ----------------------------------
 #
 # ==== Metadata and Attributes
-# 
-# The original Pub/Sub message is preserved in the special Logstash 
+#
+# The original Pub/Sub message is preserved in the special Logstash
 # `[@metadata][pubsub_message]` field so you can fetch:
-# 
+#
 # * Message attributes
 # * The origiginal base64 data
 # * Pub/Sub message ID for de-duplication
 # * Publish time
-# 
+#
 # You MUST extract any fields you want in a filter prior to the data being sent
 # to an output because Logstash deletes `@metadata` fields otherwise.
-# 
+#
 # See the PubsubMessage
 # https://cloud.google.com/pubsub/docs/reference/rest/v1/PubsubMessage[documentation]
 # for a full description of the fields.
-# 
+#
 # Example to get the message ID:
-# 
+#
 # [source,ruby]
-# ----------------------------------	
+# ----------------------------------
 # input {google_pubsub {...}}
-# 
+#
 # filter {
 #   mutate {
 #     add_field => { "messageId" => "%{[@metadata][pubsub_message][messageId]}" }
 #   }
 # }
-# 
+#
 # output {...}
 # ----------------------------------
 #
+
 class LogStash::Inputs::GooglePubSub < LogStash::Inputs::Base
+  class MessageReceiver
+    include com.google.cloud.pubsub.v1.MessageReceiver
+
+    def initialize(&blk)
+      @block = blk
+    end
+
+    def receiveMessage(message, consumer)
+      @block.call(message)
+      consumer.ack()
+    end
+  end
+
+  java_import 'com.google.api.core.ApiService$Listener'
+  class SubscriberListener < Listener
+    def initialize(&blk)
+      @block = blk
+    end
+
+    def failed(from, failure)
+      @block.call(from, failure)
+    end
+  end
+
+  include_package 'com.google.api.gax.batching'
+  include_package 'com.google.api.gax.core'
+  include_package 'com.google.auth.oauth2'
+  include_package 'com.google.common.util.concurrent'
+  include_package 'com.google.cloud.pubsub.v1'
+  include_package 'com.google.pubsub.v1'
+  include_package 'com.google.protobuf.util'
   config_name "google_pubsub"
 
   # Google Cloud Project ID (name, not number)
@@ -176,140 +208,81 @@ class LogStash::Inputs::GooglePubSub < LogStash::Inputs::Base
   # If set true, will include the full message data in the `[@metadata][pubsub_message]` field.
   config :include_metadata, :validate => :boolean, :required => false, :default => false
 
+  # If true, the plugin will try to create the subscription before publishing.
+  # Note: this requires additional permissions to be granted to the client and is _not_
+  # recommended for most use-cases.
+  config :create_subscription, :validate => :boolean, :required => false, :default => false
+
   # If undefined, Logstash will complain, even if codec is unused.
   default :codec, "plain"
 
-  private
-  def request(options)
-    begin
-      @logger.debug("Sending an API request")
-      result = @client.execute(options)
-    rescue ArgumentError => e
-      @logger.debug("Authorizing...")
-      @client.authorization.fetch_access_token!
-      @logger.debug("...authorized")
-      request(options)
-    rescue Faraday::TimeoutError => e
-      @logger.debug("Request timeout, re-trying request")
-      request(options)
-    end
-  end # def request
-
   public
   def register
     @logger.debug("Registering Google PubSub Input: project_id=#{@project_id}, topic=#{@topic}, subscription=#{@subscription}")
-    @topic = "projects/#{@project_id}/topics/#{@topic}"
-    @subscription = "projects/#{@project_id}/subscriptions/#{@subscription}"
-    @subscription_exists = false
-
-    # TODO(erjohnso): read UA data from the gemspec
-    @client = Google::APIClient.new(
-      :application_name => 'logstash-input-google_pubsub',
-      :application_version => '0.9.0'
-    )
+    @subscription_id = "projects/#{@project_id}/subscriptions/#{@subscription}"
 
-    # Initialize the pubsub API client
-    @pubsub = @client.discovered_api('pubsub', 'v1')
-
-    # Handle various kinds of auth (JSON or Application Default Creds)
-    # NOTE: Cannot use 'googleauth' gem since there are dependency conflicts
-    #       - googleauth ~> 0.5 requires mime-data-types that requires ruby2
-    #       - googleauth ~> 0.3 requires multi_json 1.11.0 that conflicts
-    #         with logstash-2.3.2's multi_json 1.11.3
     if @json_key_file
-      @logger.debug("Authorizing with JSON key file: #{@json_key_file}")
-      file_path = File.expand_path(@json_key_file)
-      key_json = File.open(file_path, "r", &:read)
-      key_json = JSON.parse(key_json)
-      unless key_json.key?("client_email") || key_json.key?("private_key")
-        raise Google::APIClient::ClientError, "Invalid JSON credentials data."
-      end
-      signing_key = ::Google::APIClient::KeyUtils.load_from_pem(key_json["private_key"], "notasecret")
-      @client.authorization = Signet::OAuth2::Client.new(
-        :audience => "https://accounts.google.com/o/oauth2/token",
-        :auth_provider_x509_cert_url => "https://www.googleapis.com/oauth2/v1/certs",
-        :client_x509_cert_url => "https://www.googleapis.com/robot/v1/metadata/x509/#{key_json['client_email']}",
-        :issuer => "#{key_json['client_email']}",
-        :scope => %w(https://www.googleapis.com/auth/cloud-platform),
-        :signing_key => signing_key,
-        :token_credential_uri => "https://accounts.google.com/o/oauth2/token"
+      @credentialsProvider = FixedCredentialsProvider.create(
+        ServiceAccountCredentials.fromStream(java.io.FileInputStream.new(@json_key_file))
       )
-      @logger.info("Client authorization with JSON key ready")
-    else
-      # Assume we're running in GCE and can use metadata tokens, if the host
-      # GCE instance was not created with the PubSub scope, then the plugin
-      # will not be authorized to read from pubsub.
-      @logger.info("Authorizing with application default credentials")
-      @client.authorization = :google_app_default
-    end # if @json_key_file...
-  end # def register
+    end
+    @topic_name = ProjectTopicName.of(@project_id, @topic)
+    @subscription_name = ProjectSubscriptionName.of(@project_id, @subscription)
+  end
+
+  def stop
+    @subscriber.stopAsync().awaitTerminated() if @subscriber != nil
+  end
 
   def run(queue)
     # Attempt to create the subscription
-    if !@subscription_exists
-      @logger.debug("Creating subscription #{subscription}")
-      result = request(
-        :api_method => @pubsub.projects.subscriptions.create,
-        :parameters => {'name' => @subscription},
-        :body_object => {
-          :topic => @topic,
-          :ackDeadlineSeconds => 15
-        }
-      )
-      if result.error? and result.status != 409
-        raise Google::APIClient::ClientError, "Error #{result.status}: #{result.error_message}"
+    if @create_subscription
+      @logger.debug("Creating subscription #{@subscription_id}")
+      subscriptionAdminClient = SubscriptionAdminClient.create
+      begin
+        subscriptionAdminClient.createSubscription(@subscription_name, @topic_name, PushConfig.getDefaultInstance(), 0)
+      rescue
+        @logger.info("Subscription already exists")
       end
-      @subscription_exists = true
-    end # if !@subscription
-
-    @logger.debug("Pulling messages from sub '#{subscription}'")
-    while !stop?
-      # Pull and queue messages
-      messages = []
-      result = request(
-        :api_method => @pubsub.projects.subscriptions.pull,
-        :parameters => {'subscription' => @subscription},
-        :body_object => {
-          :returnImmediately => false,
-          :maxMessages => @max_messages
-        }
-      )
+    end
 
-      if !result.error?
-        messages = JSON.parse(result.body)
-        if messages.key?("receivedMessages")
-          messages = messages["receivedMessages"]
-        end
-      else
-        @logger.error("Error pulling messages:'#{result.error_message}'")
+    @logger.debug("Pulling messages from sub '#{@subscription_id}'")
+    handler = MessageReceiver.new do |message|
+      # handle incoming message, then ack/nack the received message
+      data = message.getData().toStringUtf8()
+      @codec.decode(data) do |event|
+        event.set("host", event.get("host") || @host)
+        event.set("[@metadata][pubsub_message]", extract_metadata(message)) if @include_metadata
+        decorate(event)
+        queue << event
       end
+    end
+    listener = SubscriberListener.new do |from, failure|
+      @logger.error("#{failure}")
+      raise failure
+    end
+    flowControlSettings = FlowControlSettings.newBuilder().setMaxOutstandingElementCount(@max_messages).build()
+    executorProvider = InstantiatingExecutorProvider.newBuilder().setExecutorThreadCount(1).build()
+    subscriberBuilder = Subscriber.newBuilder(@subscription_name, handler)
+      .setFlowControlSettings(flowControlSettings)
+      .setExecutorProvider(executorProvider)
+      .setParallelPullCount(1)
 
-      if messages
-        messages.each do |msg|
-          if msg.key?("message") and msg["message"].key?("data")
-            decoded_msg = Base64.decode64(msg["message"]["data"])
-            @codec.decode(decoded_msg) do |event|
-              event.set("[@metadata][pubsub_message]", msg["message"]) if @include_metadata
-              decorate(event)
-              queue << event
-            end
-          end
-        end
-
-        ack_ids = messages.map{ |msg| msg["ackId"] }
-        next if ack_ids.empty?
+    if @credentialsProvider
+      subscriberBuilder.setCredentialsProvider(@credentialsProvider)
+    end
+    @subscriber = subscriberBuilder.build()
+    @subscriber.addListener(listener, MoreExecutors.directExecutor())
+    @subscriber.startAsync()
+    @subscriber.awaitTerminated()
+  end
 
-        result = request(
-          :api_method => @pubsub.projects.subscriptions.acknowledge,
-          :parameters => {'subscription' => @subscription},
-          :body_object => {
-            :ackIds => ack_ids
-          }
-        )
-        if result.error?
-          @logger.error("Error #{result.status}: #{result.error_message}")
-        end
-      end # if messages
-    end # loop
-  end # def run
-end # class LogStash::Inputs::GooglePubSub
+  def extract_metadata(java_message)
+    {
+      data: java_message.getData().toStringUtf8(),
+      attributes: java_message.getAttributesMap(),
+      messageId: java_message.getMessageId(),
+      publishTime: Timestamps.toString(java_message.getPublishTime())
+    }
+  end
+end

data/lib/logstash-input-google_pubsub_jars.rb

@@ -0,0 +1,54 @@
+# AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
+
+require 'jar_dependencies'
+require_jar('com.google.cloud', 'google-cloud-pubsub', '1.120.13')
+require_jar('io.grpc', 'grpc-api', '1.48.0')
+require_jar('io.grpc', 'grpc-context', '1.48.0')
+require_jar('io.grpc', 'grpc-stub', '1.48.0')
+require_jar('io.grpc', 'grpc-protobuf', '1.48.0')
+require_jar('io.grpc', 'grpc-protobuf-lite', '1.48.0')
+require_jar('com.google.api', 'api-common', '2.2.1')
+require_jar('javax.annotation', 'javax.annotation-api', '1.3.2')
+require_jar('com.google.protobuf', 'protobuf-java', '3.21.4')
+require_jar('com.google.api.grpc', 'proto-google-common-protos', '2.9.2')
+require_jar('com.google.auth', 'google-auth-library-oauth2-http', '1.8.1')
+require_jar('com.google.auth', 'google-auth-library-credentials', '1.8.1')
+require_jar('com.google.http-client', 'google-http-client-gson', '1.42.2')
+require_jar('com.google.api.grpc', 'proto-google-cloud-pubsub-v1', '1.102.13')
+require_jar('com.google.auto.value', 'auto-value-annotations', '1.9')
+require_jar('com.google.guava', 'failureaccess', '1.0.1')
+require_jar('com.google.guava', 'listenablefuture', '9999.0-empty-to-avoid-conflict-with-guava')
+require_jar('com.google.code.findbugs', 'jsr305', '3.0.2')
+require_jar('org.checkerframework', 'checker-qual', '3.23.0')
+require_jar('com.google.errorprone', 'error_prone_annotations', '2.14.0')
+require_jar('com.google.j2objc', 'j2objc-annotations', '1.3')
+require_jar('com.google.api.grpc', 'proto-google-iam-v1', '1.5.2')
+require_jar('com.google.guava', 'guava', '31.1-jre')
+require_jar('com.google.api', 'gax', '2.18.7')
+require_jar('com.google.api', 'gax-grpc', '2.18.7')
+require_jar('io.grpc', 'grpc-alts', '1.48.0')
+require_jar('io.grpc', 'grpc-grpclb', '1.48.0')
+require_jar('org.conscrypt', 'conscrypt-openjdk-uber', '2.5.2')
+require_jar('io.grpc', 'grpc-auth', '1.48.0')
+require_jar('io.grpc', 'grpc-netty-shaded', '1.48.0')
+require_jar('io.grpc', 'grpc-googleapis', '1.48.0')
+require_jar('io.grpc', 'grpc-xds', '1.48.0')
+require_jar('io.grpc', 'grpc-services', '1.48.0')
+require_jar('com.google.re2j', 're2j', '1.6')
+require_jar('io.opencensus', 'opencensus-proto', '0.2.0')
+require_jar('com.google.api', 'gax-httpjson', '0.103.7')
+require_jar('com.google.code.gson', 'gson', '2.9.1')
+require_jar('com.google.protobuf', 'protobuf-java-util', '3.21.4')
+require_jar('org.threeten', 'threetenbp', '1.6.0')
+require_jar('io.opencensus', 'opencensus-api', '0.31.1')
+require_jar('io.grpc', 'grpc-core', '1.48.0')
+require_jar('com.google.android', 'annotations', '4.1.1.4')
+require_jar('org.codehaus.mojo', 'animal-sniffer-annotations', '1.21')
+require_jar('io.perfmark', 'perfmark-api', '0.25.0')
+require_jar('com.google.http-client', 'google-http-client', '1.42.2')
+require_jar('org.apache.httpcomponents', 'httpclient', '4.5.13')
+require_jar('commons-logging', 'commons-logging', '1.2')
+require_jar('commons-codec', 'commons-codec', '1.15')
+require_jar('org.apache.httpcomponents', 'httpcore', '4.4.15')
+require_jar('io.opencensus', 'opencensus-contrib-http-util', '0.31.1')
+require_jar('org.logstash.inputs', 'plugin', '1.2.2')

data/logstash-input-google_pubsub.gemspec

@@ -1,13 +1,13 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-input-google_pubsub'
-  s.version         = '1.1.0'
+  s.version         = ::File.read('version').split("\n").first
   s.licenses = ['Apache-2.0']
   s.summary = "Consume events from a Google Cloud PubSub service"
   s.description = "This gem is a Logstash input plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program."
   s.authors = ["Eric Johnson"]
   s.email = 'erjohnso@google.com'
   s.homepage = "https://cloud.google.com/pubsub/overview"
-  s.require_paths = ["lib"]
+  s.require_paths = ["lib", "vendor/jar-dependencies"]
 
   # Files
   s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
@@ -18,12 +18,18 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
 
   # Gem dependencies
+  s.add_runtime_dependency 'logstash-core', '>= 8.2.0'
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'stud', '>= 0.0.22'
   # Google dependencies
-  # google-api-client >= 0.9 requires ruby2 which is not currently compatible
   # with JRuby
-  s.add_runtime_dependency 'google-api-client', '~> 0.8.6', '< 0.9'
+  s.requirements << "jar 'com.google.cloud:google-cloud-pubsub', '0.28.0-beta'"
+  s.requirements << "jar 'com.google.api.grpc:proto-google-cloud-pubsub-v1', '0.1.24'"
+  s.requirements << "jar 'com.google.api:gax', '1.14.0'"
+  s.requirements << "jar 'com.google.guava:guava', '20.0'"
+  s.requirements << "jar 'com.google.api:api-common', '1.2.0'"
+  s.requirements << "jar 'com.google.auth:google-auth-library-oauth2-http', '0.9.0'"
   s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'jar-dependencies', '~> 0.4'
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-google_pubsub
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.2
 platform: ruby
 authors:
 - Eric Johnson
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-04-26 00:00:00.000000000 Z
+date: 2022-09-13 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 8.2.0
+  name: logstash-core
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 8.2.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -61,37 +75,31 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.8.6
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.9'
-  name: google-api-client
+        version: '0'
+  name: logstash-devutils
   prerelease: false
-  type: :runtime
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.8.6
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  name: logstash-devutils
+        version: '0.4'
+  name: jar-dependencies
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.4'
 description: This gem is a Logstash input plugin required to be installed on top of
   the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
   gem is not a stand-alone program.
@@ -102,14 +110,67 @@ extra_rdoc_files: []
 files:
 - CHANGELOG.md
 - CONTRIBUTING.md
+- CONTRIBUTORS
 - Gemfile
 - LICENSE
 - NOTICE.TXT
 - README.md
 - docs/index.asciidoc
+- lib/logstash-input-google_pubsub_jars.rb
 - lib/logstash/inputs/google_pubsub.rb
 - logstash-input-google_pubsub.gemspec
 - spec/inputs/google_pubsub_spec.rb
+- vendor/jar-dependencies/com/google/android/annotations/4.1.1.4/annotations-4.1.1.4.jar
+- vendor/jar-dependencies/com/google/api/api-common/2.2.1/api-common-2.2.1.jar
+- vendor/jar-dependencies/com/google/api/gax-grpc/2.18.7/gax-grpc-2.18.7.jar
+- vendor/jar-dependencies/com/google/api/gax-httpjson/0.103.7/gax-httpjson-0.103.7.jar
+- vendor/jar-dependencies/com/google/api/gax/2.18.7/gax-2.18.7.jar
+- vendor/jar-dependencies/com/google/api/grpc/proto-google-cloud-pubsub-v1/1.102.13/proto-google-cloud-pubsub-v1-1.102.13.jar
+- vendor/jar-dependencies/com/google/api/grpc/proto-google-common-protos/2.9.2/proto-google-common-protos-2.9.2.jar
+- vendor/jar-dependencies/com/google/api/grpc/proto-google-iam-v1/1.5.2/proto-google-iam-v1-1.5.2.jar
+- vendor/jar-dependencies/com/google/auth/google-auth-library-credentials/1.8.1/google-auth-library-credentials-1.8.1.jar
+- vendor/jar-dependencies/com/google/auth/google-auth-library-oauth2-http/1.8.1/google-auth-library-oauth2-http-1.8.1.jar
+- vendor/jar-dependencies/com/google/auto/value/auto-value-annotations/1.9/auto-value-annotations-1.9.jar
+- vendor/jar-dependencies/com/google/cloud/google-cloud-pubsub/1.120.13/google-cloud-pubsub-1.120.13.jar
+- vendor/jar-dependencies/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar
+- vendor/jar-dependencies/com/google/code/gson/gson/2.9.1/gson-2.9.1.jar
+- vendor/jar-dependencies/com/google/errorprone/error_prone_annotations/2.14.0/error_prone_annotations-2.14.0.jar
+- vendor/jar-dependencies/com/google/guava/failureaccess/1.0.1/failureaccess-1.0.1.jar
+- vendor/jar-dependencies/com/google/guava/guava/31.1-jre/guava-31.1-jre.jar
+- vendor/jar-dependencies/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar
+- vendor/jar-dependencies/com/google/http-client/google-http-client-gson/1.42.2/google-http-client-gson-1.42.2.jar
+- vendor/jar-dependencies/com/google/http-client/google-http-client/1.42.2/google-http-client-1.42.2.jar
+- vendor/jar-dependencies/com/google/j2objc/j2objc-annotations/1.3/j2objc-annotations-1.3.jar
+- vendor/jar-dependencies/com/google/protobuf/protobuf-java-util/3.21.4/protobuf-java-util-3.21.4.jar
+- vendor/jar-dependencies/com/google/protobuf/protobuf-java/3.21.4/protobuf-java-3.21.4.jar
+- vendor/jar-dependencies/com/google/re2j/re2j/1.6/re2j-1.6.jar
+- vendor/jar-dependencies/commons-codec/commons-codec/1.15/commons-codec-1.15.jar
+- vendor/jar-dependencies/commons-logging/commons-logging/1.2/commons-logging-1.2.jar
+- vendor/jar-dependencies/io/grpc/grpc-alts/1.48.0/grpc-alts-1.48.0.jar
+- vendor/jar-dependencies/io/grpc/grpc-api/1.48.0/grpc-api-1.48.0.jar
+- vendor/jar-dependencies/io/grpc/grpc-auth/1.48.0/grpc-auth-1.48.0.jar
+- vendor/jar-dependencies/io/grpc/grpc-context/1.48.0/grpc-context-1.48.0.jar
+- vendor/jar-dependencies/io/grpc/grpc-core/1.48.0/grpc-core-1.48.0.jar
+- vendor/jar-dependencies/io/grpc/grpc-googleapis/1.48.0/grpc-googleapis-1.48.0.jar
+- vendor/jar-dependencies/io/grpc/grpc-grpclb/1.48.0/grpc-grpclb-1.48.0.jar
+- vendor/jar-dependencies/io/grpc/grpc-netty-shaded/1.48.0/grpc-netty-shaded-1.48.0.jar
+- vendor/jar-dependencies/io/grpc/grpc-protobuf-lite/1.48.0/grpc-protobuf-lite-1.48.0.jar
+- vendor/jar-dependencies/io/grpc/grpc-protobuf/1.48.0/grpc-protobuf-1.48.0.jar
+- vendor/jar-dependencies/io/grpc/grpc-services/1.48.0/grpc-services-1.48.0.jar
+- vendor/jar-dependencies/io/grpc/grpc-stub/1.48.0/grpc-stub-1.48.0.jar
+- vendor/jar-dependencies/io/grpc/grpc-xds/1.48.0/grpc-xds-1.48.0.jar
+- vendor/jar-dependencies/io/opencensus/opencensus-api/0.31.1/opencensus-api-0.31.1.jar
+- vendor/jar-dependencies/io/opencensus/opencensus-contrib-http-util/0.31.1/opencensus-contrib-http-util-0.31.1.jar
+- vendor/jar-dependencies/io/opencensus/opencensus-proto/0.2.0/opencensus-proto-0.2.0.jar
+- vendor/jar-dependencies/io/perfmark/perfmark-api/0.25.0/perfmark-api-0.25.0.jar
+- vendor/jar-dependencies/javax/annotation/javax.annotation-api/1.3.2/javax.annotation-api-1.3.2.jar
+- vendor/jar-dependencies/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar
+- vendor/jar-dependencies/org/apache/httpcomponents/httpcore/4.4.15/httpcore-4.4.15.jar
+- vendor/jar-dependencies/org/checkerframework/checker-qual/3.23.0/checker-qual-3.23.0.jar
+- vendor/jar-dependencies/org/codehaus/mojo/animal-sniffer-annotations/1.21/animal-sniffer-annotations-1.21.jar
+- vendor/jar-dependencies/org/conscrypt/conscrypt-openjdk-uber/2.5.2/conscrypt-openjdk-uber-2.5.2.jar
+- vendor/jar-dependencies/org/logstash/inputs/plugin/1.2.2/plugin-1.2.2.jar
+- vendor/jar-dependencies/org/threeten/threetenbp/1.6.0/threetenbp-1.6.0.jar
 homepage: https://cloud.google.com/pubsub/overview
 licenses:
 - Apache-2.0
@@ -120,6 +181,7 @@ post_install_message:
 rdoc_options: []
 require_paths:
 - lib
+- vendor/jar-dependencies
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -130,9 +192,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
+requirements:
+- jar 'com.google.cloud:google-cloud-pubsub', '0.28.0-beta'
+- jar 'com.google.api.grpc:proto-google-cloud-pubsub-v1', '0.1.24'
+- jar 'com.google.api:gax', '1.14.0'
+- jar 'com.google.guava:guava', '20.0'
+- jar 'com.google.api:api-common', '1.2.0'
+- jar 'com.google.auth:google-auth-library-oauth2-http', '0.9.0'
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Consume events from a Google Cloud PubSub service