logstash-input-google_pubsub 1.1.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +4 -0
  3. data/CONTRIBUTORS +23 -0
  4. data/README.md +1 -0
  5. data/docs/index.asciidoc +18 -0
  6. data/lib/logstash-input-google_pubsub_jars.rb +52 -0
  7. data/lib/logstash/inputs/google_pubsub.rb +105 -132
  8. data/logstash-input-google_pubsub.gemspec +9 -4
  9. data/vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-core/2.1.3/jackson-core-2.1.3.jar +0 -0
  10. data/vendor/jar-dependencies/com/google/api/api-common/1.2.0/api-common-1.2.0.jar +0 -0
  11. data/vendor/jar-dependencies/com/google/api/gax-grpc/1.14.0/gax-grpc-1.14.0.jar +0 -0
  12. data/vendor/jar-dependencies/com/google/api/gax/1.14.0/gax-1.14.0.jar +0 -0
  13. data/vendor/jar-dependencies/com/google/api/grpc/grpc-google-cloud-pubsub-v1/0.1.24/grpc-google-cloud-pubsub-v1-0.1.24.jar +0 -0
  14. data/vendor/jar-dependencies/com/google/api/grpc/proto-google-cloud-pubsub-v1/0.1.24/proto-google-cloud-pubsub-v1-0.1.24.jar +0 -0
  15. data/vendor/jar-dependencies/com/google/api/grpc/proto-google-common-protos/1.0.0/proto-google-common-protos-1.0.0.jar +0 -0
  16. data/vendor/jar-dependencies/com/google/api/grpc/proto-google-iam-v1/0.1.24/proto-google-iam-v1-0.1.24.jar +0 -0
  17. data/vendor/jar-dependencies/com/google/auth/google-auth-library-credentials/0.9.0/google-auth-library-credentials-0.9.0.jar +0 -0
  18. data/vendor/jar-dependencies/com/google/auth/google-auth-library-oauth2-http/0.9.0/google-auth-library-oauth2-http-0.9.0.jar +0 -0
  19. data/vendor/jar-dependencies/com/google/auto/value/auto-value/1.2/auto-value-1.2.jar +0 -0
  20. data/vendor/jar-dependencies/com/google/cloud/google-cloud-core-grpc/1.10.0/google-cloud-core-grpc-1.10.0.jar +0 -0
  21. data/vendor/jar-dependencies/com/google/cloud/google-cloud-core/1.10.0/google-cloud-core-1.10.0.jar +0 -0
  22. data/vendor/jar-dependencies/com/google/cloud/google-cloud-pubsub/0.28.0-beta/google-cloud-pubsub-0.28.0-beta.jar +0 -0
  23. data/vendor/jar-dependencies/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar +0 -0
  24. data/vendor/jar-dependencies/com/google/code/gson/gson/2.7/gson-2.7.jar +0 -0
  25. data/vendor/jar-dependencies/com/google/errorprone/error_prone_annotations/2.0.19/error_prone_annotations-2.0.19.jar +0 -0
  26. data/vendor/jar-dependencies/com/google/guava/guava/20.0/guava-20.0.jar +0 -0
  27. data/vendor/jar-dependencies/com/google/http-client/google-http-client-jackson2/1.23.0/google-http-client-jackson2-1.23.0.jar +0 -0
  28. data/vendor/jar-dependencies/com/google/http-client/google-http-client/1.23.0/google-http-client-1.23.0.jar +0 -0
  29. data/vendor/jar-dependencies/com/google/instrumentation/instrumentation-api/0.4.3/instrumentation-api-0.4.3.jar +0 -0
  30. data/vendor/jar-dependencies/com/google/protobuf/protobuf-java-util/3.4.0/protobuf-java-util-3.4.0.jar +0 -0
  31. data/vendor/jar-dependencies/com/google/protobuf/protobuf-java/3.5.1/protobuf-java-3.5.1.jar +0 -0
  32. data/vendor/jar-dependencies/commons-codec/commons-codec/1.3/commons-codec-1.3.jar +0 -0
  33. data/vendor/jar-dependencies/commons-logging/commons-logging/1.1.1/commons-logging-1.1.1.jar +0 -0
  34. data/vendor/jar-dependencies/io/grpc/grpc-auth/1.7.0/grpc-auth-1.7.0.jar +0 -0
  35. data/vendor/jar-dependencies/io/grpc/grpc-context/1.7.0/grpc-context-1.7.0.jar +0 -0
  36. data/vendor/jar-dependencies/io/grpc/grpc-core/1.7.0/grpc-core-1.7.0.jar +0 -0
  37. data/vendor/jar-dependencies/io/grpc/grpc-netty/1.7.0/grpc-netty-1.7.0.jar +0 -0
  38. data/vendor/jar-dependencies/io/grpc/grpc-protobuf-lite/1.7.0/grpc-protobuf-lite-1.7.0.jar +0 -0
  39. data/vendor/jar-dependencies/io/grpc/grpc-protobuf/1.7.0/grpc-protobuf-1.7.0.jar +0 -0
  40. data/vendor/jar-dependencies/io/grpc/grpc-stub/1.7.0/grpc-stub-1.7.0.jar +0 -0
  41. data/vendor/jar-dependencies/io/netty/netty-buffer/4.1.16.Final/netty-buffer-4.1.16.Final.jar +0 -0
  42. data/vendor/jar-dependencies/io/netty/netty-codec-http/4.1.16.Final/netty-codec-http-4.1.16.Final.jar +0 -0
  43. data/vendor/jar-dependencies/io/netty/netty-codec-http2/4.1.16.Final/netty-codec-http2-4.1.16.Final.jar +0 -0
  44. data/vendor/jar-dependencies/io/netty/netty-codec-socks/4.1.16.Final/netty-codec-socks-4.1.16.Final.jar +0 -0
  45. data/vendor/jar-dependencies/io/netty/netty-codec/4.1.16.Final/netty-codec-4.1.16.Final.jar +0 -0
  46. data/vendor/jar-dependencies/io/netty/netty-common/4.1.16.Final/netty-common-4.1.16.Final.jar +0 -0
  47. data/vendor/jar-dependencies/io/netty/netty-handler-proxy/4.1.16.Final/netty-handler-proxy-4.1.16.Final.jar +0 -0
  48. data/vendor/jar-dependencies/io/netty/netty-handler/4.1.16.Final/netty-handler-4.1.16.Final.jar +0 -0
  49. data/vendor/jar-dependencies/io/netty/netty-resolver/4.1.16.Final/netty-resolver-4.1.16.Final.jar +0 -0
  50. data/vendor/jar-dependencies/io/netty/netty-tcnative-boringssl-static/2.0.6.Final/netty-tcnative-boringssl-static-2.0.6.Final.jar +0 -0
  51. data/vendor/jar-dependencies/io/netty/netty-transport/4.1.16.Final/netty-transport-4.1.16.Final.jar +0 -0
  52. data/vendor/jar-dependencies/io/opencensus/opencensus-api/0.6.0/opencensus-api-0.6.0.jar +0 -0
  53. data/vendor/jar-dependencies/joda-time/joda-time/2.9.2/joda-time-2.9.2.jar +0 -0
  54. data/vendor/jar-dependencies/org/apache/httpcomponents/httpclient/4.0.1/httpclient-4.0.1.jar +0 -0
  55. data/vendor/jar-dependencies/org/apache/httpcomponents/httpcore/4.0.1/httpcore-4.0.1.jar +0 -0
  56. data/vendor/jar-dependencies/org/json/json/20160810/json-20160810.jar +0 -0
  57. data/vendor/jar-dependencies/org/threeten/threetenbp/1.3.3/threetenbp-1.3.3.jar +0 -0
  58. metadata +72 -20
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 3f39efa6b21fafa252ad3e27a1a1a301681145c88498ac242e7b85a77a64382f
4
- data.tar.gz: b92fe3cebc49c21702868a61a06a90c88b0f0372b5a7f57272add4a911359c1c
3
+ metadata.gz: f5bc9362c985c9940b9b4ae677adf15afe2fa6ea86bd85a83beafcd0a5092d42
4
+ data.tar.gz: 3f8279cf6a25d045d2776a2325fc90fd3edb81dfae312dc04efa82880e013e90
5
5
  SHA512:
6
- metadata.gz: a844cf0059e14822f1323a146b4f03b0745ab0f3c5a7c961626f35523059a632ef0169c96cf212612520b1a6ad5c6e428a08c30a82f8a8ffaab15d9c35d6b86b
7
- data.tar.gz: 1ca3f6026de8645af0ea71ab648bb52fc76ac4513373d21ad6bd6e39352576b403c56e738905f0f1372182f40e08485fed375b21c7dedd117df866010783d02b
6
+ metadata.gz: 4822ec58447e1c8e2d33f8ffb5f9e5d2766aeadf87d411107b588f304249f6069a419cc15703f5f4838f8c746133b2ea977c344d2ae3e3d9aa206a70b32e4de7
7
+ data.tar.gz: b3e81b01e38863bbefe468c4cff5234745e049a5468a1f9b4d5b5e1eac1ee6c024688aa24347ecf6aedd9655051bf6091f78b064659baed4afde176f8943f3a5
@@ -1,3 +1,7 @@
1
+ ## 1.2.0
2
+ - Change to Java client
3
+ - Add `create_subscription` setting. Fixes [#9](https://github.com/logstash-plugins/logstash-input-google_pubsub/issues/9)
4
+
1
5
  ## 1.1.0
2
6
  - Add additional attributes in the `[@metadata][pubsub_message]` field. Fixes [#7](https://github.com/logstash-plugins/logstash-input-google_pubsub/issues/7)
3
7
 
@@ -0,0 +1,23 @@
1
+ The following is a list of people who have contributed ideas, code, bug
2
+ reports, or in general have helped Logstash along its way.
3
+
4
+ Contributors:
5
+
6
+ * DeDe Morton (dedemorton)
7
+ * Eric Johnson (erjohnso)
8
+ * Google LLC
9
+ * Jake Landis (jakelandis)
10
+ * Jakub Sacha (jakubsacha)
11
+ * James Bloomer (jamesbloomer)
12
+ * João Duarte (jsvd)
13
+ * Jordan Sissel (jordansissel)
14
+ * Joseph Lewis III (josephlewis42)
15
+ * Pier-Hugues Pellerin (ph)
16
+ * Scott Brenner (ScottBrenner)
17
+ * Suyog Rao (suyograo)
18
+ * Sven Widén (SvenW)
19
+
20
+ Note: If you've sent us patches, bug reports, or otherwise contributed to
21
+ Logstash, and you aren't on the list above and want to be, please let us know
22
+ and we'll make sure you're here. Contributions from folks like you are what make
23
+ open source awesome.
data/README.md CHANGED
@@ -130,6 +130,7 @@ Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/log
130
130
  - Install dependencies
131
131
  ```sh
132
132
  bundle install
133
+ rake install_jars
133
134
  ```
134
135
 
135
136
  #### Test
@@ -133,6 +133,11 @@ input {
133
133
  # outside of GCE, you will need to specify the service account's
134
134
  # JSON key file below.
135
135
  #json_key_file => "/home/erjohnso/pkey.json"
136
+
137
+ # Should the plugin attempt to create the subscription on startup?
138
+ # This is not recommended for security reasons but may be useful in
139
+ # some cases.
140
+ #create_subscription => false
136
141
  }
137
142
  }
138
143
  output { stdout { codec => rubydebug } }
@@ -185,6 +190,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
185
190
  | <<plugins-{type}s-{plugin}-subscription>> |<<string,string>>|Yes
186
191
  | <<plugins-{type}s-{plugin}-topic>> |<<string,string>>|Yes
187
192
  | <<plugins-{type}s-{plugin}-include_metadata>> |<<boolean,boolean>>|No
193
+ | <<plugins-{type}s-{plugin}-create_subscription>> |<<boolean,boolean>>|No
188
194
  |=======================================================================
189
195
 
190
196
  Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
@@ -250,6 +256,18 @@ The subscription will be created automatically by the plugin.
250
256
 
251
257
  If set true, will include the full message data in the `[@metadata][pubsub_message]` field.
252
258
 
259
+ [id="plugins-{type}s-{plugin}-create_subscription"]
260
+ ===== `create_subscription`
261
+
262
+ added[1.2.0]
263
+
264
+ * Value type is <<boolean,boolean>>
265
+ * Default value is `false`.
266
+
267
+ If true, the plugin will try to create the subscription before publishing.
268
+ Note: this requires additional permissions to be granted to the client and is _not_
269
+ recommended for most use-cases.
270
+
253
271
  [id="plugins-{type}s-{plugin}-common-options"]
254
272
  include::{include_path}/{type}.asciidoc[]
255
273
 
@@ -0,0 +1,52 @@
1
+ # AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
2
+
3
+ require 'jar_dependencies'
4
+ require_jar('com.google.api.grpc', 'proto-google-iam-v1', '0.1.24')
5
+ require_jar('io.netty', 'netty-buffer', '4.1.16.Final')
6
+ require_jar('org.threeten', 'threetenbp', '1.3.3')
7
+ require_jar('com.google.errorprone', 'error_prone_annotations', '2.0.19')
8
+ require_jar('com.google.api.grpc', 'grpc-google-cloud-pubsub-v1', '0.1.24')
9
+ require_jar('io.netty', 'netty-codec-http', '4.1.16.Final')
10
+ require_jar('io.grpc', 'grpc-protobuf-lite', '1.7.0')
11
+ require_jar('io.netty', 'netty-common', '4.1.16.Final')
12
+ require_jar('com.google.auto.value', 'auto-value', '1.2')
13
+ require_jar('com.google.protobuf', 'protobuf-java-util', '3.4.0')
14
+ require_jar('com.google.http-client', 'google-http-client-jackson2', '1.23.0')
15
+ require_jar('com.fasterxml.jackson.core', 'jackson-core', '2.1.3')
16
+ require_jar('com.google.code.findbugs', 'jsr305', '3.0.2')
17
+ require_jar('joda-time', 'joda-time', '2.9.2')
18
+ require_jar('io.netty', 'netty-handler', '4.1.16.Final')
19
+ require_jar('com.google.http-client', 'google-http-client', '1.23.0')
20
+ require_jar('commons-logging', 'commons-logging', '1.1.1')
21
+ require_jar('org.json', 'json', '20160810')
22
+ require_jar('com.google.cloud', 'google-cloud-core', '1.10.0')
23
+ require_jar('io.netty', 'netty-codec-http2', '4.1.16.Final')
24
+ require_jar('com.google.guava', 'guava', '20.0')
25
+ require_jar('com.google.cloud', 'google-cloud-core-grpc', '1.10.0')
26
+ require_jar('com.google.protobuf', 'protobuf-java', '3.5.1')
27
+ require_jar('com.google.api', 'gax', '1.14.0')
28
+ require_jar('org.apache.httpcomponents', 'httpclient', '4.0.1')
29
+ require_jar('com.google.api', 'gax-grpc', '1.14.0')
30
+ require_jar('com.google.auth', 'google-auth-library-credentials', '0.9.0')
31
+ require_jar('com.google.api', 'api-common', '1.2.0')
32
+ require_jar('com.google.api.grpc', 'proto-google-common-protos', '1.0.0')
33
+ require_jar('io.grpc', 'grpc-auth', '1.7.0')
34
+ require_jar('io.grpc', 'grpc-core', '1.7.0')
35
+ require_jar('io.netty', 'netty-codec-socks', '4.1.16.Final')
36
+ require_jar('io.grpc', 'grpc-protobuf', '1.7.0')
37
+ require_jar('io.grpc', 'grpc-context', '1.7.0')
38
+ require_jar('io.netty', 'netty-transport', '4.1.16.Final')
39
+ require_jar('commons-codec', 'commons-codec', '1.3')
40
+ require_jar('com.google.instrumentation', 'instrumentation-api', '0.4.3')
41
+ require_jar('io.netty', 'netty-handler-proxy', '4.1.16.Final')
42
+ require_jar('io.netty', 'netty-resolver', '4.1.16.Final')
43
+ require_jar('io.netty', 'netty-tcnative-boringssl-static', '2.0.6.Final')
44
+ require_jar('com.google.code.gson', 'gson', '2.7')
45
+ require_jar('io.grpc', 'grpc-netty', '1.7.0')
46
+ require_jar('io.opencensus', 'opencensus-api', '0.6.0')
47
+ require_jar('org.apache.httpcomponents', 'httpcore', '4.0.1')
48
+ require_jar('io.netty', 'netty-codec', '4.1.16.Final')
49
+ require_jar('io.grpc', 'grpc-stub', '1.7.0')
50
+ require_jar('com.google.cloud', 'google-cloud-pubsub', '0.28.0-beta')
51
+ require_jar('com.google.api.grpc', 'proto-google-cloud-pubsub-v1', '0.1.24')
52
+ require_jar('com.google.auth', 'google-auth-library-oauth2-http', '0.9.0')
@@ -19,8 +19,8 @@
19
19
  require "logstash/inputs/base"
20
20
  require "logstash/namespace"
21
21
 
22
- # Google deps
23
- require "google/api_client"
22
+ require 'java'
23
+ require 'logstash-input-google_pubsub_jars.rb'
24
24
 
25
25
  # This is a https://github.com/elastic/logstash[Logstash] input plugin for
26
26
  # https://cloud.google.com/pubsub/[Google Pub/Sub]. The plugin can subscribe
@@ -123,38 +123,70 @@ require "google/api_client"
123
123
  # ----------------------------------
124
124
  #
125
125
  # ==== Metadata and Attributes
126
- #
127
- # The original Pub/Sub message is preserved in the special Logstash
126
+ #
127
+ # The original Pub/Sub message is preserved in the special Logstash
128
128
  # `[@metadata][pubsub_message]` field so you can fetch:
129
- #
129
+ #
130
130
  # * Message attributes
131
131
  # * The origiginal base64 data
132
132
  # * Pub/Sub message ID for de-duplication
133
133
  # * Publish time
134
- #
134
+ #
135
135
  # You MUST extract any fields you want in a filter prior to the data being sent
136
136
  # to an output because Logstash deletes `@metadata` fields otherwise.
137
- #
137
+ #
138
138
  # See the PubsubMessage
139
139
  # https://cloud.google.com/pubsub/docs/reference/rest/v1/PubsubMessage[documentation]
140
140
  # for a full description of the fields.
141
- #
141
+ #
142
142
  # Example to get the message ID:
143
- #
143
+ #
144
144
  # [source,ruby]
145
- # ----------------------------------
145
+ # ----------------------------------
146
146
  # input {google_pubsub {...}}
147
- #
147
+ #
148
148
  # filter {
149
149
  # mutate {
150
150
  # add_field => { "messageId" => "%{[@metadata][pubsub_message][messageId]}" }
151
151
  # }
152
152
  # }
153
- #
153
+ #
154
154
  # output {...}
155
155
  # ----------------------------------
156
156
  #
157
+
157
158
  class LogStash::Inputs::GooglePubSub < LogStash::Inputs::Base
159
+ class MessageReceiver
160
+ include com.google.cloud.pubsub.v1.MessageReceiver
161
+
162
+ def initialize(&blk)
163
+ @block = blk
164
+ end
165
+
166
+ def receiveMessage(message, consumer)
167
+ @block.call(message)
168
+ consumer.ack()
169
+ end
170
+ end
171
+
172
+ java_import 'com.google.api.core.ApiService$Listener'
173
+ class SubscriberListener < Listener
174
+ def initialize(&blk)
175
+ @block = blk
176
+ end
177
+
178
+ def failed(from, failure)
179
+ @block.call(from, failure)
180
+ end
181
+ end
182
+
183
+ include_package 'com.google.api.gax.batching'
184
+ include_package 'com.google.api.gax.core'
185
+ include_package 'com.google.auth.oauth2'
186
+ include_package 'com.google.common.util.concurrent'
187
+ include_package 'com.google.cloud.pubsub.v1'
188
+ include_package 'com.google.pubsub.v1'
189
+ include_package 'com.google.protobuf.util'
158
190
  config_name "google_pubsub"
159
191
 
160
192
  # Google Cloud Project ID (name, not number)
@@ -176,140 +208,81 @@ class LogStash::Inputs::GooglePubSub < LogStash::Inputs::Base
176
208
  # If set true, will include the full message data in the `[@metadata][pubsub_message]` field.
177
209
  config :include_metadata, :validate => :boolean, :required => false, :default => false
178
210
 
211
+ # If true, the plugin will try to create the subscription before publishing.
212
+ # Note: this requires additional permissions to be granted to the client and is _not_
213
+ # recommended for most use-cases.
214
+ config :create_subscription, :validate => :boolean, :required => false, :default => false
215
+
179
216
  # If undefined, Logstash will complain, even if codec is unused.
180
217
  default :codec, "plain"
181
218
 
182
- private
183
- def request(options)
184
- begin
185
- @logger.debug("Sending an API request")
186
- result = @client.execute(options)
187
- rescue ArgumentError => e
188
- @logger.debug("Authorizing...")
189
- @client.authorization.fetch_access_token!
190
- @logger.debug("...authorized")
191
- request(options)
192
- rescue Faraday::TimeoutError => e
193
- @logger.debug("Request timeout, re-trying request")
194
- request(options)
195
- end
196
- end # def request
197
-
198
219
  public
199
220
  def register
200
221
  @logger.debug("Registering Google PubSub Input: project_id=#{@project_id}, topic=#{@topic}, subscription=#{@subscription}")
201
- @topic = "projects/#{@project_id}/topics/#{@topic}"
202
- @subscription = "projects/#{@project_id}/subscriptions/#{@subscription}"
203
- @subscription_exists = false
204
-
205
- # TODO(erjohnso): read UA data from the gemspec
206
- @client = Google::APIClient.new(
207
- :application_name => 'logstash-input-google_pubsub',
208
- :application_version => '0.9.0'
209
- )
222
+ @subscription_id = "projects/#{@project_id}/subscriptions/#{@subscription}"
210
223
 
211
- # Initialize the pubsub API client
212
- @pubsub = @client.discovered_api('pubsub', 'v1')
213
-
214
- # Handle various kinds of auth (JSON or Application Default Creds)
215
- # NOTE: Cannot use 'googleauth' gem since there are dependency conflicts
216
- # - googleauth ~> 0.5 requires mime-data-types that requires ruby2
217
- # - googleauth ~> 0.3 requires multi_json 1.11.0 that conflicts
218
- # with logstash-2.3.2's multi_json 1.11.3
219
224
  if @json_key_file
220
- @logger.debug("Authorizing with JSON key file: #{@json_key_file}")
221
- file_path = File.expand_path(@json_key_file)
222
- key_json = File.open(file_path, "r", &:read)
223
- key_json = JSON.parse(key_json)
224
- unless key_json.key?("client_email") || key_json.key?("private_key")
225
- raise Google::APIClient::ClientError, "Invalid JSON credentials data."
226
- end
227
- signing_key = ::Google::APIClient::KeyUtils.load_from_pem(key_json["private_key"], "notasecret")
228
- @client.authorization = Signet::OAuth2::Client.new(
229
- :audience => "https://accounts.google.com/o/oauth2/token",
230
- :auth_provider_x509_cert_url => "https://www.googleapis.com/oauth2/v1/certs",
231
- :client_x509_cert_url => "https://www.googleapis.com/robot/v1/metadata/x509/#{key_json['client_email']}",
232
- :issuer => "#{key_json['client_email']}",
233
- :scope => %w(https://www.googleapis.com/auth/cloud-platform),
234
- :signing_key => signing_key,
235
- :token_credential_uri => "https://accounts.google.com/o/oauth2/token"
225
+ @credentialsProvider = FixedCredentialsProvider.create(
226
+ ServiceAccountCredentials.fromStream(java.io.FileInputStream.new(@json_key_file))
236
227
  )
237
- @logger.info("Client authorization with JSON key ready")
238
- else
239
- # Assume we're running in GCE and can use metadata tokens, if the host
240
- # GCE instance was not created with the PubSub scope, then the plugin
241
- # will not be authorized to read from pubsub.
242
- @logger.info("Authorizing with application default credentials")
243
- @client.authorization = :google_app_default
244
- end # if @json_key_file...
245
- end # def register
228
+ end
229
+ @topic_name = TopicName.create(@project_id, @topic)
230
+ @subscription_name = SubscriptionName.create(@project_id, @subscription)
231
+ end
232
+
233
+ def stop
234
+ @subscriber.stopAsync().awaitTerminated() if @subscriber != nil
235
+ end
246
236
 
247
237
  def run(queue)
248
238
  # Attempt to create the subscription
249
- if !@subscription_exists
250
- @logger.debug("Creating subscription #{subscription}")
251
- result = request(
252
- :api_method => @pubsub.projects.subscriptions.create,
253
- :parameters => {'name' => @subscription},
254
- :body_object => {
255
- :topic => @topic,
256
- :ackDeadlineSeconds => 15
257
- }
258
- )
259
- if result.error? and result.status != 409
260
- raise Google::APIClient::ClientError, "Error #{result.status}: #{result.error_message}"
239
+ if @create_subscription
240
+ @logger.debug("Creating subscription #{@subscription_id}")
241
+ subscriptionAdminClient = SubscriptionAdminClient.create
242
+ begin
243
+ subscriptionAdminClient.createSubscription(@subscription_name, @topic_name, PushConfig.getDefaultInstance(), 0)
244
+ rescue
245
+ @logger.info("Subscription already exists")
261
246
  end
262
- @subscription_exists = true
263
- end # if !@subscription
264
-
265
- @logger.debug("Pulling messages from sub '#{subscription}'")
266
- while !stop?
267
- # Pull and queue messages
268
- messages = []
269
- result = request(
270
- :api_method => @pubsub.projects.subscriptions.pull,
271
- :parameters => {'subscription' => @subscription},
272
- :body_object => {
273
- :returnImmediately => false,
274
- :maxMessages => @max_messages
275
- }
276
- )
247
+ end
277
248
 
278
- if !result.error?
279
- messages = JSON.parse(result.body)
280
- if messages.key?("receivedMessages")
281
- messages = messages["receivedMessages"]
282
- end
283
- else
284
- @logger.error("Error pulling messages:'#{result.error_message}'")
249
+ @logger.debug("Pulling messages from sub '#{@subscription_id}'")
250
+ handler = MessageReceiver.new do |message|
251
+ # handle incoming message, then ack/nack the received message
252
+ data = message.getData().toStringUtf8()
253
+ @codec.decode(data) do |event|
254
+ event.set("host", event.get("host") || @host)
255
+ event.set("[@metadata][pubsub_message]", extract_metadata(message)) if @include_metadata
256
+ decorate(event)
257
+ queue << event
285
258
  end
259
+ end
260
+ listener = SubscriberListener.new do |from, failure|
261
+ @logger.error("#{failure}")
262
+ raise failure
263
+ end
264
+ flowControlSettings = FlowControlSettings.newBuilder().setMaxOutstandingElementCount(@max_messages).build()
265
+ executorProvider = InstantiatingExecutorProvider.newBuilder().setExecutorThreadCount(1).build()
266
+ subscriberBuilder = Subscriber.newBuilder(@subscription_name, handler)
267
+ .setFlowControlSettings(flowControlSettings)
268
+ .setExecutorProvider(executorProvider)
269
+ .setParallelPullCount(1)
286
270
 
287
- if messages
288
- messages.each do |msg|
289
- if msg.key?("message") and msg["message"].key?("data")
290
- decoded_msg = Base64.decode64(msg["message"]["data"])
291
- @codec.decode(decoded_msg) do |event|
292
- event.set("[@metadata][pubsub_message]", msg["message"]) if @include_metadata
293
- decorate(event)
294
- queue << event
295
- end
296
- end
297
- end
298
-
299
- ack_ids = messages.map{ |msg| msg["ackId"] }
300
- next if ack_ids.empty?
271
+ if @credentialsProvider
272
+ subscriberBuilder.setCredentialsProvider(@credentialsProvider)
273
+ end
274
+ @subscriber = subscriberBuilder.build()
275
+ @subscriber.addListener(listener, MoreExecutors.directExecutor())
276
+ @subscriber.startAsync()
277
+ @subscriber.awaitTerminated()
278
+ end
301
279
 
302
- result = request(
303
- :api_method => @pubsub.projects.subscriptions.acknowledge,
304
- :parameters => {'subscription' => @subscription},
305
- :body_object => {
306
- :ackIds => ack_ids
307
- }
308
- )
309
- if result.error?
310
- @logger.error("Error #{result.status}: #{result.error_message}")
311
- end
312
- end # if messages
313
- end # loop
314
- end # def run
315
- end # class LogStash::Inputs::GooglePubSub
280
+ def extract_metadata(java_message)
281
+ {
282
+ data: java_message.getData().toStringUtf8(),
283
+ attributes: java_message.getAttributesMap(),
284
+ messageId: java_message.getMessageId(),
285
+ publishTime: Timestamps.toString(java_message.getPublishTime())
286
+ }
287
+ end
288
+ end
@@ -1,13 +1,13 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-input-google_pubsub'
3
- s.version = '1.1.0'
3
+ s.version = '1.2.0'
4
4
  s.licenses = ['Apache-2.0']
5
5
  s.summary = "Consume events from a Google Cloud PubSub service"
6
6
  s.description = "This gem is a Logstash input plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program."
7
7
  s.authors = ["Eric Johnson"]
8
8
  s.email = 'erjohnso@google.com'
9
9
  s.homepage = "https://cloud.google.com/pubsub/overview"
10
- s.require_paths = ["lib"]
10
+ s.require_paths = ["lib", "vendor/jar-dependencies"]
11
11
 
12
12
  # Files
13
13
  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
@@ -22,8 +22,13 @@ Gem::Specification.new do |s|
22
22
  s.add_runtime_dependency 'logstash-codec-plain'
23
23
  s.add_runtime_dependency 'stud', '>= 0.0.22'
24
24
  # Google dependencies
25
- # google-api-client >= 0.9 requires ruby2 which is not currently compatible
26
25
  # with JRuby
27
- s.add_runtime_dependency 'google-api-client', '~> 0.8.6', '< 0.9'
26
+ s.requirements << "jar 'com.google.cloud:google-cloud-pubsub', '0.28.0-beta'"
27
+ s.requirements << "jar 'com.google.api.grpc:proto-google-cloud-pubsub-v1', '0.1.24'"
28
+ s.requirements << "jar 'com.google.api:gax', '1.14.0'"
29
+ s.requirements << "jar 'com.google.guava:guava', '20.0'"
30
+ s.requirements << "jar 'com.google.api:api-common', '1.2.0'"
31
+ s.requirements << "jar 'com.google.auth:google-auth-library-oauth2-http', '0.9.0'"
28
32
  s.add_development_dependency 'logstash-devutils'
33
+ s.add_development_dependency 'jar-dependencies', '~> 0.3.2'
29
34
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-input-google_pubsub
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.0
4
+ version: 1.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Eric Johnson
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-04-26 00:00:00.000000000 Z
11
+ date: 2018-05-07 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -61,37 +61,31 @@ dependencies:
61
61
  - !ruby/object:Gem::Dependency
62
62
  requirement: !ruby/object:Gem::Requirement
63
63
  requirements:
64
- - - "~>"
65
- - !ruby/object:Gem::Version
66
- version: 0.8.6
67
- - - "<"
64
+ - - ">="
68
65
  - !ruby/object:Gem::Version
69
- version: '0.9'
70
- name: google-api-client
66
+ version: '0'
67
+ name: logstash-devutils
71
68
  prerelease: false
72
- type: :runtime
69
+ type: :development
73
70
  version_requirements: !ruby/object:Gem::Requirement
74
71
  requirements:
75
- - - "~>"
76
- - !ruby/object:Gem::Version
77
- version: 0.8.6
78
- - - "<"
72
+ - - ">="
79
73
  - !ruby/object:Gem::Version
80
- version: '0.9'
74
+ version: '0'
81
75
  - !ruby/object:Gem::Dependency
82
76
  requirement: !ruby/object:Gem::Requirement
83
77
  requirements:
84
- - - ">="
78
+ - - "~>"
85
79
  - !ruby/object:Gem::Version
86
- version: '0'
87
- name: logstash-devutils
80
+ version: 0.3.2
81
+ name: jar-dependencies
88
82
  prerelease: false
89
83
  type: :development
90
84
  version_requirements: !ruby/object:Gem::Requirement
91
85
  requirements:
92
- - - ">="
86
+ - - "~>"
93
87
  - !ruby/object:Gem::Version
94
- version: '0'
88
+ version: 0.3.2
95
89
  description: This gem is a Logstash input plugin required to be installed on top of
96
90
  the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
97
91
  gem is not a stand-alone program.
@@ -102,14 +96,65 @@ extra_rdoc_files: []
102
96
  files:
103
97
  - CHANGELOG.md
104
98
  - CONTRIBUTING.md
99
+ - CONTRIBUTORS
105
100
  - Gemfile
106
101
  - LICENSE
107
102
  - NOTICE.TXT
108
103
  - README.md
109
104
  - docs/index.asciidoc
105
+ - lib/logstash-input-google_pubsub_jars.rb
110
106
  - lib/logstash/inputs/google_pubsub.rb
111
107
  - logstash-input-google_pubsub.gemspec
112
108
  - spec/inputs/google_pubsub_spec.rb
109
+ - vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-core/2.1.3/jackson-core-2.1.3.jar
110
+ - vendor/jar-dependencies/com/google/api/api-common/1.2.0/api-common-1.2.0.jar
111
+ - vendor/jar-dependencies/com/google/api/gax-grpc/1.14.0/gax-grpc-1.14.0.jar
112
+ - vendor/jar-dependencies/com/google/api/gax/1.14.0/gax-1.14.0.jar
113
+ - vendor/jar-dependencies/com/google/api/grpc/grpc-google-cloud-pubsub-v1/0.1.24/grpc-google-cloud-pubsub-v1-0.1.24.jar
114
+ - vendor/jar-dependencies/com/google/api/grpc/proto-google-cloud-pubsub-v1/0.1.24/proto-google-cloud-pubsub-v1-0.1.24.jar
115
+ - vendor/jar-dependencies/com/google/api/grpc/proto-google-common-protos/1.0.0/proto-google-common-protos-1.0.0.jar
116
+ - vendor/jar-dependencies/com/google/api/grpc/proto-google-iam-v1/0.1.24/proto-google-iam-v1-0.1.24.jar
117
+ - vendor/jar-dependencies/com/google/auth/google-auth-library-credentials/0.9.0/google-auth-library-credentials-0.9.0.jar
118
+ - vendor/jar-dependencies/com/google/auth/google-auth-library-oauth2-http/0.9.0/google-auth-library-oauth2-http-0.9.0.jar
119
+ - vendor/jar-dependencies/com/google/auto/value/auto-value/1.2/auto-value-1.2.jar
120
+ - vendor/jar-dependencies/com/google/cloud/google-cloud-core-grpc/1.10.0/google-cloud-core-grpc-1.10.0.jar
121
+ - vendor/jar-dependencies/com/google/cloud/google-cloud-core/1.10.0/google-cloud-core-1.10.0.jar
122
+ - vendor/jar-dependencies/com/google/cloud/google-cloud-pubsub/0.28.0-beta/google-cloud-pubsub-0.28.0-beta.jar
123
+ - vendor/jar-dependencies/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar
124
+ - vendor/jar-dependencies/com/google/code/gson/gson/2.7/gson-2.7.jar
125
+ - vendor/jar-dependencies/com/google/errorprone/error_prone_annotations/2.0.19/error_prone_annotations-2.0.19.jar
126
+ - vendor/jar-dependencies/com/google/guava/guava/20.0/guava-20.0.jar
127
+ - vendor/jar-dependencies/com/google/http-client/google-http-client-jackson2/1.23.0/google-http-client-jackson2-1.23.0.jar
128
+ - vendor/jar-dependencies/com/google/http-client/google-http-client/1.23.0/google-http-client-1.23.0.jar
129
+ - vendor/jar-dependencies/com/google/instrumentation/instrumentation-api/0.4.3/instrumentation-api-0.4.3.jar
130
+ - vendor/jar-dependencies/com/google/protobuf/protobuf-java-util/3.4.0/protobuf-java-util-3.4.0.jar
131
+ - vendor/jar-dependencies/com/google/protobuf/protobuf-java/3.5.1/protobuf-java-3.5.1.jar
132
+ - vendor/jar-dependencies/commons-codec/commons-codec/1.3/commons-codec-1.3.jar
133
+ - vendor/jar-dependencies/commons-logging/commons-logging/1.1.1/commons-logging-1.1.1.jar
134
+ - vendor/jar-dependencies/io/grpc/grpc-auth/1.7.0/grpc-auth-1.7.0.jar
135
+ - vendor/jar-dependencies/io/grpc/grpc-context/1.7.0/grpc-context-1.7.0.jar
136
+ - vendor/jar-dependencies/io/grpc/grpc-core/1.7.0/grpc-core-1.7.0.jar
137
+ - vendor/jar-dependencies/io/grpc/grpc-netty/1.7.0/grpc-netty-1.7.0.jar
138
+ - vendor/jar-dependencies/io/grpc/grpc-protobuf-lite/1.7.0/grpc-protobuf-lite-1.7.0.jar
139
+ - vendor/jar-dependencies/io/grpc/grpc-protobuf/1.7.0/grpc-protobuf-1.7.0.jar
140
+ - vendor/jar-dependencies/io/grpc/grpc-stub/1.7.0/grpc-stub-1.7.0.jar
141
+ - vendor/jar-dependencies/io/netty/netty-buffer/4.1.16.Final/netty-buffer-4.1.16.Final.jar
142
+ - vendor/jar-dependencies/io/netty/netty-codec-http/4.1.16.Final/netty-codec-http-4.1.16.Final.jar
143
+ - vendor/jar-dependencies/io/netty/netty-codec-http2/4.1.16.Final/netty-codec-http2-4.1.16.Final.jar
144
+ - vendor/jar-dependencies/io/netty/netty-codec-socks/4.1.16.Final/netty-codec-socks-4.1.16.Final.jar
145
+ - vendor/jar-dependencies/io/netty/netty-codec/4.1.16.Final/netty-codec-4.1.16.Final.jar
146
+ - vendor/jar-dependencies/io/netty/netty-common/4.1.16.Final/netty-common-4.1.16.Final.jar
147
+ - vendor/jar-dependencies/io/netty/netty-handler-proxy/4.1.16.Final/netty-handler-proxy-4.1.16.Final.jar
148
+ - vendor/jar-dependencies/io/netty/netty-handler/4.1.16.Final/netty-handler-4.1.16.Final.jar
149
+ - vendor/jar-dependencies/io/netty/netty-resolver/4.1.16.Final/netty-resolver-4.1.16.Final.jar
150
+ - vendor/jar-dependencies/io/netty/netty-tcnative-boringssl-static/2.0.6.Final/netty-tcnative-boringssl-static-2.0.6.Final.jar
151
+ - vendor/jar-dependencies/io/netty/netty-transport/4.1.16.Final/netty-transport-4.1.16.Final.jar
152
+ - vendor/jar-dependencies/io/opencensus/opencensus-api/0.6.0/opencensus-api-0.6.0.jar
153
+ - vendor/jar-dependencies/joda-time/joda-time/2.9.2/joda-time-2.9.2.jar
154
+ - vendor/jar-dependencies/org/apache/httpcomponents/httpclient/4.0.1/httpclient-4.0.1.jar
155
+ - vendor/jar-dependencies/org/apache/httpcomponents/httpcore/4.0.1/httpcore-4.0.1.jar
156
+ - vendor/jar-dependencies/org/json/json/20160810/json-20160810.jar
157
+ - vendor/jar-dependencies/org/threeten/threetenbp/1.3.3/threetenbp-1.3.3.jar
113
158
  homepage: https://cloud.google.com/pubsub/overview
114
159
  licenses:
115
160
  - Apache-2.0
@@ -120,6 +165,7 @@ post_install_message:
120
165
  rdoc_options: []
121
166
  require_paths:
122
167
  - lib
168
+ - vendor/jar-dependencies
123
169
  required_ruby_version: !ruby/object:Gem::Requirement
124
170
  requirements:
125
171
  - - ">="
@@ -130,7 +176,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
130
176
  - - ">="
131
177
  - !ruby/object:Gem::Version
132
178
  version: '0'
133
- requirements: []
179
+ requirements:
180
+ - jar 'com.google.cloud:google-cloud-pubsub', '0.28.0-beta'
181
+ - jar 'com.google.api.grpc:proto-google-cloud-pubsub-v1', '0.1.24'
182
+ - jar 'com.google.api:gax', '1.14.0'
183
+ - jar 'com.google.guava:guava', '20.0'
184
+ - jar 'com.google.api:api-common', '1.2.0'
185
+ - jar 'com.google.auth:google-auth-library-oauth2-http', '0.9.0'
134
186
  rubyforge_project:
135
187
  rubygems_version: 2.6.13
136
188
  signing_key: