logstash-input-google_pubsub 1.1.0 → 1.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (58) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +4 -0
  3. data/CONTRIBUTORS +23 -0
  4. data/README.md +1 -0
  5. data/docs/index.asciidoc +18 -0
  6. data/lib/logstash-input-google_pubsub_jars.rb +52 -0
  7. data/lib/logstash/inputs/google_pubsub.rb +105 -132
  8. data/logstash-input-google_pubsub.gemspec +9 -4
  9. data/vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-core/2.1.3/jackson-core-2.1.3.jar +0 -0
  10. data/vendor/jar-dependencies/com/google/api/api-common/1.2.0/api-common-1.2.0.jar +0 -0
  11. data/vendor/jar-dependencies/com/google/api/gax-grpc/1.14.0/gax-grpc-1.14.0.jar +0 -0
  12. data/vendor/jar-dependencies/com/google/api/gax/1.14.0/gax-1.14.0.jar +0 -0
  13. data/vendor/jar-dependencies/com/google/api/grpc/grpc-google-cloud-pubsub-v1/0.1.24/grpc-google-cloud-pubsub-v1-0.1.24.jar +0 -0
  14. data/vendor/jar-dependencies/com/google/api/grpc/proto-google-cloud-pubsub-v1/0.1.24/proto-google-cloud-pubsub-v1-0.1.24.jar +0 -0
  15. data/vendor/jar-dependencies/com/google/api/grpc/proto-google-common-protos/1.0.0/proto-google-common-protos-1.0.0.jar +0 -0
  16. data/vendor/jar-dependencies/com/google/api/grpc/proto-google-iam-v1/0.1.24/proto-google-iam-v1-0.1.24.jar +0 -0
  17. data/vendor/jar-dependencies/com/google/auth/google-auth-library-credentials/0.9.0/google-auth-library-credentials-0.9.0.jar +0 -0
  18. data/vendor/jar-dependencies/com/google/auth/google-auth-library-oauth2-http/0.9.0/google-auth-library-oauth2-http-0.9.0.jar +0 -0
  19. data/vendor/jar-dependencies/com/google/auto/value/auto-value/1.2/auto-value-1.2.jar +0 -0
  20. data/vendor/jar-dependencies/com/google/cloud/google-cloud-core-grpc/1.10.0/google-cloud-core-grpc-1.10.0.jar +0 -0
  21. data/vendor/jar-dependencies/com/google/cloud/google-cloud-core/1.10.0/google-cloud-core-1.10.0.jar +0 -0
  22. data/vendor/jar-dependencies/com/google/cloud/google-cloud-pubsub/0.28.0-beta/google-cloud-pubsub-0.28.0-beta.jar +0 -0
  23. data/vendor/jar-dependencies/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar +0 -0
  24. data/vendor/jar-dependencies/com/google/code/gson/gson/2.7/gson-2.7.jar +0 -0
  25. data/vendor/jar-dependencies/com/google/errorprone/error_prone_annotations/2.0.19/error_prone_annotations-2.0.19.jar +0 -0
  26. data/vendor/jar-dependencies/com/google/guava/guava/20.0/guava-20.0.jar +0 -0
  27. data/vendor/jar-dependencies/com/google/http-client/google-http-client-jackson2/1.23.0/google-http-client-jackson2-1.23.0.jar +0 -0
  28. data/vendor/jar-dependencies/com/google/http-client/google-http-client/1.23.0/google-http-client-1.23.0.jar +0 -0
  29. data/vendor/jar-dependencies/com/google/instrumentation/instrumentation-api/0.4.3/instrumentation-api-0.4.3.jar +0 -0
  30. data/vendor/jar-dependencies/com/google/protobuf/protobuf-java-util/3.4.0/protobuf-java-util-3.4.0.jar +0 -0
  31. data/vendor/jar-dependencies/com/google/protobuf/protobuf-java/3.5.1/protobuf-java-3.5.1.jar +0 -0
  32. data/vendor/jar-dependencies/commons-codec/commons-codec/1.3/commons-codec-1.3.jar +0 -0
  33. data/vendor/jar-dependencies/commons-logging/commons-logging/1.1.1/commons-logging-1.1.1.jar +0 -0
  34. data/vendor/jar-dependencies/io/grpc/grpc-auth/1.7.0/grpc-auth-1.7.0.jar +0 -0
  35. data/vendor/jar-dependencies/io/grpc/grpc-context/1.7.0/grpc-context-1.7.0.jar +0 -0
  36. data/vendor/jar-dependencies/io/grpc/grpc-core/1.7.0/grpc-core-1.7.0.jar +0 -0
  37. data/vendor/jar-dependencies/io/grpc/grpc-netty/1.7.0/grpc-netty-1.7.0.jar +0 -0
  38. data/vendor/jar-dependencies/io/grpc/grpc-protobuf-lite/1.7.0/grpc-protobuf-lite-1.7.0.jar +0 -0
  39. data/vendor/jar-dependencies/io/grpc/grpc-protobuf/1.7.0/grpc-protobuf-1.7.0.jar +0 -0
  40. data/vendor/jar-dependencies/io/grpc/grpc-stub/1.7.0/grpc-stub-1.7.0.jar +0 -0
  41. data/vendor/jar-dependencies/io/netty/netty-buffer/4.1.16.Final/netty-buffer-4.1.16.Final.jar +0 -0
  42. data/vendor/jar-dependencies/io/netty/netty-codec-http/4.1.16.Final/netty-codec-http-4.1.16.Final.jar +0 -0
  43. data/vendor/jar-dependencies/io/netty/netty-codec-http2/4.1.16.Final/netty-codec-http2-4.1.16.Final.jar +0 -0
  44. data/vendor/jar-dependencies/io/netty/netty-codec-socks/4.1.16.Final/netty-codec-socks-4.1.16.Final.jar +0 -0
  45. data/vendor/jar-dependencies/io/netty/netty-codec/4.1.16.Final/netty-codec-4.1.16.Final.jar +0 -0
  46. data/vendor/jar-dependencies/io/netty/netty-common/4.1.16.Final/netty-common-4.1.16.Final.jar +0 -0
  47. data/vendor/jar-dependencies/io/netty/netty-handler-proxy/4.1.16.Final/netty-handler-proxy-4.1.16.Final.jar +0 -0
  48. data/vendor/jar-dependencies/io/netty/netty-handler/4.1.16.Final/netty-handler-4.1.16.Final.jar +0 -0
  49. data/vendor/jar-dependencies/io/netty/netty-resolver/4.1.16.Final/netty-resolver-4.1.16.Final.jar +0 -0
  50. data/vendor/jar-dependencies/io/netty/netty-tcnative-boringssl-static/2.0.6.Final/netty-tcnative-boringssl-static-2.0.6.Final.jar +0 -0
  51. data/vendor/jar-dependencies/io/netty/netty-transport/4.1.16.Final/netty-transport-4.1.16.Final.jar +0 -0
  52. data/vendor/jar-dependencies/io/opencensus/opencensus-api/0.6.0/opencensus-api-0.6.0.jar +0 -0
  53. data/vendor/jar-dependencies/joda-time/joda-time/2.9.2/joda-time-2.9.2.jar +0 -0
  54. data/vendor/jar-dependencies/org/apache/httpcomponents/httpclient/4.0.1/httpclient-4.0.1.jar +0 -0
  55. data/vendor/jar-dependencies/org/apache/httpcomponents/httpcore/4.0.1/httpcore-4.0.1.jar +0 -0
  56. data/vendor/jar-dependencies/org/json/json/20160810/json-20160810.jar +0 -0
  57. data/vendor/jar-dependencies/org/threeten/threetenbp/1.3.3/threetenbp-1.3.3.jar +0 -0
  58. metadata +72 -20
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 3f39efa6b21fafa252ad3e27a1a1a301681145c88498ac242e7b85a77a64382f
4
- data.tar.gz: b92fe3cebc49c21702868a61a06a90c88b0f0372b5a7f57272add4a911359c1c
3
+ metadata.gz: f5bc9362c985c9940b9b4ae677adf15afe2fa6ea86bd85a83beafcd0a5092d42
4
+ data.tar.gz: 3f8279cf6a25d045d2776a2325fc90fd3edb81dfae312dc04efa82880e013e90
5
5
  SHA512:
6
- metadata.gz: a844cf0059e14822f1323a146b4f03b0745ab0f3c5a7c961626f35523059a632ef0169c96cf212612520b1a6ad5c6e428a08c30a82f8a8ffaab15d9c35d6b86b
7
- data.tar.gz: 1ca3f6026de8645af0ea71ab648bb52fc76ac4513373d21ad6bd6e39352576b403c56e738905f0f1372182f40e08485fed375b21c7dedd117df866010783d02b
6
+ metadata.gz: 4822ec58447e1c8e2d33f8ffb5f9e5d2766aeadf87d411107b588f304249f6069a419cc15703f5f4838f8c746133b2ea977c344d2ae3e3d9aa206a70b32e4de7
7
+ data.tar.gz: b3e81b01e38863bbefe468c4cff5234745e049a5468a1f9b4d5b5e1eac1ee6c024688aa24347ecf6aedd9655051bf6091f78b064659baed4afde176f8943f3a5
@@ -1,3 +1,7 @@
1
+ ## 1.2.0
2
+ - Change to Java client
3
+ - Add `create_subscription` setting. Fixes [#9](https://github.com/logstash-plugins/logstash-input-google_pubsub/issues/9)
4
+
1
5
  ## 1.1.0
2
6
  - Add additional attributes in the `[@metadata][pubsub_message]` field. Fixes [#7](https://github.com/logstash-plugins/logstash-input-google_pubsub/issues/7)
3
7
 
@@ -0,0 +1,23 @@
1
+ The following is a list of people who have contributed ideas, code, bug
2
+ reports, or in general have helped Logstash along its way.
3
+
4
+ Contributors:
5
+
6
+ * DeDe Morton (dedemorton)
7
+ * Eric Johnson (erjohnso)
8
+ * Google LLC
9
+ * Jake Landis (jakelandis)
10
+ * Jakub Sacha (jakubsacha)
11
+ * James Bloomer (jamesbloomer)
12
+ * João Duarte (jsvd)
13
+ * Jordan Sissel (jordansissel)
14
+ * Joseph Lewis III (josephlewis42)
15
+ * Pier-Hugues Pellerin (ph)
16
+ * Scott Brenner (ScottBrenner)
17
+ * Suyog Rao (suyograo)
18
+ * Sven Widén (SvenW)
19
+
20
+ Note: If you've sent us patches, bug reports, or otherwise contributed to
21
+ Logstash, and you aren't on the list above and want to be, please let us know
22
+ and we'll make sure you're here. Contributions from folks like you are what make
23
+ open source awesome.
data/README.md CHANGED
@@ -130,6 +130,7 @@ Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/log
130
130
  - Install dependencies
131
131
  ```sh
132
132
  bundle install
133
+ rake install_jars
133
134
  ```
134
135
 
135
136
  #### Test
@@ -133,6 +133,11 @@ input {
133
133
  # outside of GCE, you will need to specify the service account's
134
134
  # JSON key file below.
135
135
  #json_key_file => "/home/erjohnso/pkey.json"
136
+
137
+ # Should the plugin attempt to create the subscription on startup?
138
+ # This is not recommended for security reasons but may be useful in
139
+ # some cases.
140
+ #create_subscription => false
136
141
  }
137
142
  }
138
143
  output { stdout { codec => rubydebug } }
@@ -185,6 +190,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
185
190
  | <<plugins-{type}s-{plugin}-subscription>> |<<string,string>>|Yes
186
191
  | <<plugins-{type}s-{plugin}-topic>> |<<string,string>>|Yes
187
192
  | <<plugins-{type}s-{plugin}-include_metadata>> |<<boolean,boolean>>|No
193
+ | <<plugins-{type}s-{plugin}-create_subscription>> |<<boolean,boolean>>|No
188
194
  |=======================================================================
189
195
 
190
196
  Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
@@ -250,6 +256,18 @@ The subscription will be created automatically by the plugin.
250
256
 
251
257
  If set true, will include the full message data in the `[@metadata][pubsub_message]` field.
252
258
 
259
+ [id="plugins-{type}s-{plugin}-create_subscription"]
260
+ ===== `create_subscription`
261
+
262
+ added[1.2.0]
263
+
264
+ * Value type is <<boolean,boolean>>
265
+ * Default value is `false`.
266
+
267
+ If true, the plugin will try to create the subscription before publishing.
268
+ Note: this requires additional permissions to be granted to the client and is _not_
269
+ recommended for most use-cases.
270
+
253
271
  [id="plugins-{type}s-{plugin}-common-options"]
254
272
  include::{include_path}/{type}.asciidoc[]
255
273
 
@@ -0,0 +1,52 @@
1
+ # AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
2
+
3
+ require 'jar_dependencies'
4
+ require_jar('com.google.api.grpc', 'proto-google-iam-v1', '0.1.24')
5
+ require_jar('io.netty', 'netty-buffer', '4.1.16.Final')
6
+ require_jar('org.threeten', 'threetenbp', '1.3.3')
7
+ require_jar('com.google.errorprone', 'error_prone_annotations', '2.0.19')
8
+ require_jar('com.google.api.grpc', 'grpc-google-cloud-pubsub-v1', '0.1.24')
9
+ require_jar('io.netty', 'netty-codec-http', '4.1.16.Final')
10
+ require_jar('io.grpc', 'grpc-protobuf-lite', '1.7.0')
11
+ require_jar('io.netty', 'netty-common', '4.1.16.Final')
12
+ require_jar('com.google.auto.value', 'auto-value', '1.2')
13
+ require_jar('com.google.protobuf', 'protobuf-java-util', '3.4.0')
14
+ require_jar('com.google.http-client', 'google-http-client-jackson2', '1.23.0')
15
+ require_jar('com.fasterxml.jackson.core', 'jackson-core', '2.1.3')
16
+ require_jar('com.google.code.findbugs', 'jsr305', '3.0.2')
17
+ require_jar('joda-time', 'joda-time', '2.9.2')
18
+ require_jar('io.netty', 'netty-handler', '4.1.16.Final')
19
+ require_jar('com.google.http-client', 'google-http-client', '1.23.0')
20
+ require_jar('commons-logging', 'commons-logging', '1.1.1')
21
+ require_jar('org.json', 'json', '20160810')
22
+ require_jar('com.google.cloud', 'google-cloud-core', '1.10.0')
23
+ require_jar('io.netty', 'netty-codec-http2', '4.1.16.Final')
24
+ require_jar('com.google.guava', 'guava', '20.0')
25
+ require_jar('com.google.cloud', 'google-cloud-core-grpc', '1.10.0')
26
+ require_jar('com.google.protobuf', 'protobuf-java', '3.5.1')
27
+ require_jar('com.google.api', 'gax', '1.14.0')
28
+ require_jar('org.apache.httpcomponents', 'httpclient', '4.0.1')
29
+ require_jar('com.google.api', 'gax-grpc', '1.14.0')
30
+ require_jar('com.google.auth', 'google-auth-library-credentials', '0.9.0')
31
+ require_jar('com.google.api', 'api-common', '1.2.0')
32
+ require_jar('com.google.api.grpc', 'proto-google-common-protos', '1.0.0')
33
+ require_jar('io.grpc', 'grpc-auth', '1.7.0')
34
+ require_jar('io.grpc', 'grpc-core', '1.7.0')
35
+ require_jar('io.netty', 'netty-codec-socks', '4.1.16.Final')
36
+ require_jar('io.grpc', 'grpc-protobuf', '1.7.0')
37
+ require_jar('io.grpc', 'grpc-context', '1.7.0')
38
+ require_jar('io.netty', 'netty-transport', '4.1.16.Final')
39
+ require_jar('commons-codec', 'commons-codec', '1.3')
40
+ require_jar('com.google.instrumentation', 'instrumentation-api', '0.4.3')
41
+ require_jar('io.netty', 'netty-handler-proxy', '4.1.16.Final')
42
+ require_jar('io.netty', 'netty-resolver', '4.1.16.Final')
43
+ require_jar('io.netty', 'netty-tcnative-boringssl-static', '2.0.6.Final')
44
+ require_jar('com.google.code.gson', 'gson', '2.7')
45
+ require_jar('io.grpc', 'grpc-netty', '1.7.0')
46
+ require_jar('io.opencensus', 'opencensus-api', '0.6.0')
47
+ require_jar('org.apache.httpcomponents', 'httpcore', '4.0.1')
48
+ require_jar('io.netty', 'netty-codec', '4.1.16.Final')
49
+ require_jar('io.grpc', 'grpc-stub', '1.7.0')
50
+ require_jar('com.google.cloud', 'google-cloud-pubsub', '0.28.0-beta')
51
+ require_jar('com.google.api.grpc', 'proto-google-cloud-pubsub-v1', '0.1.24')
52
+ require_jar('com.google.auth', 'google-auth-library-oauth2-http', '0.9.0')
@@ -19,8 +19,8 @@
19
19
  require "logstash/inputs/base"
20
20
  require "logstash/namespace"
21
21
 
22
- # Google deps
23
- require "google/api_client"
22
+ require 'java'
23
+ require 'logstash-input-google_pubsub_jars.rb'
24
24
 
25
25
  # This is a https://github.com/elastic/logstash[Logstash] input plugin for
26
26
  # https://cloud.google.com/pubsub/[Google Pub/Sub]. The plugin can subscribe
@@ -123,38 +123,70 @@ require "google/api_client"
123
123
  # ----------------------------------
124
124
  #
125
125
  # ==== Metadata and Attributes
126
- #
127
- # The original Pub/Sub message is preserved in the special Logstash
126
+ #
127
+ # The original Pub/Sub message is preserved in the special Logstash
128
128
  # `[@metadata][pubsub_message]` field so you can fetch:
129
- #
129
+ #
130
130
  # * Message attributes
131
131
  # * The origiginal base64 data
132
132
  # * Pub/Sub message ID for de-duplication
133
133
  # * Publish time
134
- #
134
+ #
135
135
  # You MUST extract any fields you want in a filter prior to the data being sent
136
136
  # to an output because Logstash deletes `@metadata` fields otherwise.
137
- #
137
+ #
138
138
  # See the PubsubMessage
139
139
  # https://cloud.google.com/pubsub/docs/reference/rest/v1/PubsubMessage[documentation]
140
140
  # for a full description of the fields.
141
- #
141
+ #
142
142
  # Example to get the message ID:
143
- #
143
+ #
144
144
  # [source,ruby]
145
- # ----------------------------------
145
+ # ----------------------------------
146
146
  # input {google_pubsub {...}}
147
- #
147
+ #
148
148
  # filter {
149
149
  # mutate {
150
150
  # add_field => { "messageId" => "%{[@metadata][pubsub_message][messageId]}" }
151
151
  # }
152
152
  # }
153
- #
153
+ #
154
154
  # output {...}
155
155
  # ----------------------------------
156
156
  #
157
+
157
158
  class LogStash::Inputs::GooglePubSub < LogStash::Inputs::Base
159
+ class MessageReceiver
160
+ include com.google.cloud.pubsub.v1.MessageReceiver
161
+
162
+ def initialize(&blk)
163
+ @block = blk
164
+ end
165
+
166
+ def receiveMessage(message, consumer)
167
+ @block.call(message)
168
+ consumer.ack()
169
+ end
170
+ end
171
+
172
+ java_import 'com.google.api.core.ApiService$Listener'
173
+ class SubscriberListener < Listener
174
+ def initialize(&blk)
175
+ @block = blk
176
+ end
177
+
178
+ def failed(from, failure)
179
+ @block.call(from, failure)
180
+ end
181
+ end
182
+
183
+ include_package 'com.google.api.gax.batching'
184
+ include_package 'com.google.api.gax.core'
185
+ include_package 'com.google.auth.oauth2'
186
+ include_package 'com.google.common.util.concurrent'
187
+ include_package 'com.google.cloud.pubsub.v1'
188
+ include_package 'com.google.pubsub.v1'
189
+ include_package 'com.google.protobuf.util'
158
190
  config_name "google_pubsub"
159
191
 
160
192
  # Google Cloud Project ID (name, not number)
@@ -176,140 +208,81 @@ class LogStash::Inputs::GooglePubSub < LogStash::Inputs::Base
176
208
  # If set true, will include the full message data in the `[@metadata][pubsub_message]` field.
177
209
  config :include_metadata, :validate => :boolean, :required => false, :default => false
178
210
 
211
+ # If true, the plugin will try to create the subscription before publishing.
212
+ # Note: this requires additional permissions to be granted to the client and is _not_
213
+ # recommended for most use-cases.
214
+ config :create_subscription, :validate => :boolean, :required => false, :default => false
215
+
179
216
  # If undefined, Logstash will complain, even if codec is unused.
180
217
  default :codec, "plain"
181
218
 
182
- private
183
- def request(options)
184
- begin
185
- @logger.debug("Sending an API request")
186
- result = @client.execute(options)
187
- rescue ArgumentError => e
188
- @logger.debug("Authorizing...")
189
- @client.authorization.fetch_access_token!
190
- @logger.debug("...authorized")
191
- request(options)
192
- rescue Faraday::TimeoutError => e
193
- @logger.debug("Request timeout, re-trying request")
194
- request(options)
195
- end
196
- end # def request
197
-
198
219
  public
199
220
  def register
200
221
  @logger.debug("Registering Google PubSub Input: project_id=#{@project_id}, topic=#{@topic}, subscription=#{@subscription}")
201
- @topic = "projects/#{@project_id}/topics/#{@topic}"
202
- @subscription = "projects/#{@project_id}/subscriptions/#{@subscription}"
203
- @subscription_exists = false
204
-
205
- # TODO(erjohnso): read UA data from the gemspec
206
- @client = Google::APIClient.new(
207
- :application_name => 'logstash-input-google_pubsub',
208
- :application_version => '0.9.0'
209
- )
222
+ @subscription_id = "projects/#{@project_id}/subscriptions/#{@subscription}"
210
223
 
211
- # Initialize the pubsub API client
212
- @pubsub = @client.discovered_api('pubsub', 'v1')
213
-
214
- # Handle various kinds of auth (JSON or Application Default Creds)
215
- # NOTE: Cannot use 'googleauth' gem since there are dependency conflicts
216
- # - googleauth ~> 0.5 requires mime-data-types that requires ruby2
217
- # - googleauth ~> 0.3 requires multi_json 1.11.0 that conflicts
218
- # with logstash-2.3.2's multi_json 1.11.3
219
224
  if @json_key_file
220
- @logger.debug("Authorizing with JSON key file: #{@json_key_file}")
221
- file_path = File.expand_path(@json_key_file)
222
- key_json = File.open(file_path, "r", &:read)
223
- key_json = JSON.parse(key_json)
224
- unless key_json.key?("client_email") || key_json.key?("private_key")
225
- raise Google::APIClient::ClientError, "Invalid JSON credentials data."
226
- end
227
- signing_key = ::Google::APIClient::KeyUtils.load_from_pem(key_json["private_key"], "notasecret")
228
- @client.authorization = Signet::OAuth2::Client.new(
229
- :audience => "https://accounts.google.com/o/oauth2/token",
230
- :auth_provider_x509_cert_url => "https://www.googleapis.com/oauth2/v1/certs",
231
- :client_x509_cert_url => "https://www.googleapis.com/robot/v1/metadata/x509/#{key_json['client_email']}",
232
- :issuer => "#{key_json['client_email']}",
233
- :scope => %w(https://www.googleapis.com/auth/cloud-platform),
234
- :signing_key => signing_key,
235
- :token_credential_uri => "https://accounts.google.com/o/oauth2/token"
225
+ @credentialsProvider = FixedCredentialsProvider.create(
226
+ ServiceAccountCredentials.fromStream(java.io.FileInputStream.new(@json_key_file))
236
227
  )
237
- @logger.info("Client authorization with JSON key ready")
238
- else
239
- # Assume we're running in GCE and can use metadata tokens, if the host
240
- # GCE instance was not created with the PubSub scope, then the plugin
241
- # will not be authorized to read from pubsub.
242
- @logger.info("Authorizing with application default credentials")
243
- @client.authorization = :google_app_default
244
- end # if @json_key_file...
245
- end # def register
228
+ end
229
+ @topic_name = TopicName.create(@project_id, @topic)
230
+ @subscription_name = SubscriptionName.create(@project_id, @subscription)
231
+ end
232
+
233
+ def stop
234
+ @subscriber.stopAsync().awaitTerminated() if @subscriber != nil
235
+ end
246
236
 
247
237
  def run(queue)
248
238
  # Attempt to create the subscription
249
- if !@subscription_exists
250
- @logger.debug("Creating subscription #{subscription}")
251
- result = request(
252
- :api_method => @pubsub.projects.subscriptions.create,
253
- :parameters => {'name' => @subscription},
254
- :body_object => {
255
- :topic => @topic,
256
- :ackDeadlineSeconds => 15
257
- }
258
- )
259
- if result.error? and result.status != 409
260
- raise Google::APIClient::ClientError, "Error #{result.status}: #{result.error_message}"
239
+ if @create_subscription
240
+ @logger.debug("Creating subscription #{@subscription_id}")
241
+ subscriptionAdminClient = SubscriptionAdminClient.create
242
+ begin
243
+ subscriptionAdminClient.createSubscription(@subscription_name, @topic_name, PushConfig.getDefaultInstance(), 0)
244
+ rescue
245
+ @logger.info("Subscription already exists")
261
246
  end
262
- @subscription_exists = true
263
- end # if !@subscription
264
-
265
- @logger.debug("Pulling messages from sub '#{subscription}'")
266
- while !stop?
267
- # Pull and queue messages
268
- messages = []
269
- result = request(
270
- :api_method => @pubsub.projects.subscriptions.pull,
271
- :parameters => {'subscription' => @subscription},
272
- :body_object => {
273
- :returnImmediately => false,
274
- :maxMessages => @max_messages
275
- }
276
- )
247
+ end
277
248
 
278
- if !result.error?
279
- messages = JSON.parse(result.body)
280
- if messages.key?("receivedMessages")
281
- messages = messages["receivedMessages"]
282
- end
283
- else
284
- @logger.error("Error pulling messages:'#{result.error_message}'")
249
+ @logger.debug("Pulling messages from sub '#{@subscription_id}'")
250
+ handler = MessageReceiver.new do |message|
251
+ # handle incoming message, then ack/nack the received message
252
+ data = message.getData().toStringUtf8()
253
+ @codec.decode(data) do |event|
254
+ event.set("host", event.get("host") || @host)
255
+ event.set("[@metadata][pubsub_message]", extract_metadata(message)) if @include_metadata
256
+ decorate(event)
257
+ queue << event
285
258
  end
259
+ end
260
+ listener = SubscriberListener.new do |from, failure|
261
+ @logger.error("#{failure}")
262
+ raise failure
263
+ end
264
+ flowControlSettings = FlowControlSettings.newBuilder().setMaxOutstandingElementCount(@max_messages).build()
265
+ executorProvider = InstantiatingExecutorProvider.newBuilder().setExecutorThreadCount(1).build()
266
+ subscriberBuilder = Subscriber.newBuilder(@subscription_name, handler)
267
+ .setFlowControlSettings(flowControlSettings)
268
+ .setExecutorProvider(executorProvider)
269
+ .setParallelPullCount(1)
286
270
 
287
- if messages
288
- messages.each do |msg|
289
- if msg.key?("message") and msg["message"].key?("data")
290
- decoded_msg = Base64.decode64(msg["message"]["data"])
291
- @codec.decode(decoded_msg) do |event|
292
- event.set("[@metadata][pubsub_message]", msg["message"]) if @include_metadata
293
- decorate(event)
294
- queue << event
295
- end
296
- end
297
- end
298
-
299
- ack_ids = messages.map{ |msg| msg["ackId"] }
300
- next if ack_ids.empty?
271
+ if @credentialsProvider
272
+ subscriberBuilder.setCredentialsProvider(@credentialsProvider)
273
+ end
274
+ @subscriber = subscriberBuilder.build()
275
+ @subscriber.addListener(listener, MoreExecutors.directExecutor())
276
+ @subscriber.startAsync()
277
+ @subscriber.awaitTerminated()
278
+ end
301
279
 
302
- result = request(
303
- :api_method => @pubsub.projects.subscriptions.acknowledge,
304
- :parameters => {'subscription' => @subscription},
305
- :body_object => {
306
- :ackIds => ack_ids
307
- }
308
- )
309
- if result.error?
310
- @logger.error("Error #{result.status}: #{result.error_message}")
311
- end
312
- end # if messages
313
- end # loop
314
- end # def run
315
- end # class LogStash::Inputs::GooglePubSub
280
+ def extract_metadata(java_message)
281
+ {
282
+ data: java_message.getData().toStringUtf8(),
283
+ attributes: java_message.getAttributesMap(),
284
+ messageId: java_message.getMessageId(),
285
+ publishTime: Timestamps.toString(java_message.getPublishTime())
286
+ }
287
+ end
288
+ end
@@ -1,13 +1,13 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-input-google_pubsub'
3
- s.version = '1.1.0'
3
+ s.version = '1.2.0'
4
4
  s.licenses = ['Apache-2.0']
5
5
  s.summary = "Consume events from a Google Cloud PubSub service"
6
6
  s.description = "This gem is a Logstash input plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program."
7
7
  s.authors = ["Eric Johnson"]
8
8
  s.email = 'erjohnso@google.com'
9
9
  s.homepage = "https://cloud.google.com/pubsub/overview"
10
- s.require_paths = ["lib"]
10
+ s.require_paths = ["lib", "vendor/jar-dependencies"]
11
11
 
12
12
  # Files
13
13
  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
@@ -22,8 +22,13 @@ Gem::Specification.new do |s|
22
22
  s.add_runtime_dependency 'logstash-codec-plain'
23
23
  s.add_runtime_dependency 'stud', '>= 0.0.22'
24
24
  # Google dependencies
25
- # google-api-client >= 0.9 requires ruby2 which is not currently compatible
26
25
  # with JRuby
27
- s.add_runtime_dependency 'google-api-client', '~> 0.8.6', '< 0.9'
26
+ s.requirements << "jar 'com.google.cloud:google-cloud-pubsub', '0.28.0-beta'"
27
+ s.requirements << "jar 'com.google.api.grpc:proto-google-cloud-pubsub-v1', '0.1.24'"
28
+ s.requirements << "jar 'com.google.api:gax', '1.14.0'"
29
+ s.requirements << "jar 'com.google.guava:guava', '20.0'"
30
+ s.requirements << "jar 'com.google.api:api-common', '1.2.0'"
31
+ s.requirements << "jar 'com.google.auth:google-auth-library-oauth2-http', '0.9.0'"
28
32
  s.add_development_dependency 'logstash-devutils'
33
+ s.add_development_dependency 'jar-dependencies', '~> 0.3.2'
29
34
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-input-google_pubsub
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.0
4
+ version: 1.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Eric Johnson
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-04-26 00:00:00.000000000 Z
11
+ date: 2018-05-07 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -61,37 +61,31 @@ dependencies:
61
61
  - !ruby/object:Gem::Dependency
62
62
  requirement: !ruby/object:Gem::Requirement
63
63
  requirements:
64
- - - "~>"
65
- - !ruby/object:Gem::Version
66
- version: 0.8.6
67
- - - "<"
64
+ - - ">="
68
65
  - !ruby/object:Gem::Version
69
- version: '0.9'
70
- name: google-api-client
66
+ version: '0'
67
+ name: logstash-devutils
71
68
  prerelease: false
72
- type: :runtime
69
+ type: :development
73
70
  version_requirements: !ruby/object:Gem::Requirement
74
71
  requirements:
75
- - - "~>"
76
- - !ruby/object:Gem::Version
77
- version: 0.8.6
78
- - - "<"
72
+ - - ">="
79
73
  - !ruby/object:Gem::Version
80
- version: '0.9'
74
+ version: '0'
81
75
  - !ruby/object:Gem::Dependency
82
76
  requirement: !ruby/object:Gem::Requirement
83
77
  requirements:
84
- - - ">="
78
+ - - "~>"
85
79
  - !ruby/object:Gem::Version
86
- version: '0'
87
- name: logstash-devutils
80
+ version: 0.3.2
81
+ name: jar-dependencies
88
82
  prerelease: false
89
83
  type: :development
90
84
  version_requirements: !ruby/object:Gem::Requirement
91
85
  requirements:
92
- - - ">="
86
+ - - "~>"
93
87
  - !ruby/object:Gem::Version
94
- version: '0'
88
+ version: 0.3.2
95
89
  description: This gem is a Logstash input plugin required to be installed on top of
96
90
  the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
97
91
  gem is not a stand-alone program.
@@ -102,14 +96,65 @@ extra_rdoc_files: []
102
96
  files:
103
97
  - CHANGELOG.md
104
98
  - CONTRIBUTING.md
99
+ - CONTRIBUTORS
105
100
  - Gemfile
106
101
  - LICENSE
107
102
  - NOTICE.TXT
108
103
  - README.md
109
104
  - docs/index.asciidoc
105
+ - lib/logstash-input-google_pubsub_jars.rb
110
106
  - lib/logstash/inputs/google_pubsub.rb
111
107
  - logstash-input-google_pubsub.gemspec
112
108
  - spec/inputs/google_pubsub_spec.rb
109
+ - vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-core/2.1.3/jackson-core-2.1.3.jar
110
+ - vendor/jar-dependencies/com/google/api/api-common/1.2.0/api-common-1.2.0.jar
111
+ - vendor/jar-dependencies/com/google/api/gax-grpc/1.14.0/gax-grpc-1.14.0.jar
112
+ - vendor/jar-dependencies/com/google/api/gax/1.14.0/gax-1.14.0.jar
113
+ - vendor/jar-dependencies/com/google/api/grpc/grpc-google-cloud-pubsub-v1/0.1.24/grpc-google-cloud-pubsub-v1-0.1.24.jar
114
+ - vendor/jar-dependencies/com/google/api/grpc/proto-google-cloud-pubsub-v1/0.1.24/proto-google-cloud-pubsub-v1-0.1.24.jar
115
+ - vendor/jar-dependencies/com/google/api/grpc/proto-google-common-protos/1.0.0/proto-google-common-protos-1.0.0.jar
116
+ - vendor/jar-dependencies/com/google/api/grpc/proto-google-iam-v1/0.1.24/proto-google-iam-v1-0.1.24.jar
117
+ - vendor/jar-dependencies/com/google/auth/google-auth-library-credentials/0.9.0/google-auth-library-credentials-0.9.0.jar
118
+ - vendor/jar-dependencies/com/google/auth/google-auth-library-oauth2-http/0.9.0/google-auth-library-oauth2-http-0.9.0.jar
119
+ - vendor/jar-dependencies/com/google/auto/value/auto-value/1.2/auto-value-1.2.jar
120
+ - vendor/jar-dependencies/com/google/cloud/google-cloud-core-grpc/1.10.0/google-cloud-core-grpc-1.10.0.jar
121
+ - vendor/jar-dependencies/com/google/cloud/google-cloud-core/1.10.0/google-cloud-core-1.10.0.jar
122
+ - vendor/jar-dependencies/com/google/cloud/google-cloud-pubsub/0.28.0-beta/google-cloud-pubsub-0.28.0-beta.jar
123
+ - vendor/jar-dependencies/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar
124
+ - vendor/jar-dependencies/com/google/code/gson/gson/2.7/gson-2.7.jar
125
+ - vendor/jar-dependencies/com/google/errorprone/error_prone_annotations/2.0.19/error_prone_annotations-2.0.19.jar
126
+ - vendor/jar-dependencies/com/google/guava/guava/20.0/guava-20.0.jar
127
+ - vendor/jar-dependencies/com/google/http-client/google-http-client-jackson2/1.23.0/google-http-client-jackson2-1.23.0.jar
128
+ - vendor/jar-dependencies/com/google/http-client/google-http-client/1.23.0/google-http-client-1.23.0.jar
129
+ - vendor/jar-dependencies/com/google/instrumentation/instrumentation-api/0.4.3/instrumentation-api-0.4.3.jar
130
+ - vendor/jar-dependencies/com/google/protobuf/protobuf-java-util/3.4.0/protobuf-java-util-3.4.0.jar
131
+ - vendor/jar-dependencies/com/google/protobuf/protobuf-java/3.5.1/protobuf-java-3.5.1.jar
132
+ - vendor/jar-dependencies/commons-codec/commons-codec/1.3/commons-codec-1.3.jar
133
+ - vendor/jar-dependencies/commons-logging/commons-logging/1.1.1/commons-logging-1.1.1.jar
134
+ - vendor/jar-dependencies/io/grpc/grpc-auth/1.7.0/grpc-auth-1.7.0.jar
135
+ - vendor/jar-dependencies/io/grpc/grpc-context/1.7.0/grpc-context-1.7.0.jar
136
+ - vendor/jar-dependencies/io/grpc/grpc-core/1.7.0/grpc-core-1.7.0.jar
137
+ - vendor/jar-dependencies/io/grpc/grpc-netty/1.7.0/grpc-netty-1.7.0.jar
138
+ - vendor/jar-dependencies/io/grpc/grpc-protobuf-lite/1.7.0/grpc-protobuf-lite-1.7.0.jar
139
+ - vendor/jar-dependencies/io/grpc/grpc-protobuf/1.7.0/grpc-protobuf-1.7.0.jar
140
+ - vendor/jar-dependencies/io/grpc/grpc-stub/1.7.0/grpc-stub-1.7.0.jar
141
+ - vendor/jar-dependencies/io/netty/netty-buffer/4.1.16.Final/netty-buffer-4.1.16.Final.jar
142
+ - vendor/jar-dependencies/io/netty/netty-codec-http/4.1.16.Final/netty-codec-http-4.1.16.Final.jar
143
+ - vendor/jar-dependencies/io/netty/netty-codec-http2/4.1.16.Final/netty-codec-http2-4.1.16.Final.jar
144
+ - vendor/jar-dependencies/io/netty/netty-codec-socks/4.1.16.Final/netty-codec-socks-4.1.16.Final.jar
145
+ - vendor/jar-dependencies/io/netty/netty-codec/4.1.16.Final/netty-codec-4.1.16.Final.jar
146
+ - vendor/jar-dependencies/io/netty/netty-common/4.1.16.Final/netty-common-4.1.16.Final.jar
147
+ - vendor/jar-dependencies/io/netty/netty-handler-proxy/4.1.16.Final/netty-handler-proxy-4.1.16.Final.jar
148
+ - vendor/jar-dependencies/io/netty/netty-handler/4.1.16.Final/netty-handler-4.1.16.Final.jar
149
+ - vendor/jar-dependencies/io/netty/netty-resolver/4.1.16.Final/netty-resolver-4.1.16.Final.jar
150
+ - vendor/jar-dependencies/io/netty/netty-tcnative-boringssl-static/2.0.6.Final/netty-tcnative-boringssl-static-2.0.6.Final.jar
151
+ - vendor/jar-dependencies/io/netty/netty-transport/4.1.16.Final/netty-transport-4.1.16.Final.jar
152
+ - vendor/jar-dependencies/io/opencensus/opencensus-api/0.6.0/opencensus-api-0.6.0.jar
153
+ - vendor/jar-dependencies/joda-time/joda-time/2.9.2/joda-time-2.9.2.jar
154
+ - vendor/jar-dependencies/org/apache/httpcomponents/httpclient/4.0.1/httpclient-4.0.1.jar
155
+ - vendor/jar-dependencies/org/apache/httpcomponents/httpcore/4.0.1/httpcore-4.0.1.jar
156
+ - vendor/jar-dependencies/org/json/json/20160810/json-20160810.jar
157
+ - vendor/jar-dependencies/org/threeten/threetenbp/1.3.3/threetenbp-1.3.3.jar
113
158
  homepage: https://cloud.google.com/pubsub/overview
114
159
  licenses:
115
160
  - Apache-2.0
@@ -120,6 +165,7 @@ post_install_message:
120
165
  rdoc_options: []
121
166
  require_paths:
122
167
  - lib
168
+ - vendor/jar-dependencies
123
169
  required_ruby_version: !ruby/object:Gem::Requirement
124
170
  requirements:
125
171
  - - ">="
@@ -130,7 +176,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
130
176
  - - ">="
131
177
  - !ruby/object:Gem::Version
132
178
  version: '0'
133
- requirements: []
179
+ requirements:
180
+ - jar 'com.google.cloud:google-cloud-pubsub', '0.28.0-beta'
181
+ - jar 'com.google.api.grpc:proto-google-cloud-pubsub-v1', '0.1.24'
182
+ - jar 'com.google.api:gax', '1.14.0'
183
+ - jar 'com.google.guava:guava', '20.0'
184
+ - jar 'com.google.api:api-common', '1.2.0'
185
+ - jar 'com.google.auth:google-auth-library-oauth2-http', '0.9.0'
134
186
  rubyforge_project:
135
187
  rubygems_version: 2.6.13
136
188
  signing_key: