logstash-input-google_pubsub 1.1.0 → 1.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/CONTRIBUTORS +23 -0
- data/README.md +1 -0
- data/docs/index.asciidoc +18 -0
- data/lib/logstash-input-google_pubsub_jars.rb +52 -0
- data/lib/logstash/inputs/google_pubsub.rb +105 -132
- data/logstash-input-google_pubsub.gemspec +9 -4
- data/vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-core/2.1.3/jackson-core-2.1.3.jar +0 -0
- data/vendor/jar-dependencies/com/google/api/api-common/1.2.0/api-common-1.2.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/api/gax-grpc/1.14.0/gax-grpc-1.14.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/api/gax/1.14.0/gax-1.14.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/api/grpc/grpc-google-cloud-pubsub-v1/0.1.24/grpc-google-cloud-pubsub-v1-0.1.24.jar +0 -0
- data/vendor/jar-dependencies/com/google/api/grpc/proto-google-cloud-pubsub-v1/0.1.24/proto-google-cloud-pubsub-v1-0.1.24.jar +0 -0
- data/vendor/jar-dependencies/com/google/api/grpc/proto-google-common-protos/1.0.0/proto-google-common-protos-1.0.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/api/grpc/proto-google-iam-v1/0.1.24/proto-google-iam-v1-0.1.24.jar +0 -0
- data/vendor/jar-dependencies/com/google/auth/google-auth-library-credentials/0.9.0/google-auth-library-credentials-0.9.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/auth/google-auth-library-oauth2-http/0.9.0/google-auth-library-oauth2-http-0.9.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/auto/value/auto-value/1.2/auto-value-1.2.jar +0 -0
- data/vendor/jar-dependencies/com/google/cloud/google-cloud-core-grpc/1.10.0/google-cloud-core-grpc-1.10.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/cloud/google-cloud-core/1.10.0/google-cloud-core-1.10.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/cloud/google-cloud-pubsub/0.28.0-beta/google-cloud-pubsub-0.28.0-beta.jar +0 -0
- data/vendor/jar-dependencies/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar +0 -0
- data/vendor/jar-dependencies/com/google/code/gson/gson/2.7/gson-2.7.jar +0 -0
- data/vendor/jar-dependencies/com/google/errorprone/error_prone_annotations/2.0.19/error_prone_annotations-2.0.19.jar +0 -0
- data/vendor/jar-dependencies/com/google/guava/guava/20.0/guava-20.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/http-client/google-http-client-jackson2/1.23.0/google-http-client-jackson2-1.23.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/http-client/google-http-client/1.23.0/google-http-client-1.23.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/instrumentation/instrumentation-api/0.4.3/instrumentation-api-0.4.3.jar +0 -0
- data/vendor/jar-dependencies/com/google/protobuf/protobuf-java-util/3.4.0/protobuf-java-util-3.4.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/protobuf/protobuf-java/3.5.1/protobuf-java-3.5.1.jar +0 -0
- data/vendor/jar-dependencies/commons-codec/commons-codec/1.3/commons-codec-1.3.jar +0 -0
- data/vendor/jar-dependencies/commons-logging/commons-logging/1.1.1/commons-logging-1.1.1.jar +0 -0
- data/vendor/jar-dependencies/io/grpc/grpc-auth/1.7.0/grpc-auth-1.7.0.jar +0 -0
- data/vendor/jar-dependencies/io/grpc/grpc-context/1.7.0/grpc-context-1.7.0.jar +0 -0
- data/vendor/jar-dependencies/io/grpc/grpc-core/1.7.0/grpc-core-1.7.0.jar +0 -0
- data/vendor/jar-dependencies/io/grpc/grpc-netty/1.7.0/grpc-netty-1.7.0.jar +0 -0
- data/vendor/jar-dependencies/io/grpc/grpc-protobuf-lite/1.7.0/grpc-protobuf-lite-1.7.0.jar +0 -0
- data/vendor/jar-dependencies/io/grpc/grpc-protobuf/1.7.0/grpc-protobuf-1.7.0.jar +0 -0
- data/vendor/jar-dependencies/io/grpc/grpc-stub/1.7.0/grpc-stub-1.7.0.jar +0 -0
- data/vendor/jar-dependencies/io/netty/netty-buffer/4.1.16.Final/netty-buffer-4.1.16.Final.jar +0 -0
- data/vendor/jar-dependencies/io/netty/netty-codec-http/4.1.16.Final/netty-codec-http-4.1.16.Final.jar +0 -0
- data/vendor/jar-dependencies/io/netty/netty-codec-http2/4.1.16.Final/netty-codec-http2-4.1.16.Final.jar +0 -0
- data/vendor/jar-dependencies/io/netty/netty-codec-socks/4.1.16.Final/netty-codec-socks-4.1.16.Final.jar +0 -0
- data/vendor/jar-dependencies/io/netty/netty-codec/4.1.16.Final/netty-codec-4.1.16.Final.jar +0 -0
- data/vendor/jar-dependencies/io/netty/netty-common/4.1.16.Final/netty-common-4.1.16.Final.jar +0 -0
- data/vendor/jar-dependencies/io/netty/netty-handler-proxy/4.1.16.Final/netty-handler-proxy-4.1.16.Final.jar +0 -0
- data/vendor/jar-dependencies/io/netty/netty-handler/4.1.16.Final/netty-handler-4.1.16.Final.jar +0 -0
- data/vendor/jar-dependencies/io/netty/netty-resolver/4.1.16.Final/netty-resolver-4.1.16.Final.jar +0 -0
- data/vendor/jar-dependencies/io/netty/netty-tcnative-boringssl-static/2.0.6.Final/netty-tcnative-boringssl-static-2.0.6.Final.jar +0 -0
- data/vendor/jar-dependencies/io/netty/netty-transport/4.1.16.Final/netty-transport-4.1.16.Final.jar +0 -0
- data/vendor/jar-dependencies/io/opencensus/opencensus-api/0.6.0/opencensus-api-0.6.0.jar +0 -0
- data/vendor/jar-dependencies/joda-time/joda-time/2.9.2/joda-time-2.9.2.jar +0 -0
- data/vendor/jar-dependencies/org/apache/httpcomponents/httpclient/4.0.1/httpclient-4.0.1.jar +0 -0
- data/vendor/jar-dependencies/org/apache/httpcomponents/httpcore/4.0.1/httpcore-4.0.1.jar +0 -0
- data/vendor/jar-dependencies/org/json/json/20160810/json-20160810.jar +0 -0
- data/vendor/jar-dependencies/org/threeten/threetenbp/1.3.3/threetenbp-1.3.3.jar +0 -0
- metadata +72 -20
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f5bc9362c985c9940b9b4ae677adf15afe2fa6ea86bd85a83beafcd0a5092d42
|
4
|
+
data.tar.gz: 3f8279cf6a25d045d2776a2325fc90fd3edb81dfae312dc04efa82880e013e90
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4822ec58447e1c8e2d33f8ffb5f9e5d2766aeadf87d411107b588f304249f6069a419cc15703f5f4838f8c746133b2ea977c344d2ae3e3d9aa206a70b32e4de7
|
7
|
+
data.tar.gz: b3e81b01e38863bbefe468c4cff5234745e049a5468a1f9b4d5b5e1eac1ee6c024688aa24347ecf6aedd9655051bf6091f78b064659baed4afde176f8943f3a5
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,7 @@
|
|
1
|
+
## 1.2.0
|
2
|
+
- Change to Java client
|
3
|
+
- Add `create_subscription` setting. Fixes [#9](https://github.com/logstash-plugins/logstash-input-google_pubsub/issues/9)
|
4
|
+
|
1
5
|
## 1.1.0
|
2
6
|
- Add additional attributes in the `[@metadata][pubsub_message]` field. Fixes [#7](https://github.com/logstash-plugins/logstash-input-google_pubsub/issues/7)
|
3
7
|
|
data/CONTRIBUTORS
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
The following is a list of people who have contributed ideas, code, bug
|
2
|
+
reports, or in general have helped Logstash along its way.
|
3
|
+
|
4
|
+
Contributors:
|
5
|
+
|
6
|
+
* DeDe Morton (dedemorton)
|
7
|
+
* Eric Johnson (erjohnso)
|
8
|
+
* Google LLC
|
9
|
+
* Jake Landis (jakelandis)
|
10
|
+
* Jakub Sacha (jakubsacha)
|
11
|
+
* James Bloomer (jamesbloomer)
|
12
|
+
* João Duarte (jsvd)
|
13
|
+
* Jordan Sissel (jordansissel)
|
14
|
+
* Joseph Lewis III (josephlewis42)
|
15
|
+
* Pier-Hugues Pellerin (ph)
|
16
|
+
* Scott Brenner (ScottBrenner)
|
17
|
+
* Suyog Rao (suyograo)
|
18
|
+
* Sven Widén (SvenW)
|
19
|
+
|
20
|
+
Note: If you've sent us patches, bug reports, or otherwise contributed to
|
21
|
+
Logstash, and you aren't on the list above and want to be, please let us know
|
22
|
+
and we'll make sure you're here. Contributions from folks like you are what make
|
23
|
+
open source awesome.
|
data/README.md
CHANGED
data/docs/index.asciidoc
CHANGED
@@ -133,6 +133,11 @@ input {
|
|
133
133
|
# outside of GCE, you will need to specify the service account's
|
134
134
|
# JSON key file below.
|
135
135
|
#json_key_file => "/home/erjohnso/pkey.json"
|
136
|
+
|
137
|
+
# Should the plugin attempt to create the subscription on startup?
|
138
|
+
# This is not recommended for security reasons but may be useful in
|
139
|
+
# some cases.
|
140
|
+
#create_subscription => false
|
136
141
|
}
|
137
142
|
}
|
138
143
|
output { stdout { codec => rubydebug } }
|
@@ -185,6 +190,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
|
|
185
190
|
| <<plugins-{type}s-{plugin}-subscription>> |<<string,string>>|Yes
|
186
191
|
| <<plugins-{type}s-{plugin}-topic>> |<<string,string>>|Yes
|
187
192
|
| <<plugins-{type}s-{plugin}-include_metadata>> |<<boolean,boolean>>|No
|
193
|
+
| <<plugins-{type}s-{plugin}-create_subscription>> |<<boolean,boolean>>|No
|
188
194
|
|=======================================================================
|
189
195
|
|
190
196
|
Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
|
@@ -250,6 +256,18 @@ The subscription will be created automatically by the plugin.
|
|
250
256
|
|
251
257
|
If set true, will include the full message data in the `[@metadata][pubsub_message]` field.
|
252
258
|
|
259
|
+
[id="plugins-{type}s-{plugin}-create_subscription"]
|
260
|
+
===== `create_subscription`
|
261
|
+
|
262
|
+
added[1.2.0]
|
263
|
+
|
264
|
+
* Value type is <<boolean,boolean>>
|
265
|
+
* Default value is `false`.
|
266
|
+
|
267
|
+
If true, the plugin will try to create the subscription before publishing.
|
268
|
+
Note: this requires additional permissions to be granted to the client and is _not_
|
269
|
+
recommended for most use-cases.
|
270
|
+
|
253
271
|
[id="plugins-{type}s-{plugin}-common-options"]
|
254
272
|
include::{include_path}/{type}.asciidoc[]
|
255
273
|
|
@@ -0,0 +1,52 @@
|
|
1
|
+
# AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
|
2
|
+
|
3
|
+
require 'jar_dependencies'
|
4
|
+
require_jar('com.google.api.grpc', 'proto-google-iam-v1', '0.1.24')
|
5
|
+
require_jar('io.netty', 'netty-buffer', '4.1.16.Final')
|
6
|
+
require_jar('org.threeten', 'threetenbp', '1.3.3')
|
7
|
+
require_jar('com.google.errorprone', 'error_prone_annotations', '2.0.19')
|
8
|
+
require_jar('com.google.api.grpc', 'grpc-google-cloud-pubsub-v1', '0.1.24')
|
9
|
+
require_jar('io.netty', 'netty-codec-http', '4.1.16.Final')
|
10
|
+
require_jar('io.grpc', 'grpc-protobuf-lite', '1.7.0')
|
11
|
+
require_jar('io.netty', 'netty-common', '4.1.16.Final')
|
12
|
+
require_jar('com.google.auto.value', 'auto-value', '1.2')
|
13
|
+
require_jar('com.google.protobuf', 'protobuf-java-util', '3.4.0')
|
14
|
+
require_jar('com.google.http-client', 'google-http-client-jackson2', '1.23.0')
|
15
|
+
require_jar('com.fasterxml.jackson.core', 'jackson-core', '2.1.3')
|
16
|
+
require_jar('com.google.code.findbugs', 'jsr305', '3.0.2')
|
17
|
+
require_jar('joda-time', 'joda-time', '2.9.2')
|
18
|
+
require_jar('io.netty', 'netty-handler', '4.1.16.Final')
|
19
|
+
require_jar('com.google.http-client', 'google-http-client', '1.23.0')
|
20
|
+
require_jar('commons-logging', 'commons-logging', '1.1.1')
|
21
|
+
require_jar('org.json', 'json', '20160810')
|
22
|
+
require_jar('com.google.cloud', 'google-cloud-core', '1.10.0')
|
23
|
+
require_jar('io.netty', 'netty-codec-http2', '4.1.16.Final')
|
24
|
+
require_jar('com.google.guava', 'guava', '20.0')
|
25
|
+
require_jar('com.google.cloud', 'google-cloud-core-grpc', '1.10.0')
|
26
|
+
require_jar('com.google.protobuf', 'protobuf-java', '3.5.1')
|
27
|
+
require_jar('com.google.api', 'gax', '1.14.0')
|
28
|
+
require_jar('org.apache.httpcomponents', 'httpclient', '4.0.1')
|
29
|
+
require_jar('com.google.api', 'gax-grpc', '1.14.0')
|
30
|
+
require_jar('com.google.auth', 'google-auth-library-credentials', '0.9.0')
|
31
|
+
require_jar('com.google.api', 'api-common', '1.2.0')
|
32
|
+
require_jar('com.google.api.grpc', 'proto-google-common-protos', '1.0.0')
|
33
|
+
require_jar('io.grpc', 'grpc-auth', '1.7.0')
|
34
|
+
require_jar('io.grpc', 'grpc-core', '1.7.0')
|
35
|
+
require_jar('io.netty', 'netty-codec-socks', '4.1.16.Final')
|
36
|
+
require_jar('io.grpc', 'grpc-protobuf', '1.7.0')
|
37
|
+
require_jar('io.grpc', 'grpc-context', '1.7.0')
|
38
|
+
require_jar('io.netty', 'netty-transport', '4.1.16.Final')
|
39
|
+
require_jar('commons-codec', 'commons-codec', '1.3')
|
40
|
+
require_jar('com.google.instrumentation', 'instrumentation-api', '0.4.3')
|
41
|
+
require_jar('io.netty', 'netty-handler-proxy', '4.1.16.Final')
|
42
|
+
require_jar('io.netty', 'netty-resolver', '4.1.16.Final')
|
43
|
+
require_jar('io.netty', 'netty-tcnative-boringssl-static', '2.0.6.Final')
|
44
|
+
require_jar('com.google.code.gson', 'gson', '2.7')
|
45
|
+
require_jar('io.grpc', 'grpc-netty', '1.7.0')
|
46
|
+
require_jar('io.opencensus', 'opencensus-api', '0.6.0')
|
47
|
+
require_jar('org.apache.httpcomponents', 'httpcore', '4.0.1')
|
48
|
+
require_jar('io.netty', 'netty-codec', '4.1.16.Final')
|
49
|
+
require_jar('io.grpc', 'grpc-stub', '1.7.0')
|
50
|
+
require_jar('com.google.cloud', 'google-cloud-pubsub', '0.28.0-beta')
|
51
|
+
require_jar('com.google.api.grpc', 'proto-google-cloud-pubsub-v1', '0.1.24')
|
52
|
+
require_jar('com.google.auth', 'google-auth-library-oauth2-http', '0.9.0')
|
@@ -19,8 +19,8 @@
|
|
19
19
|
require "logstash/inputs/base"
|
20
20
|
require "logstash/namespace"
|
21
21
|
|
22
|
-
|
23
|
-
require
|
22
|
+
require 'java'
|
23
|
+
require 'logstash-input-google_pubsub_jars.rb'
|
24
24
|
|
25
25
|
# This is a https://github.com/elastic/logstash[Logstash] input plugin for
|
26
26
|
# https://cloud.google.com/pubsub/[Google Pub/Sub]. The plugin can subscribe
|
@@ -123,38 +123,70 @@ require "google/api_client"
|
|
123
123
|
# ----------------------------------
|
124
124
|
#
|
125
125
|
# ==== Metadata and Attributes
|
126
|
-
#
|
127
|
-
# The original Pub/Sub message is preserved in the special Logstash
|
126
|
+
#
|
127
|
+
# The original Pub/Sub message is preserved in the special Logstash
|
128
128
|
# `[@metadata][pubsub_message]` field so you can fetch:
|
129
|
-
#
|
129
|
+
#
|
130
130
|
# * Message attributes
|
131
131
|
# * The origiginal base64 data
|
132
132
|
# * Pub/Sub message ID for de-duplication
|
133
133
|
# * Publish time
|
134
|
-
#
|
134
|
+
#
|
135
135
|
# You MUST extract any fields you want in a filter prior to the data being sent
|
136
136
|
# to an output because Logstash deletes `@metadata` fields otherwise.
|
137
|
-
#
|
137
|
+
#
|
138
138
|
# See the PubsubMessage
|
139
139
|
# https://cloud.google.com/pubsub/docs/reference/rest/v1/PubsubMessage[documentation]
|
140
140
|
# for a full description of the fields.
|
141
|
-
#
|
141
|
+
#
|
142
142
|
# Example to get the message ID:
|
143
|
-
#
|
143
|
+
#
|
144
144
|
# [source,ruby]
|
145
|
-
# ----------------------------------
|
145
|
+
# ----------------------------------
|
146
146
|
# input {google_pubsub {...}}
|
147
|
-
#
|
147
|
+
#
|
148
148
|
# filter {
|
149
149
|
# mutate {
|
150
150
|
# add_field => { "messageId" => "%{[@metadata][pubsub_message][messageId]}" }
|
151
151
|
# }
|
152
152
|
# }
|
153
|
-
#
|
153
|
+
#
|
154
154
|
# output {...}
|
155
155
|
# ----------------------------------
|
156
156
|
#
|
157
|
+
|
157
158
|
class LogStash::Inputs::GooglePubSub < LogStash::Inputs::Base
|
159
|
+
class MessageReceiver
|
160
|
+
include com.google.cloud.pubsub.v1.MessageReceiver
|
161
|
+
|
162
|
+
def initialize(&blk)
|
163
|
+
@block = blk
|
164
|
+
end
|
165
|
+
|
166
|
+
def receiveMessage(message, consumer)
|
167
|
+
@block.call(message)
|
168
|
+
consumer.ack()
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
172
|
+
java_import 'com.google.api.core.ApiService$Listener'
|
173
|
+
class SubscriberListener < Listener
|
174
|
+
def initialize(&blk)
|
175
|
+
@block = blk
|
176
|
+
end
|
177
|
+
|
178
|
+
def failed(from, failure)
|
179
|
+
@block.call(from, failure)
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
183
|
+
include_package 'com.google.api.gax.batching'
|
184
|
+
include_package 'com.google.api.gax.core'
|
185
|
+
include_package 'com.google.auth.oauth2'
|
186
|
+
include_package 'com.google.common.util.concurrent'
|
187
|
+
include_package 'com.google.cloud.pubsub.v1'
|
188
|
+
include_package 'com.google.pubsub.v1'
|
189
|
+
include_package 'com.google.protobuf.util'
|
158
190
|
config_name "google_pubsub"
|
159
191
|
|
160
192
|
# Google Cloud Project ID (name, not number)
|
@@ -176,140 +208,81 @@ class LogStash::Inputs::GooglePubSub < LogStash::Inputs::Base
|
|
176
208
|
# If set true, will include the full message data in the `[@metadata][pubsub_message]` field.
|
177
209
|
config :include_metadata, :validate => :boolean, :required => false, :default => false
|
178
210
|
|
211
|
+
# If true, the plugin will try to create the subscription before publishing.
|
212
|
+
# Note: this requires additional permissions to be granted to the client and is _not_
|
213
|
+
# recommended for most use-cases.
|
214
|
+
config :create_subscription, :validate => :boolean, :required => false, :default => false
|
215
|
+
|
179
216
|
# If undefined, Logstash will complain, even if codec is unused.
|
180
217
|
default :codec, "plain"
|
181
218
|
|
182
|
-
private
|
183
|
-
def request(options)
|
184
|
-
begin
|
185
|
-
@logger.debug("Sending an API request")
|
186
|
-
result = @client.execute(options)
|
187
|
-
rescue ArgumentError => e
|
188
|
-
@logger.debug("Authorizing...")
|
189
|
-
@client.authorization.fetch_access_token!
|
190
|
-
@logger.debug("...authorized")
|
191
|
-
request(options)
|
192
|
-
rescue Faraday::TimeoutError => e
|
193
|
-
@logger.debug("Request timeout, re-trying request")
|
194
|
-
request(options)
|
195
|
-
end
|
196
|
-
end # def request
|
197
|
-
|
198
219
|
public
|
199
220
|
def register
|
200
221
|
@logger.debug("Registering Google PubSub Input: project_id=#{@project_id}, topic=#{@topic}, subscription=#{@subscription}")
|
201
|
-
@
|
202
|
-
@subscription = "projects/#{@project_id}/subscriptions/#{@subscription}"
|
203
|
-
@subscription_exists = false
|
204
|
-
|
205
|
-
# TODO(erjohnso): read UA data from the gemspec
|
206
|
-
@client = Google::APIClient.new(
|
207
|
-
:application_name => 'logstash-input-google_pubsub',
|
208
|
-
:application_version => '0.9.0'
|
209
|
-
)
|
222
|
+
@subscription_id = "projects/#{@project_id}/subscriptions/#{@subscription}"
|
210
223
|
|
211
|
-
# Initialize the pubsub API client
|
212
|
-
@pubsub = @client.discovered_api('pubsub', 'v1')
|
213
|
-
|
214
|
-
# Handle various kinds of auth (JSON or Application Default Creds)
|
215
|
-
# NOTE: Cannot use 'googleauth' gem since there are dependency conflicts
|
216
|
-
# - googleauth ~> 0.5 requires mime-data-types that requires ruby2
|
217
|
-
# - googleauth ~> 0.3 requires multi_json 1.11.0 that conflicts
|
218
|
-
# with logstash-2.3.2's multi_json 1.11.3
|
219
224
|
if @json_key_file
|
220
|
-
@
|
221
|
-
|
222
|
-
key_json = File.open(file_path, "r", &:read)
|
223
|
-
key_json = JSON.parse(key_json)
|
224
|
-
unless key_json.key?("client_email") || key_json.key?("private_key")
|
225
|
-
raise Google::APIClient::ClientError, "Invalid JSON credentials data."
|
226
|
-
end
|
227
|
-
signing_key = ::Google::APIClient::KeyUtils.load_from_pem(key_json["private_key"], "notasecret")
|
228
|
-
@client.authorization = Signet::OAuth2::Client.new(
|
229
|
-
:audience => "https://accounts.google.com/o/oauth2/token",
|
230
|
-
:auth_provider_x509_cert_url => "https://www.googleapis.com/oauth2/v1/certs",
|
231
|
-
:client_x509_cert_url => "https://www.googleapis.com/robot/v1/metadata/x509/#{key_json['client_email']}",
|
232
|
-
:issuer => "#{key_json['client_email']}",
|
233
|
-
:scope => %w(https://www.googleapis.com/auth/cloud-platform),
|
234
|
-
:signing_key => signing_key,
|
235
|
-
:token_credential_uri => "https://accounts.google.com/o/oauth2/token"
|
225
|
+
@credentialsProvider = FixedCredentialsProvider.create(
|
226
|
+
ServiceAccountCredentials.fromStream(java.io.FileInputStream.new(@json_key_file))
|
236
227
|
)
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
end # def register
|
228
|
+
end
|
229
|
+
@topic_name = TopicName.create(@project_id, @topic)
|
230
|
+
@subscription_name = SubscriptionName.create(@project_id, @subscription)
|
231
|
+
end
|
232
|
+
|
233
|
+
def stop
|
234
|
+
@subscriber.stopAsync().awaitTerminated() if @subscriber != nil
|
235
|
+
end
|
246
236
|
|
247
237
|
def run(queue)
|
248
238
|
# Attempt to create the subscription
|
249
|
-
if
|
250
|
-
@logger.debug("Creating subscription #{
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
:ackDeadlineSeconds => 15
|
257
|
-
}
|
258
|
-
)
|
259
|
-
if result.error? and result.status != 409
|
260
|
-
raise Google::APIClient::ClientError, "Error #{result.status}: #{result.error_message}"
|
239
|
+
if @create_subscription
|
240
|
+
@logger.debug("Creating subscription #{@subscription_id}")
|
241
|
+
subscriptionAdminClient = SubscriptionAdminClient.create
|
242
|
+
begin
|
243
|
+
subscriptionAdminClient.createSubscription(@subscription_name, @topic_name, PushConfig.getDefaultInstance(), 0)
|
244
|
+
rescue
|
245
|
+
@logger.info("Subscription already exists")
|
261
246
|
end
|
262
|
-
|
263
|
-
end # if !@subscription
|
264
|
-
|
265
|
-
@logger.debug("Pulling messages from sub '#{subscription}'")
|
266
|
-
while !stop?
|
267
|
-
# Pull and queue messages
|
268
|
-
messages = []
|
269
|
-
result = request(
|
270
|
-
:api_method => @pubsub.projects.subscriptions.pull,
|
271
|
-
:parameters => {'subscription' => @subscription},
|
272
|
-
:body_object => {
|
273
|
-
:returnImmediately => false,
|
274
|
-
:maxMessages => @max_messages
|
275
|
-
}
|
276
|
-
)
|
247
|
+
end
|
277
248
|
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
249
|
+
@logger.debug("Pulling messages from sub '#{@subscription_id}'")
|
250
|
+
handler = MessageReceiver.new do |message|
|
251
|
+
# handle incoming message, then ack/nack the received message
|
252
|
+
data = message.getData().toStringUtf8()
|
253
|
+
@codec.decode(data) do |event|
|
254
|
+
event.set("host", event.get("host") || @host)
|
255
|
+
event.set("[@metadata][pubsub_message]", extract_metadata(message)) if @include_metadata
|
256
|
+
decorate(event)
|
257
|
+
queue << event
|
285
258
|
end
|
259
|
+
end
|
260
|
+
listener = SubscriberListener.new do |from, failure|
|
261
|
+
@logger.error("#{failure}")
|
262
|
+
raise failure
|
263
|
+
end
|
264
|
+
flowControlSettings = FlowControlSettings.newBuilder().setMaxOutstandingElementCount(@max_messages).build()
|
265
|
+
executorProvider = InstantiatingExecutorProvider.newBuilder().setExecutorThreadCount(1).build()
|
266
|
+
subscriberBuilder = Subscriber.newBuilder(@subscription_name, handler)
|
267
|
+
.setFlowControlSettings(flowControlSettings)
|
268
|
+
.setExecutorProvider(executorProvider)
|
269
|
+
.setParallelPullCount(1)
|
286
270
|
|
287
|
-
|
288
|
-
|
289
|
-
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
end
|
296
|
-
end
|
297
|
-
end
|
298
|
-
|
299
|
-
ack_ids = messages.map{ |msg| msg["ackId"] }
|
300
|
-
next if ack_ids.empty?
|
271
|
+
if @credentialsProvider
|
272
|
+
subscriberBuilder.setCredentialsProvider(@credentialsProvider)
|
273
|
+
end
|
274
|
+
@subscriber = subscriberBuilder.build()
|
275
|
+
@subscriber.addListener(listener, MoreExecutors.directExecutor())
|
276
|
+
@subscriber.startAsync()
|
277
|
+
@subscriber.awaitTerminated()
|
278
|
+
end
|
301
279
|
|
302
|
-
|
303
|
-
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
end
|
312
|
-
end # if messages
|
313
|
-
end # loop
|
314
|
-
end # def run
|
315
|
-
end # class LogStash::Inputs::GooglePubSub
|
280
|
+
def extract_metadata(java_message)
|
281
|
+
{
|
282
|
+
data: java_message.getData().toStringUtf8(),
|
283
|
+
attributes: java_message.getAttributesMap(),
|
284
|
+
messageId: java_message.getMessageId(),
|
285
|
+
publishTime: Timestamps.toString(java_message.getPublishTime())
|
286
|
+
}
|
287
|
+
end
|
288
|
+
end
|
@@ -1,13 +1,13 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-input-google_pubsub'
|
3
|
-
s.version = '1.
|
3
|
+
s.version = '1.2.0'
|
4
4
|
s.licenses = ['Apache-2.0']
|
5
5
|
s.summary = "Consume events from a Google Cloud PubSub service"
|
6
6
|
s.description = "This gem is a Logstash input plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program."
|
7
7
|
s.authors = ["Eric Johnson"]
|
8
8
|
s.email = 'erjohnso@google.com'
|
9
9
|
s.homepage = "https://cloud.google.com/pubsub/overview"
|
10
|
-
s.require_paths = ["lib"]
|
10
|
+
s.require_paths = ["lib", "vendor/jar-dependencies"]
|
11
11
|
|
12
12
|
# Files
|
13
13
|
s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
|
@@ -22,8 +22,13 @@ Gem::Specification.new do |s|
|
|
22
22
|
s.add_runtime_dependency 'logstash-codec-plain'
|
23
23
|
s.add_runtime_dependency 'stud', '>= 0.0.22'
|
24
24
|
# Google dependencies
|
25
|
-
# google-api-client >= 0.9 requires ruby2 which is not currently compatible
|
26
25
|
# with JRuby
|
27
|
-
s.
|
26
|
+
s.requirements << "jar 'com.google.cloud:google-cloud-pubsub', '0.28.0-beta'"
|
27
|
+
s.requirements << "jar 'com.google.api.grpc:proto-google-cloud-pubsub-v1', '0.1.24'"
|
28
|
+
s.requirements << "jar 'com.google.api:gax', '1.14.0'"
|
29
|
+
s.requirements << "jar 'com.google.guava:guava', '20.0'"
|
30
|
+
s.requirements << "jar 'com.google.api:api-common', '1.2.0'"
|
31
|
+
s.requirements << "jar 'com.google.auth:google-auth-library-oauth2-http', '0.9.0'"
|
28
32
|
s.add_development_dependency 'logstash-devutils'
|
33
|
+
s.add_development_dependency 'jar-dependencies', '~> 0.3.2'
|
29
34
|
end
|
data/vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-core/2.1.3/jackson-core-2.1.3.jar
ADDED
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
data/vendor/jar-dependencies/com/google/cloud/google-cloud-core/1.10.0/google-cloud-core-1.10.0.jar
ADDED
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
data/vendor/jar-dependencies/io/netty/netty-handler/4.1.16.Final/netty-handler-4.1.16.Final.jar
ADDED
Binary file
|
data/vendor/jar-dependencies/io/netty/netty-resolver/4.1.16.Final/netty-resolver-4.1.16.Final.jar
ADDED
Binary file
|
Binary file
|
data/vendor/jar-dependencies/io/netty/netty-transport/4.1.16.Final/netty-transport-4.1.16.Final.jar
ADDED
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-google_pubsub
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Eric Johnson
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-05-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -61,37 +61,31 @@ dependencies:
|
|
61
61
|
- !ruby/object:Gem::Dependency
|
62
62
|
requirement: !ruby/object:Gem::Requirement
|
63
63
|
requirements:
|
64
|
-
- - "
|
65
|
-
- !ruby/object:Gem::Version
|
66
|
-
version: 0.8.6
|
67
|
-
- - "<"
|
64
|
+
- - ">="
|
68
65
|
- !ruby/object:Gem::Version
|
69
|
-
version: '0
|
70
|
-
name:
|
66
|
+
version: '0'
|
67
|
+
name: logstash-devutils
|
71
68
|
prerelease: false
|
72
|
-
type: :
|
69
|
+
type: :development
|
73
70
|
version_requirements: !ruby/object:Gem::Requirement
|
74
71
|
requirements:
|
75
|
-
- - "
|
76
|
-
- !ruby/object:Gem::Version
|
77
|
-
version: 0.8.6
|
78
|
-
- - "<"
|
72
|
+
- - ">="
|
79
73
|
- !ruby/object:Gem::Version
|
80
|
-
version: '0
|
74
|
+
version: '0'
|
81
75
|
- !ruby/object:Gem::Dependency
|
82
76
|
requirement: !ruby/object:Gem::Requirement
|
83
77
|
requirements:
|
84
|
-
- - "
|
78
|
+
- - "~>"
|
85
79
|
- !ruby/object:Gem::Version
|
86
|
-
version:
|
87
|
-
name:
|
80
|
+
version: 0.3.2
|
81
|
+
name: jar-dependencies
|
88
82
|
prerelease: false
|
89
83
|
type: :development
|
90
84
|
version_requirements: !ruby/object:Gem::Requirement
|
91
85
|
requirements:
|
92
|
-
- - "
|
86
|
+
- - "~>"
|
93
87
|
- !ruby/object:Gem::Version
|
94
|
-
version:
|
88
|
+
version: 0.3.2
|
95
89
|
description: This gem is a Logstash input plugin required to be installed on top of
|
96
90
|
the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
|
97
91
|
gem is not a stand-alone program.
|
@@ -102,14 +96,65 @@ extra_rdoc_files: []
|
|
102
96
|
files:
|
103
97
|
- CHANGELOG.md
|
104
98
|
- CONTRIBUTING.md
|
99
|
+
- CONTRIBUTORS
|
105
100
|
- Gemfile
|
106
101
|
- LICENSE
|
107
102
|
- NOTICE.TXT
|
108
103
|
- README.md
|
109
104
|
- docs/index.asciidoc
|
105
|
+
- lib/logstash-input-google_pubsub_jars.rb
|
110
106
|
- lib/logstash/inputs/google_pubsub.rb
|
111
107
|
- logstash-input-google_pubsub.gemspec
|
112
108
|
- spec/inputs/google_pubsub_spec.rb
|
109
|
+
- vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-core/2.1.3/jackson-core-2.1.3.jar
|
110
|
+
- vendor/jar-dependencies/com/google/api/api-common/1.2.0/api-common-1.2.0.jar
|
111
|
+
- vendor/jar-dependencies/com/google/api/gax-grpc/1.14.0/gax-grpc-1.14.0.jar
|
112
|
+
- vendor/jar-dependencies/com/google/api/gax/1.14.0/gax-1.14.0.jar
|
113
|
+
- vendor/jar-dependencies/com/google/api/grpc/grpc-google-cloud-pubsub-v1/0.1.24/grpc-google-cloud-pubsub-v1-0.1.24.jar
|
114
|
+
- vendor/jar-dependencies/com/google/api/grpc/proto-google-cloud-pubsub-v1/0.1.24/proto-google-cloud-pubsub-v1-0.1.24.jar
|
115
|
+
- vendor/jar-dependencies/com/google/api/grpc/proto-google-common-protos/1.0.0/proto-google-common-protos-1.0.0.jar
|
116
|
+
- vendor/jar-dependencies/com/google/api/grpc/proto-google-iam-v1/0.1.24/proto-google-iam-v1-0.1.24.jar
|
117
|
+
- vendor/jar-dependencies/com/google/auth/google-auth-library-credentials/0.9.0/google-auth-library-credentials-0.9.0.jar
|
118
|
+
- vendor/jar-dependencies/com/google/auth/google-auth-library-oauth2-http/0.9.0/google-auth-library-oauth2-http-0.9.0.jar
|
119
|
+
- vendor/jar-dependencies/com/google/auto/value/auto-value/1.2/auto-value-1.2.jar
|
120
|
+
- vendor/jar-dependencies/com/google/cloud/google-cloud-core-grpc/1.10.0/google-cloud-core-grpc-1.10.0.jar
|
121
|
+
- vendor/jar-dependencies/com/google/cloud/google-cloud-core/1.10.0/google-cloud-core-1.10.0.jar
|
122
|
+
- vendor/jar-dependencies/com/google/cloud/google-cloud-pubsub/0.28.0-beta/google-cloud-pubsub-0.28.0-beta.jar
|
123
|
+
- vendor/jar-dependencies/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar
|
124
|
+
- vendor/jar-dependencies/com/google/code/gson/gson/2.7/gson-2.7.jar
|
125
|
+
- vendor/jar-dependencies/com/google/errorprone/error_prone_annotations/2.0.19/error_prone_annotations-2.0.19.jar
|
126
|
+
- vendor/jar-dependencies/com/google/guava/guava/20.0/guava-20.0.jar
|
127
|
+
- vendor/jar-dependencies/com/google/http-client/google-http-client-jackson2/1.23.0/google-http-client-jackson2-1.23.0.jar
|
128
|
+
- vendor/jar-dependencies/com/google/http-client/google-http-client/1.23.0/google-http-client-1.23.0.jar
|
129
|
+
- vendor/jar-dependencies/com/google/instrumentation/instrumentation-api/0.4.3/instrumentation-api-0.4.3.jar
|
130
|
+
- vendor/jar-dependencies/com/google/protobuf/protobuf-java-util/3.4.0/protobuf-java-util-3.4.0.jar
|
131
|
+
- vendor/jar-dependencies/com/google/protobuf/protobuf-java/3.5.1/protobuf-java-3.5.1.jar
|
132
|
+
- vendor/jar-dependencies/commons-codec/commons-codec/1.3/commons-codec-1.3.jar
|
133
|
+
- vendor/jar-dependencies/commons-logging/commons-logging/1.1.1/commons-logging-1.1.1.jar
|
134
|
+
- vendor/jar-dependencies/io/grpc/grpc-auth/1.7.0/grpc-auth-1.7.0.jar
|
135
|
+
- vendor/jar-dependencies/io/grpc/grpc-context/1.7.0/grpc-context-1.7.0.jar
|
136
|
+
- vendor/jar-dependencies/io/grpc/grpc-core/1.7.0/grpc-core-1.7.0.jar
|
137
|
+
- vendor/jar-dependencies/io/grpc/grpc-netty/1.7.0/grpc-netty-1.7.0.jar
|
138
|
+
- vendor/jar-dependencies/io/grpc/grpc-protobuf-lite/1.7.0/grpc-protobuf-lite-1.7.0.jar
|
139
|
+
- vendor/jar-dependencies/io/grpc/grpc-protobuf/1.7.0/grpc-protobuf-1.7.0.jar
|
140
|
+
- vendor/jar-dependencies/io/grpc/grpc-stub/1.7.0/grpc-stub-1.7.0.jar
|
141
|
+
- vendor/jar-dependencies/io/netty/netty-buffer/4.1.16.Final/netty-buffer-4.1.16.Final.jar
|
142
|
+
- vendor/jar-dependencies/io/netty/netty-codec-http/4.1.16.Final/netty-codec-http-4.1.16.Final.jar
|
143
|
+
- vendor/jar-dependencies/io/netty/netty-codec-http2/4.1.16.Final/netty-codec-http2-4.1.16.Final.jar
|
144
|
+
- vendor/jar-dependencies/io/netty/netty-codec-socks/4.1.16.Final/netty-codec-socks-4.1.16.Final.jar
|
145
|
+
- vendor/jar-dependencies/io/netty/netty-codec/4.1.16.Final/netty-codec-4.1.16.Final.jar
|
146
|
+
- vendor/jar-dependencies/io/netty/netty-common/4.1.16.Final/netty-common-4.1.16.Final.jar
|
147
|
+
- vendor/jar-dependencies/io/netty/netty-handler-proxy/4.1.16.Final/netty-handler-proxy-4.1.16.Final.jar
|
148
|
+
- vendor/jar-dependencies/io/netty/netty-handler/4.1.16.Final/netty-handler-4.1.16.Final.jar
|
149
|
+
- vendor/jar-dependencies/io/netty/netty-resolver/4.1.16.Final/netty-resolver-4.1.16.Final.jar
|
150
|
+
- vendor/jar-dependencies/io/netty/netty-tcnative-boringssl-static/2.0.6.Final/netty-tcnative-boringssl-static-2.0.6.Final.jar
|
151
|
+
- vendor/jar-dependencies/io/netty/netty-transport/4.1.16.Final/netty-transport-4.1.16.Final.jar
|
152
|
+
- vendor/jar-dependencies/io/opencensus/opencensus-api/0.6.0/opencensus-api-0.6.0.jar
|
153
|
+
- vendor/jar-dependencies/joda-time/joda-time/2.9.2/joda-time-2.9.2.jar
|
154
|
+
- vendor/jar-dependencies/org/apache/httpcomponents/httpclient/4.0.1/httpclient-4.0.1.jar
|
155
|
+
- vendor/jar-dependencies/org/apache/httpcomponents/httpcore/4.0.1/httpcore-4.0.1.jar
|
156
|
+
- vendor/jar-dependencies/org/json/json/20160810/json-20160810.jar
|
157
|
+
- vendor/jar-dependencies/org/threeten/threetenbp/1.3.3/threetenbp-1.3.3.jar
|
113
158
|
homepage: https://cloud.google.com/pubsub/overview
|
114
159
|
licenses:
|
115
160
|
- Apache-2.0
|
@@ -120,6 +165,7 @@ post_install_message:
|
|
120
165
|
rdoc_options: []
|
121
166
|
require_paths:
|
122
167
|
- lib
|
168
|
+
- vendor/jar-dependencies
|
123
169
|
required_ruby_version: !ruby/object:Gem::Requirement
|
124
170
|
requirements:
|
125
171
|
- - ">="
|
@@ -130,7 +176,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
130
176
|
- - ">="
|
131
177
|
- !ruby/object:Gem::Version
|
132
178
|
version: '0'
|
133
|
-
requirements:
|
179
|
+
requirements:
|
180
|
+
- jar 'com.google.cloud:google-cloud-pubsub', '0.28.0-beta'
|
181
|
+
- jar 'com.google.api.grpc:proto-google-cloud-pubsub-v1', '0.1.24'
|
182
|
+
- jar 'com.google.api:gax', '1.14.0'
|
183
|
+
- jar 'com.google.guava:guava', '20.0'
|
184
|
+
- jar 'com.google.api:api-common', '1.2.0'
|
185
|
+
- jar 'com.google.auth:google-auth-library-oauth2-http', '0.9.0'
|
134
186
|
rubyforge_project:
|
135
187
|
rubygems_version: 2.6.13
|
136
188
|
signing_key:
|