google-cloud-pubsub 1.0.2 → 2.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AUTHENTICATION.md +16 -54
- data/CHANGELOG.md +464 -0
- data/CONTRIBUTING.md +328 -116
- data/EMULATOR.md +1 -1
- data/LOGGING.md +94 -2
- data/OVERVIEW.md +121 -68
- data/TROUBLESHOOTING.md +2 -8
- data/lib/google/cloud/pubsub/acknowledge_result.rb +79 -0
- data/lib/google/cloud/pubsub/async_publisher/batch.rb +319 -0
- data/lib/google/cloud/pubsub/async_publisher.rb +231 -156
- data/lib/google/cloud/pubsub/batch_publisher.rb +60 -30
- data/lib/google/cloud/pubsub/convert.rb +33 -7
- data/lib/google/cloud/pubsub/credentials.rb +2 -2
- data/lib/google/cloud/pubsub/errors.rb +93 -0
- data/lib/google/cloud/pubsub/flow_controller.rb +137 -0
- data/lib/google/cloud/pubsub/message.rb +45 -4
- data/lib/google/cloud/pubsub/policy.rb +3 -2
- data/lib/google/cloud/pubsub/project.rb +316 -49
- data/lib/google/cloud/pubsub/publish_result.rb +6 -1
- data/lib/google/cloud/pubsub/received_message.rb +171 -10
- data/lib/google/cloud/pubsub/retry_policy.rb +88 -0
- data/lib/google/cloud/pubsub/schema/list.rb +180 -0
- data/lib/google/cloud/pubsub/schema.rb +310 -0
- data/lib/google/cloud/pubsub/service.rb +285 -269
- data/lib/google/cloud/pubsub/snapshot/list.rb +4 -6
- data/lib/google/cloud/pubsub/snapshot.rb +5 -2
- data/lib/google/cloud/pubsub/subscriber/inventory.rb +69 -32
- data/lib/google/cloud/pubsub/subscriber/sequencer.rb +115 -0
- data/lib/google/cloud/pubsub/subscriber/stream.rb +108 -49
- data/lib/google/cloud/pubsub/subscriber/timed_unary_buffer.rb +191 -30
- data/lib/google/cloud/pubsub/subscriber.rb +155 -45
- data/lib/google/cloud/pubsub/subscription/list.rb +4 -6
- data/lib/google/cloud/pubsub/subscription/push_config.rb +55 -31
- data/lib/google/cloud/pubsub/subscription.rb +561 -77
- data/lib/google/cloud/pubsub/topic/list.rb +4 -6
- data/lib/google/cloud/pubsub/topic.rb +372 -52
- data/lib/google/cloud/pubsub/version.rb +1 -1
- data/lib/google/cloud/pubsub.rb +35 -46
- data/lib/google-cloud-pubsub.rb +21 -27
- metadata +26 -189
- data/lib/google/cloud/pubsub/v1/credentials.rb +0 -41
- data/lib/google/cloud/pubsub/v1/doc/google/iam/v1/iam_policy.rb +0 -21
- data/lib/google/cloud/pubsub/v1/doc/google/iam/v1/options.rb +0 -21
- data/lib/google/cloud/pubsub/v1/doc/google/iam/v1/policy.rb +0 -21
- data/lib/google/cloud/pubsub/v1/doc/google/protobuf/duration.rb +0 -91
- data/lib/google/cloud/pubsub/v1/doc/google/protobuf/empty.rb +0 -29
- data/lib/google/cloud/pubsub/v1/doc/google/protobuf/field_mask.rb +0 -222
- data/lib/google/cloud/pubsub/v1/doc/google/protobuf/timestamp.rb +0 -113
- data/lib/google/cloud/pubsub/v1/doc/google/pubsub/v1/pubsub.rb +0 -744
- data/lib/google/cloud/pubsub/v1/doc/google/type/expr.rb +0 -19
- data/lib/google/cloud/pubsub/v1/publisher_client.rb +0 -786
- data/lib/google/cloud/pubsub/v1/publisher_client_config.json +0 -105
- data/lib/google/cloud/pubsub/v1/subscriber_client.rb +0 -1385
- data/lib/google/cloud/pubsub/v1/subscriber_client_config.json +0 -138
- data/lib/google/cloud/pubsub/v1.rb +0 -17
- data/lib/google/pubsub/v1/pubsub_pb.rb +0 -249
- data/lib/google/pubsub/v1/pubsub_services_pb.rb +0 -211
data/OVERVIEW.md
CHANGED
@@ -8,11 +8,11 @@ developers to communicate between independently written applications.
|
|
8
8
|
|
9
9
|
The goal of google-cloud is to provide an API that is comfortable to Rubyists.
|
10
10
|
Your authentication credentials are detected automatically in Google Cloud
|
11
|
-
Platform
|
12
|
-
Google
|
13
|
-
|
14
|
-
|
15
|
-
Guide}.
|
11
|
+
Platform (GCP), including Google Compute Engine (GCE), Google Kubernetes Engine
|
12
|
+
(GKE), Google App Engine (GAE), Google Cloud Functions (GCF) and Cloud Run. In
|
13
|
+
other environments you can configure authentication easily, either directly in
|
14
|
+
your code or via environment variables. Read more about the options for
|
15
|
+
connecting in the {file:AUTHENTICATION.md Authentication Guide}.
|
16
16
|
|
17
17
|
```ruby
|
18
18
|
require "google/cloud/pubsub"
|
@@ -142,7 +142,7 @@ topic.publish_async "task completed" do |result|
|
|
142
142
|
end
|
143
143
|
end
|
144
144
|
|
145
|
-
topic.async_publisher.stop
|
145
|
+
topic.async_publisher.stop!
|
146
146
|
```
|
147
147
|
|
148
148
|
Or multiple messages can be published in batches at the same time by passing a
|
@@ -161,7 +161,7 @@ msgs = topic.publish do |batch|
|
|
161
161
|
end
|
162
162
|
```
|
163
163
|
|
164
|
-
## Receiving
|
164
|
+
## Receiving Messages
|
165
165
|
|
166
166
|
Messages can be streamed from a subscription with a subscriber object that is
|
167
167
|
created using `listen`. (See {Google::Cloud::PubSub::Subscription#listen
|
@@ -174,28 +174,47 @@ pubsub = Google::Cloud::PubSub.new
|
|
174
174
|
|
175
175
|
sub = pubsub.subscription "my-topic-sub"
|
176
176
|
|
177
|
-
subscriber
|
177
|
+
# Create a subscriber to listen for available messages.
|
178
|
+
# By default, this block will be called on 8 concurrent threads
|
179
|
+
# but this can be tuned with the `threads` option.
|
180
|
+
# The `streams` and `inventory` parameters allow further tuning.
|
181
|
+
subscriber = sub.listen threads: { callback: 16 } do |received_message|
|
178
182
|
# process message
|
183
|
+
puts "Data: #{received_message.message.data}, published at #{received_message.message.published_at}"
|
179
184
|
received_message.acknowledge!
|
180
185
|
end
|
181
186
|
|
187
|
+
# Handle exceptions from listener
|
188
|
+
subscriber.on_error do |exception|
|
189
|
+
puts "Exception: #{exception.class} #{exception.message}"
|
190
|
+
end
|
191
|
+
|
192
|
+
# Gracefully shut down the subscriber on program exit, blocking until
|
193
|
+
# all received messages have been processed or 10 seconds have passed
|
194
|
+
at_exit do
|
195
|
+
subscriber.stop!(10)
|
196
|
+
end
|
197
|
+
|
182
198
|
# Start background threads that will call the block passed to listen.
|
183
199
|
subscriber.start
|
184
200
|
|
185
|
-
#
|
186
|
-
|
201
|
+
# Block, letting processing threads continue in the background
|
202
|
+
sleep
|
187
203
|
```
|
188
204
|
|
189
205
|
Messages also can be pulled directly in a one-time operation. (See
|
190
206
|
{Google::Cloud::PubSub::Subscription#pull Subscription#pull})
|
191
207
|
|
208
|
+
The `immediate: false` option is recommended to avoid adverse impacts on the
|
209
|
+
performance of pull operations.
|
210
|
+
|
192
211
|
```ruby
|
193
212
|
require "google/cloud/pubsub"
|
194
213
|
|
195
214
|
pubsub = Google::Cloud::PubSub.new
|
196
215
|
|
197
216
|
sub = pubsub.subscription "my-topic-sub"
|
198
|
-
received_messages = sub.pull
|
217
|
+
received_messages = sub.pull immediate: false
|
199
218
|
```
|
200
219
|
|
201
220
|
A maximum number of messages to pull can be specified:
|
@@ -206,7 +225,7 @@ require "google/cloud/pubsub"
|
|
206
225
|
pubsub = Google::Cloud::PubSub.new
|
207
226
|
|
208
227
|
sub = pubsub.subscription "my-topic-sub"
|
209
|
-
received_messages = sub.pull max: 10
|
228
|
+
received_messages = sub.pull immediate: false, max: 10
|
210
229
|
```
|
211
230
|
|
212
231
|
## Acknowledging a Message
|
@@ -235,7 +254,7 @@ end
|
|
235
254
|
subscriber.start
|
236
255
|
|
237
256
|
# Shut down the subscriber when ready to stop receiving messages.
|
238
|
-
subscriber.stop
|
257
|
+
subscriber.stop!
|
239
258
|
```
|
240
259
|
|
241
260
|
Or, multiple messages can be acknowledged in a single API call: (See
|
@@ -247,7 +266,7 @@ require "google/cloud/pubsub"
|
|
247
266
|
pubsub = Google::Cloud::PubSub.new
|
248
267
|
|
249
268
|
sub = pubsub.subscription "my-topic-sub"
|
250
|
-
received_messages = sub.pull
|
269
|
+
received_messages = sub.pull immediate: false
|
251
270
|
sub.acknowledge received_messages
|
252
271
|
```
|
253
272
|
|
@@ -277,7 +296,7 @@ end
|
|
277
296
|
subscriber.start
|
278
297
|
|
279
298
|
# Shut down the subscriber when ready to stop receiving messages.
|
280
|
-
subscriber.stop
|
299
|
+
subscriber.stop!
|
281
300
|
```
|
282
301
|
|
283
302
|
The message can also be made available for immediate redelivery:
|
@@ -299,7 +318,7 @@ end
|
|
299
318
|
subscriber.start
|
300
319
|
|
301
320
|
# Shut down the subscriber when ready to stop receiving messages.
|
302
|
-
subscriber.stop
|
321
|
+
subscriber.stop!
|
303
322
|
```
|
304
323
|
|
305
324
|
Multiple messages can be delayed or made available for immediate redelivery:
|
@@ -312,10 +331,92 @@ require "google/cloud/pubsub"
|
|
312
331
|
pubsub = Google::Cloud::PubSub.new
|
313
332
|
|
314
333
|
sub = pubsub.subscription "my-topic-sub"
|
315
|
-
received_messages = sub.pull
|
334
|
+
received_messages = sub.pull immediate: false
|
316
335
|
sub.modify_ack_deadline 120, received_messages
|
317
336
|
```
|
318
337
|
|
338
|
+
## Using Ordering Keys
|
339
|
+
|
340
|
+
Google Cloud Pub/Sub ordering keys provide the ability to ensure related
|
341
|
+
messages are sent to subscribers in the order in which they were published.
|
342
|
+
Messages can be tagged with an ordering key, a string that identifies related
|
343
|
+
messages for which publish order should be respected. The service guarantees
|
344
|
+
that, for a given ordering key and publisher, messages are sent to subscribers
|
345
|
+
in the order in which they were published. Ordering does not require sacrificing
|
346
|
+
high throughput or scalability, as the service automatically distributes
|
347
|
+
messages for different ordering keys across subscribers.
|
348
|
+
|
349
|
+
Note: At the time of this release, ordering keys are not yet publicly enabled
|
350
|
+
and requires special project enablements.
|
351
|
+
|
352
|
+
### Publishing Ordered Messages
|
353
|
+
|
354
|
+
To use ordering keys when publishing messages, a call to
|
355
|
+
{Google::Cloud::PubSub::Topic#enable_message_ordering!
|
356
|
+
Topic#enable_message_ordering!} must be made and the `ordering_key` argument
|
357
|
+
must be provided when calling {Google::Cloud::PubSub::Topic#publish_async
|
358
|
+
Topic#publish_async}.
|
359
|
+
|
360
|
+
```ruby
|
361
|
+
require "google/cloud/pubsub"
|
362
|
+
|
363
|
+
pubsub = Google::Cloud::PubSub.new
|
364
|
+
|
365
|
+
topic = pubsub.topic "my-ordered-topic"
|
366
|
+
|
367
|
+
# Ensure that message ordering is enabled.
|
368
|
+
topic.enable_message_ordering!
|
369
|
+
|
370
|
+
# Publish an ordered message with an ordering key.
|
371
|
+
topic.publish_async "task completed",
|
372
|
+
ordering_key: "task-key"
|
373
|
+
|
374
|
+
# Shut down the publisher when ready to stop publishing messages.
|
375
|
+
topic.async_publisher.stop!
|
376
|
+
```
|
377
|
+
|
378
|
+
### Handling errors with Ordered Keys
|
379
|
+
|
380
|
+
Ordered messages that fail to publish to the Pub/Sub API due to error will put
|
381
|
+
the `ordering_key` in a failed state, and future calls to
|
382
|
+
{Google::Cloud::PubSub::Topic#publish_async Topic#publish_async} with the
|
383
|
+
`ordering_key` will raise {Google::Cloud::PubSub::OrderingKeyError
|
384
|
+
OrderingKeyError}. To allow future messages with the `ordering_key` to be
|
385
|
+
published, the `ordering_key` must be passed to
|
386
|
+
{Google::Cloud::PubSub::Topic#resume_publish Topic#resume_publish}.
|
387
|
+
|
388
|
+
### Receiving Ordered Messages
|
389
|
+
|
390
|
+
To use ordering keys when subscribing to messages, the subscription must be
|
391
|
+
created with message ordering enabled (See
|
392
|
+
{Google::Cloud::PubSub::Topic#subscribe Topic#subscribe} and
|
393
|
+
{Google::Cloud::PubSub::Subscription#message_ordering?
|
394
|
+
Subscription#message_ordering?}) before calling
|
395
|
+
{Google::Cloud::PubSub::Subscription#listen Subscription#listen}. When enabled,
|
396
|
+
the subscriber will deliver messages with the same `ordering_key` in the order
|
397
|
+
they were published.
|
398
|
+
|
399
|
+
```ruby
|
400
|
+
require "google/cloud/pubsub"
|
401
|
+
|
402
|
+
pubsub = Google::Cloud::PubSub.new
|
403
|
+
|
404
|
+
sub = pubsub.subscription "my-ordered-topic-sub"
|
405
|
+
sub.message_ordering? #=> true
|
406
|
+
|
407
|
+
subscriber = sub.listen do |received_message|
|
408
|
+
# Messsages with the same ordering_key are received
|
409
|
+
# in the order in which they were published.
|
410
|
+
received_message.acknowledge!
|
411
|
+
end
|
412
|
+
|
413
|
+
# Start background threads that will call block passed to listen.
|
414
|
+
subscriber.start
|
415
|
+
|
416
|
+
# Shut down the subscriber when ready to stop receiving messages.
|
417
|
+
subscriber.stop!
|
418
|
+
```
|
419
|
+
|
319
420
|
## Minimizing API calls before receiving and acknowledging messages
|
320
421
|
|
321
422
|
A subscription object can be created without making any API calls by providing
|
@@ -343,7 +444,7 @@ end
|
|
343
444
|
subscriber.start
|
344
445
|
|
345
446
|
# Shut down the subscriber when ready to stop receiving messages.
|
346
|
-
subscriber.stop
|
447
|
+
subscriber.stop!
|
347
448
|
```
|
348
449
|
|
349
450
|
Skipping API calls may be used to avoid `Google::Cloud::PermissionDeniedError`
|
@@ -351,7 +452,7 @@ if your account has limited access to the Pub/Sub API. In particular, the role
|
|
351
452
|
`roles/pubsub.subscriber` does not have the permission
|
352
453
|
`pubsub.subscriptions.get`, which is required to retrieve a subscription
|
353
454
|
resource. See [Access Control -
|
354
|
-
Roles](https://cloud.google.com/pubsub/docs/access-control#
|
455
|
+
Roles](https://cloud.google.com/pubsub/docs/access-control#roles) for the
|
355
456
|
complete list of Pub/Sub roles and permissions.
|
356
457
|
|
357
458
|
## Creating a snapshot and using seek
|
@@ -376,60 +477,12 @@ sub = pubsub.subscription "my-topic-sub"
|
|
376
477
|
|
377
478
|
snapshot = sub.create_snapshot
|
378
479
|
|
379
|
-
received_messages = sub.pull
|
480
|
+
received_messages = sub.pull immediate: false
|
380
481
|
sub.acknowledge received_messages
|
381
482
|
|
382
483
|
sub.seek snapshot
|
383
484
|
```
|
384
485
|
|
385
|
-
## Listening for Messages
|
386
|
-
|
387
|
-
A subscriber object can be created using `listen`, which streams messages from
|
388
|
-
the backend and processes them as they are received. (See
|
389
|
-
{Google::Cloud::PubSub::Subscription#listen Subscription#listen} and
|
390
|
-
{Google::Cloud::PubSub::Subscriber Subscriber})
|
391
|
-
|
392
|
-
```ruby
|
393
|
-
require "google/cloud/pubsub"
|
394
|
-
|
395
|
-
pubsub = Google::Cloud::PubSub.new
|
396
|
-
|
397
|
-
sub = pubsub.subscription "my-topic-sub"
|
398
|
-
|
399
|
-
subscriber = sub.listen do |received_message|
|
400
|
-
# process message
|
401
|
-
received_message.acknowledge!
|
402
|
-
end
|
403
|
-
|
404
|
-
# Start background threads that will call the block passed to listen.
|
405
|
-
subscriber.start
|
406
|
-
|
407
|
-
# Shut down the subscriber when ready to stop receiving messages.
|
408
|
-
subscriber.stop.wait!
|
409
|
-
```
|
410
|
-
|
411
|
-
The subscriber object can be configured to control the number of concurrent
|
412
|
-
streams to open, the number of received messages to be collected, and the number
|
413
|
-
of threads each stream opens for concurrent calls made to handle the received
|
414
|
-
messages.
|
415
|
-
|
416
|
-
```ruby
|
417
|
-
require "google/cloud/pubsub"
|
418
|
-
|
419
|
-
pubsub = Google::Cloud::PubSub.new
|
420
|
-
|
421
|
-
sub = pubsub.subscription "my-topic-sub"
|
422
|
-
|
423
|
-
subscriber = sub.listen threads: { callback: 16 } do |received_message|
|
424
|
-
# store the message somewhere before acknowledging
|
425
|
-
store_in_backend received_message.data # takes a few seconds
|
426
|
-
received_message.acknowledge!
|
427
|
-
end
|
428
|
-
|
429
|
-
# Start background threads that will call the block passed to listen.
|
430
|
-
subscriber.start
|
431
|
-
```
|
432
|
-
|
433
486
|
## Working Across Projects
|
434
487
|
|
435
488
|
All calls to the Pub/Sub service use the same project and credentials provided
|
data/TROUBLESHOOTING.md
CHANGED
@@ -24,14 +24,8 @@ improved, *please* create a new issue on GitHub so we can talk about it.
|
|
24
24
|
|
25
25
|
- [New issue][gh-ruby]
|
26
26
|
|
27
|
-
Or, you can ask questions on the [Google Cloud Platform Slack][slack-ruby]. You
|
28
|
-
can use the "ruby" channel for general Ruby questions, or use the
|
29
|
-
"google-cloud-ruby" channel if you have questions about this gem in particular.
|
30
|
-
|
31
27
|
[so-ruby]: http://stackoverflow.com/questions/tagged/google-cloud-platform+ruby+pubsub
|
32
28
|
|
33
|
-
[gh-search-ruby]: https://github.com/
|
34
|
-
|
35
|
-
[gh-ruby]: https://github.com/googlecloudplatform/google-cloud-ruby/issues/new
|
29
|
+
[gh-search-ruby]: https://github.com/googleapis/google-cloud-ruby/issues?q=label%3A%22api%3A+pubsub%22
|
36
30
|
|
37
|
-
[
|
31
|
+
[gh-ruby]: https://github.com/googleapis/google-cloud-ruby/issues/new
|
@@ -0,0 +1,79 @@
|
|
1
|
+
# Copyright 2022 Google LLC
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
|
16
|
+
module Google
|
17
|
+
module Cloud
|
18
|
+
module PubSub
|
19
|
+
##
|
20
|
+
# The result of a ack/nack/modack on messages.
|
21
|
+
#
|
22
|
+
# When the operation was successful the result will be marked
|
23
|
+
# {#succeeded?}. Otherwise, the result will be marked {#failed?} and the
|
24
|
+
# error raised will be availabe on {#error}.
|
25
|
+
#
|
26
|
+
class AcknowledgeResult
|
27
|
+
##
|
28
|
+
# The constants below represents the status of ack/modack operations.
|
29
|
+
# Indicates successful ack/modack
|
30
|
+
SUCCESS = 1
|
31
|
+
|
32
|
+
##
|
33
|
+
# Indicates occurence of permenant permission denied error
|
34
|
+
PERMISSION_DENIED = 2
|
35
|
+
|
36
|
+
##
|
37
|
+
# Indicates occurence of permenant failed precondition error
|
38
|
+
FAILED_PRECONDITION = 3
|
39
|
+
|
40
|
+
##
|
41
|
+
# Indicates occurence of permenant permission denied error
|
42
|
+
INVALID_ACK_ID = 4
|
43
|
+
|
44
|
+
##
|
45
|
+
# Indicates occurence of permenant uncatogorised error
|
46
|
+
OTHER = 5
|
47
|
+
|
48
|
+
##
|
49
|
+
# @return [Google::Cloud::Error] Error object of ack/modack operation
|
50
|
+
attr_reader :error
|
51
|
+
|
52
|
+
##
|
53
|
+
# @return [Numeric] Status of the ack/modack operation.
|
54
|
+
attr_reader :status
|
55
|
+
|
56
|
+
##
|
57
|
+
# @private Create an PublishResult object.
|
58
|
+
def initialize status, error = nil
|
59
|
+
@error = error
|
60
|
+
@status = status
|
61
|
+
end
|
62
|
+
|
63
|
+
##
|
64
|
+
# @return [Boolean] Whether the operation was successful.
|
65
|
+
def succeeded?
|
66
|
+
@status == SUCCESS
|
67
|
+
end
|
68
|
+
|
69
|
+
##
|
70
|
+
# @return [Boolean] Whether the operation failed.
|
71
|
+
def failed?
|
72
|
+
!succeeded?
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
Pubsub = PubSub unless const_defined? :Pubsub
|
78
|
+
end
|
79
|
+
end
|