google-cloud-pubsub 2.4.0 → 2.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AUTHENTICATION.md +2 -1
- data/CHANGELOG.md +55 -0
- data/CONTRIBUTING.md +2 -3
- data/OVERVIEW.md +9 -6
- data/lib/google/cloud/pubsub.rb +0 -4
- data/lib/google/cloud/pubsub/async_publisher.rb +42 -6
- data/lib/google/cloud/pubsub/async_publisher/batch.rb +2 -6
- data/lib/google/cloud/pubsub/batch_publisher.rb +31 -30
- data/lib/google/cloud/pubsub/errors.rb +8 -0
- data/lib/google/cloud/pubsub/flow_controller.rb +139 -0
- data/lib/google/cloud/pubsub/project.rb +248 -11
- data/lib/google/cloud/pubsub/schema.rb +310 -0
- data/lib/google/cloud/pubsub/schema/list.rb +180 -0
- data/lib/google/cloud/pubsub/service.rb +143 -31
- data/lib/google/cloud/pubsub/snapshot.rb +1 -0
- data/lib/google/cloud/pubsub/subscriber.rb +22 -22
- data/lib/google/cloud/pubsub/subscription.rb +27 -25
- data/lib/google/cloud/pubsub/topic.rb +103 -3
- data/lib/google/cloud/pubsub/version.rb +1 -1
- metadata +20 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 55eb80f365cf82b22f7cb9383a88155e0c79e0d5c1e897f2c7ee6cb974e2989b
|
4
|
+
data.tar.gz: 4417a272576757638dbe5b122e832412303f55761fb6d4d9ae24706ecdc9842f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8c1a8a9c9e62d737a468fbaca9c3909be1c608ec64d223f7ade2a478870c4e275cfad10e8c743dd213ce2f3d9084271f29cca6ba91205cda224177897d174016
|
7
|
+
data.tar.gz: 96cd5e4e593894f6c98401c9dff2ca99d7be2acdbb37504f5f53ab72cda2c9e29bde031facd7d779947deb892454716bc151f31d67eddf58b3f8944e46e66018
|
data/AUTHENTICATION.md
CHANGED
@@ -96,7 +96,8 @@ client = Google::Cloud::PubSub.new
|
|
96
96
|
|
97
97
|
### Configuration
|
98
98
|
|
99
|
-
The **Project ID** and **Credentials JSON** can be configured
|
99
|
+
The **Project ID** and the path to the **Credentials JSON** file can be configured
|
100
|
+
instead of placing them in environment variables or providing them as arguments.
|
100
101
|
|
101
102
|
```ruby
|
102
103
|
require "google/cloud/pubsub"
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,60 @@
|
|
1
1
|
# Release History
|
2
2
|
|
3
|
+
### 2.7.1 / 2021-07-08
|
4
|
+
|
5
|
+
#### Documentation
|
6
|
+
|
7
|
+
* Update AUTHENTICATION.md in handwritten packages
|
8
|
+
|
9
|
+
### 2.7.0 / 2021-06-15
|
10
|
+
|
11
|
+
#### Features
|
12
|
+
|
13
|
+
* Add Publisher Flow Control
|
14
|
+
* Add flow_control to async options in Project#create_topic and Project#topic
|
15
|
+
* Add FlowControlLimitError
|
16
|
+
|
17
|
+
#### Bug Fixes
|
18
|
+
|
19
|
+
* Fix Project#schema and #schemas to return full resource
|
20
|
+
* Include schema definition in default return values.
|
21
|
+
* Fix Schema#definition to return nil instead of empty string when not present.
|
22
|
+
|
23
|
+
### 2.6.1 / 2021-04-28
|
24
|
+
|
25
|
+
#### Bug Fixes
|
26
|
+
|
27
|
+
* Add final flush of pending requests to Subscriber#wait!
|
28
|
+
* fix(pubsub): Add final flush of pending requests to Subscriber#wait!
|
29
|
+
|
30
|
+
### 2.6.0 / 2021-04-19
|
31
|
+
|
32
|
+
#### Features
|
33
|
+
|
34
|
+
* Add ordering_key to Topic#publish
|
35
|
+
* Add ordering_key to BatchPublisher#publish
|
36
|
+
|
37
|
+
#### Documentation
|
38
|
+
|
39
|
+
* The immediate: false option is recommended to avoid adverse impacts on the performance of pull operations ([#11153](https://www.github.com/googleapis/google-cloud-ruby/issues/11153))
|
40
|
+
* Update Subscription#pull docs and samples to recommend immediate: false
|
41
|
+
|
42
|
+
### 2.5.0 / 2021-04-01
|
43
|
+
|
44
|
+
#### Features
|
45
|
+
|
46
|
+
* Add Schema support
|
47
|
+
* Add Schema
|
48
|
+
* Add Project#create_schema
|
49
|
+
* Add Project#schema
|
50
|
+
* Add Project#schemas (Schema::List)
|
51
|
+
* Add Project#valid_schema?
|
52
|
+
* Add schema options to Project#create_topic
|
53
|
+
* Add Topic#schema_name
|
54
|
+
* Add Topic#message_encoding
|
55
|
+
* Add Topic#message_encoding_binary?
|
56
|
+
* Add Topic#message_encoding_json?
|
57
|
+
|
3
58
|
### 2.4.0 / 2021-03-10
|
4
59
|
|
5
60
|
#### Features
|
data/CONTRIBUTING.md
CHANGED
@@ -119,15 +119,14 @@ If you alter an example's title, you may encounter breaking tests.
|
|
119
119
|
### Pub/Sub Acceptance Tests
|
120
120
|
|
121
121
|
The Pub/Sub acceptance tests interact with the live service API. Follow the
|
122
|
-
instructions in the {file:AUTHENTICATION.md Authentication
|
122
|
+
instructions in the {file:AUTHENTICATION.md Authentication Guide} for enabling
|
123
123
|
the Pub/Sub API. Occasionally, some API features may not yet be generally
|
124
124
|
available, making it difficult for some contributors to successfully run the
|
125
125
|
entire acceptance test suite. However, please ensure that you do successfully
|
126
126
|
run acceptance tests for any code areas covered by your pull request.
|
127
127
|
|
128
128
|
To run the acceptance tests, first create and configure a project in the Google
|
129
|
-
Developers Console, as described in the {file:AUTHENTICATION.md Authentication
|
130
|
-
guide}. Be sure to download the JSON KEY file. Make note of the PROJECT_ID and
|
129
|
+
Developers Console, as described in the {file:AUTHENTICATION.md Authentication Guide}. Be sure to download the JSON KEY file. Make note of the PROJECT_ID and
|
131
130
|
the KEYFILE location on your system.
|
132
131
|
|
133
132
|
Before you can run the Pub/Sub acceptance tests, you must first create indexes
|
data/OVERVIEW.md
CHANGED
@@ -205,13 +205,16 @@ sleep
|
|
205
205
|
Messages also can be pulled directly in a one-time operation. (See
|
206
206
|
{Google::Cloud::PubSub::Subscription#pull Subscription#pull})
|
207
207
|
|
208
|
+
The `immediate: false` option is recommended to avoid adverse impacts on the
|
209
|
+
performance of pull operations.
|
210
|
+
|
208
211
|
```ruby
|
209
212
|
require "google/cloud/pubsub"
|
210
213
|
|
211
214
|
pubsub = Google::Cloud::PubSub.new
|
212
215
|
|
213
216
|
sub = pubsub.subscription "my-topic-sub"
|
214
|
-
received_messages = sub.pull
|
217
|
+
received_messages = sub.pull immediate: false
|
215
218
|
```
|
216
219
|
|
217
220
|
A maximum number of messages to pull can be specified:
|
@@ -222,7 +225,7 @@ require "google/cloud/pubsub"
|
|
222
225
|
pubsub = Google::Cloud::PubSub.new
|
223
226
|
|
224
227
|
sub = pubsub.subscription "my-topic-sub"
|
225
|
-
received_messages = sub.pull max: 10
|
228
|
+
received_messages = sub.pull immediate: false, max: 10
|
226
229
|
```
|
227
230
|
|
228
231
|
## Acknowledging a Message
|
@@ -263,7 +266,7 @@ require "google/cloud/pubsub"
|
|
263
266
|
pubsub = Google::Cloud::PubSub.new
|
264
267
|
|
265
268
|
sub = pubsub.subscription "my-topic-sub"
|
266
|
-
received_messages = sub.pull
|
269
|
+
received_messages = sub.pull immediate: false
|
267
270
|
sub.acknowledge received_messages
|
268
271
|
```
|
269
272
|
|
@@ -328,7 +331,7 @@ require "google/cloud/pubsub"
|
|
328
331
|
pubsub = Google::Cloud::PubSub.new
|
329
332
|
|
330
333
|
sub = pubsub.subscription "my-topic-sub"
|
331
|
-
received_messages = sub.pull
|
334
|
+
received_messages = sub.pull immediate: false
|
332
335
|
sub.modify_ack_deadline 120, received_messages
|
333
336
|
```
|
334
337
|
|
@@ -474,7 +477,7 @@ sub = pubsub.subscription "my-topic-sub"
|
|
474
477
|
|
475
478
|
snapshot = sub.create_snapshot
|
476
479
|
|
477
|
-
received_messages = sub.pull
|
480
|
+
received_messages = sub.pull immediate: false
|
478
481
|
sub.acknowledge received_messages
|
479
482
|
|
480
483
|
sub.seek snapshot
|
@@ -521,5 +524,5 @@ sub.topic.name #=> "projects/other-project-id/topics/other-topic"
|
|
521
524
|
## Additional information
|
522
525
|
|
523
526
|
Google Cloud Pub/Sub can be configured to use an emulator or to enable gRPC's
|
524
|
-
logging. To learn more, see the {file:EMULATOR.md Emulator guide} and
|
527
|
+
logging. To learn more, see the {file:EMULATOR.md Emulator guide}} and
|
525
528
|
{file:LOGGING.md Logging guide}.
|
data/lib/google/cloud/pubsub.rb
CHANGED
@@ -33,8 +33,6 @@ module Google
|
|
33
33
|
# See {file:OVERVIEW.md Google Cloud Pub/Sub Overview}.
|
34
34
|
#
|
35
35
|
module PubSub
|
36
|
-
# rubocop:disable Metrics/AbcSize
|
37
|
-
|
38
36
|
##
|
39
37
|
# Creates a new object for connecting to the Pub/Sub service.
|
40
38
|
# Each call creates a new connection.
|
@@ -110,8 +108,6 @@ module Google
|
|
110
108
|
PubSub::Project.new service
|
111
109
|
end
|
112
110
|
|
113
|
-
# rubocop:enable Metrics/AbcSize
|
114
|
-
|
115
111
|
##
|
116
112
|
# Configure the Google Cloud PubSub library.
|
117
113
|
#
|
@@ -16,6 +16,7 @@
|
|
16
16
|
require "monitor"
|
17
17
|
require "concurrent"
|
18
18
|
require "google/cloud/pubsub/errors"
|
19
|
+
require "google/cloud/pubsub/flow_controller"
|
19
20
|
require "google/cloud/pubsub/async_publisher/batch"
|
20
21
|
require "google/cloud/pubsub/publish_result"
|
21
22
|
require "google/cloud/pubsub/service"
|
@@ -65,14 +66,21 @@ module Google
|
|
65
66
|
attr_reader :interval
|
66
67
|
attr_reader :publish_threads
|
67
68
|
attr_reader :callback_threads
|
69
|
+
attr_reader :flow_control
|
68
70
|
##
|
69
71
|
# @private Implementation accessors
|
70
72
|
attr_reader :service, :batch, :publish_thread_pool,
|
71
|
-
:callback_thread_pool
|
73
|
+
:callback_thread_pool, :flow_controller
|
72
74
|
|
73
75
|
##
|
74
76
|
# @private Create a new instance of the object.
|
75
|
-
def initialize topic_name,
|
77
|
+
def initialize topic_name,
|
78
|
+
service,
|
79
|
+
max_bytes: 1_000_000,
|
80
|
+
max_messages: 100,
|
81
|
+
interval: 0.01,
|
82
|
+
threads: {},
|
83
|
+
flow_control: {}
|
76
84
|
# init MonitorMixin
|
77
85
|
super()
|
78
86
|
@topic_name = service.topic_path topic_name
|
@@ -83,6 +91,10 @@ module Google
|
|
83
91
|
@interval = interval
|
84
92
|
@publish_threads = (threads[:publish] || 2).to_i
|
85
93
|
@callback_threads = (threads[:callback] || 4).to_i
|
94
|
+
@flow_control = {
|
95
|
+
message_limit: 10 * @max_messages,
|
96
|
+
byte_limit: 10 * @max_bytes
|
97
|
+
}.merge(flow_control).freeze
|
86
98
|
|
87
99
|
@published_at = nil
|
88
100
|
@publish_thread_pool = Concurrent::ThreadPoolExecutor.new max_threads: @publish_threads
|
@@ -91,7 +103,7 @@ module Google
|
|
91
103
|
@ordered = false
|
92
104
|
@batches = {}
|
93
105
|
@cond = new_cond
|
94
|
-
|
106
|
+
@flow_controller = FlowController.new(**@flow_control)
|
95
107
|
@thread = Thread.new { run_background }
|
96
108
|
end
|
97
109
|
|
@@ -121,13 +133,22 @@ module Google
|
|
121
133
|
#
|
122
134
|
def publish data = nil, attributes = nil, ordering_key: nil, **extra_attrs, &callback
|
123
135
|
msg = Convert.pubsub_message data, attributes, ordering_key, extra_attrs
|
136
|
+
begin
|
137
|
+
@flow_controller.acquire msg.to_proto.bytesize
|
138
|
+
rescue FlowControlLimitError => e
|
139
|
+
stop_publish ordering_key, e if ordering_key
|
140
|
+
raise
|
141
|
+
end
|
124
142
|
|
125
143
|
synchronize do
|
126
144
|
raise AsyncPublisherStopped if @stopped
|
127
145
|
raise OrderedMessagesDisabled if !@ordered && !msg.ordering_key.empty? # default is empty string
|
128
146
|
|
129
147
|
batch = resolve_batch_for_message msg
|
130
|
-
|
148
|
+
if batch.canceled?
|
149
|
+
@flow_controller.release msg.to_proto.bytesize
|
150
|
+
raise OrderingKeyError, batch.ordering_key
|
151
|
+
end
|
131
152
|
batch_action = batch.add msg, callback
|
132
153
|
if batch_action == :full
|
133
154
|
publish_batches!
|
@@ -305,6 +326,21 @@ module Google
|
|
305
326
|
@batches[ordering_key]
|
306
327
|
end
|
307
328
|
|
329
|
+
def stop_publish ordering_key, err
|
330
|
+
synchronize do
|
331
|
+
batch = resolve_batch_for_ordering_key ordering_key
|
332
|
+
return if batch.nil?
|
333
|
+
items = batch.cancel!
|
334
|
+
items.each do |item|
|
335
|
+
@flow_controller.release item.bytesize
|
336
|
+
next unless item.callback
|
337
|
+
|
338
|
+
publish_result = PublishResult.from_error item.msg, err
|
339
|
+
execute_callback_async item.callback, publish_result
|
340
|
+
end
|
341
|
+
end
|
342
|
+
end
|
343
|
+
|
308
344
|
def publish_batches! stop: nil
|
309
345
|
@batches.reject! { |_ordering_key, batch| batch.empty? }
|
310
346
|
@batches.each_value do |batch|
|
@@ -325,7 +361,6 @@ module Google
|
|
325
361
|
end
|
326
362
|
|
327
363
|
# rubocop:disable Metrics/AbcSize
|
328
|
-
# rubocop:disable Metrics/MethodLength
|
329
364
|
|
330
365
|
def publish_batch_sync topic_name, batch
|
331
366
|
# The only batch methods that are safe to call from the loop are
|
@@ -337,6 +372,7 @@ module Google
|
|
337
372
|
unless items.empty?
|
338
373
|
grpc = @service.publish topic_name, items.map(&:msg)
|
339
374
|
items.zip Array(grpc.message_ids) do |item, id|
|
375
|
+
@flow_controller.release item.bytesize
|
340
376
|
next unless item.callback
|
341
377
|
|
342
378
|
item.msg.message_id = id
|
@@ -363,6 +399,7 @@ module Google
|
|
363
399
|
end
|
364
400
|
|
365
401
|
items.each do |item|
|
402
|
+
@flow_controller.release item.bytesize
|
366
403
|
next unless item.callback
|
367
404
|
|
368
405
|
publish_result = PublishResult.from_error item.msg, e
|
@@ -374,7 +411,6 @@ module Google
|
|
374
411
|
end
|
375
412
|
|
376
413
|
# rubocop:enable Metrics/AbcSize
|
377
|
-
# rubocop:enable Metrics/MethodLength
|
378
414
|
|
379
415
|
PUBLISH_RETRY_ERRORS = [
|
380
416
|
GRPC::Cancelled, GRPC::DeadlineExceeded, GRPC::Internal,
|
@@ -129,8 +129,8 @@ module Google
|
|
129
129
|
end
|
130
130
|
|
131
131
|
##
|
132
|
-
#
|
133
|
-
# the active item list.
|
132
|
+
# Fills the batch by sequentially moving the queued items that will
|
133
|
+
# fit into the active item list.
|
134
134
|
#
|
135
135
|
# This method is only intended to be used by the active publishing
|
136
136
|
# job.
|
@@ -152,8 +152,6 @@ module Google
|
|
152
152
|
end
|
153
153
|
end
|
154
154
|
|
155
|
-
# rubocop:disable Metrics/MethodLength
|
156
|
-
|
157
155
|
##
|
158
156
|
# Resets the batch after a successful publish. This clears the active
|
159
157
|
# item list and moves the queued items that will fit into the active
|
@@ -202,8 +200,6 @@ module Google
|
|
202
200
|
true
|
203
201
|
end
|
204
202
|
|
205
|
-
# rubocop:enable Metrics/MethodLength
|
206
|
-
|
207
203
|
##
|
208
204
|
# Cancel the batch and hault futher batches until resumed. See
|
209
205
|
# {#resume!} and {#canceled?}.
|
@@ -13,6 +13,8 @@
|
|
13
13
|
# limitations under the License.
|
14
14
|
|
15
15
|
|
16
|
+
require "google/cloud/pubsub/convert"
|
17
|
+
|
16
18
|
module Google
|
17
19
|
module Cloud
|
18
20
|
module PubSub
|
@@ -26,11 +28,12 @@ module Google
|
|
26
28
|
# pubsub = Google::Cloud::PubSub.new
|
27
29
|
#
|
28
30
|
# topic = pubsub.topic "my-topic"
|
29
|
-
# msgs = topic.publish do |
|
30
|
-
#
|
31
|
-
#
|
32
|
-
#
|
31
|
+
# msgs = topic.publish do |batch_publisher|
|
32
|
+
# batch_publisher.publish "task 1 completed", foo: :bar
|
33
|
+
# batch_publisher.publish "task 2 completed", foo: :baz
|
34
|
+
# batch_publisher.publish "task 3 completed", foo: :bif
|
33
35
|
# end
|
36
|
+
#
|
34
37
|
class BatchPublisher
|
35
38
|
##
|
36
39
|
# @private The messages to publish
|
@@ -38,20 +41,40 @@ module Google
|
|
38
41
|
|
39
42
|
##
|
40
43
|
# @private Create a new instance of the object.
|
41
|
-
def initialize data
|
44
|
+
def initialize data, attributes, ordering_key, extra_attrs
|
42
45
|
@messages = []
|
43
46
|
@mode = :batch
|
44
47
|
return if data.nil?
|
45
48
|
@mode = :single
|
46
|
-
publish data, attributes
|
49
|
+
publish data, attributes, ordering_key: ordering_key, **extra_attrs
|
47
50
|
end
|
48
51
|
|
49
52
|
##
|
50
53
|
# Add a message to the batch to be published to the topic.
|
51
54
|
# All messages added to the batch will be published at once.
|
52
55
|
# See {Google::Cloud::PubSub::Topic#publish}
|
53
|
-
|
54
|
-
|
56
|
+
#
|
57
|
+
# @param [String, File] data The message payload. This will be converted
|
58
|
+
# to bytes encoded as ASCII-8BIT.
|
59
|
+
# @param [Hash] attributes Optional attributes for the message.
|
60
|
+
# @param [String] ordering_key Identifies related messages for which
|
61
|
+
# publish order should be respected.
|
62
|
+
#
|
63
|
+
# @example Multiple messages can be sent at the same time using a block:
|
64
|
+
# require "google/cloud/pubsub"
|
65
|
+
#
|
66
|
+
# pubsub = Google::Cloud::PubSub.new
|
67
|
+
#
|
68
|
+
# topic = pubsub.topic "my-topic"
|
69
|
+
# msgs = topic.publish do |batch_publisher|
|
70
|
+
# batch_publisher.publish "task 1 completed", foo: :bar
|
71
|
+
# batch_publisher.publish "task 2 completed", foo: :baz
|
72
|
+
# batch_publisher.publish "task 3 completed", foo: :bif
|
73
|
+
# end
|
74
|
+
#
|
75
|
+
def publish data, attributes = nil, ordering_key: nil, **extra_attrs
|
76
|
+
msg = Convert.pubsub_message data, attributes, ordering_key, extra_attrs
|
77
|
+
@messages << msg
|
55
78
|
end
|
56
79
|
|
57
80
|
##
|
@@ -69,28 +92,6 @@ module Google
|
|
69
92
|
msgs
|
70
93
|
end
|
71
94
|
end
|
72
|
-
|
73
|
-
protected
|
74
|
-
|
75
|
-
def create_pubsub_message data, attributes
|
76
|
-
attributes ||= {}
|
77
|
-
if data.is_a?(::Hash) && attributes.empty?
|
78
|
-
attributes = data
|
79
|
-
data = nil
|
80
|
-
end
|
81
|
-
# Convert IO-ish objects to strings
|
82
|
-
if data.respond_to?(:read) && data.respond_to?(:rewind)
|
83
|
-
data.rewind
|
84
|
-
data = data.read
|
85
|
-
end
|
86
|
-
# Convert data to encoded byte array to match the protobuf defn
|
87
|
-
data_bytes = String(data).dup.force_encoding(Encoding::ASCII_8BIT).freeze
|
88
|
-
|
89
|
-
# Convert attributes to strings to match the protobuf definition
|
90
|
-
attributes = Hash[attributes.map { |k, v| [String(k), String(v)] }]
|
91
|
-
|
92
|
-
Google::Cloud::PubSub::V1::PubsubMessage.new data: data_bytes, attributes: attributes
|
93
|
-
end
|
94
95
|
end
|
95
96
|
end
|
96
97
|
|
@@ -78,6 +78,14 @@ module Google
|
|
78
78
|
super "Can't publish message using #{ordering_key}."
|
79
79
|
end
|
80
80
|
end
|
81
|
+
|
82
|
+
##
|
83
|
+
# Raised when the desired action is `error` and the message would exceed
|
84
|
+
# flow control limits, or when the desired action is `block` and the
|
85
|
+
# message would block forever against the flow control limits.
|
86
|
+
#
|
87
|
+
class FlowControlLimitError < Google::Cloud::Error
|
88
|
+
end
|
81
89
|
end
|
82
90
|
|
83
91
|
Pubsub = PubSub unless const_defined? :Pubsub
|