google-cloud-pubsub 2.1.1 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +88 -0
- data/CONTRIBUTING.md +4 -5
- data/LOGGING.md +1 -1
- data/OVERVIEW.md +9 -6
- data/lib/google/cloud/pubsub.rb +0 -4
- data/lib/google/cloud/pubsub/async_publisher.rb +51 -11
- data/lib/google/cloud/pubsub/async_publisher/batch.rb +4 -7
- data/lib/google/cloud/pubsub/batch_publisher.rb +31 -30
- data/lib/google/cloud/pubsub/credentials.rb +1 -1
- data/lib/google/cloud/pubsub/errors.rb +8 -0
- data/lib/google/cloud/pubsub/flow_controller.rb +139 -0
- data/lib/google/cloud/pubsub/policy.rb +3 -2
- data/lib/google/cloud/pubsub/project.rb +272 -18
- data/lib/google/cloud/pubsub/retry_policy.rb +2 -4
- data/lib/google/cloud/pubsub/schema.rb +310 -0
- data/lib/google/cloud/pubsub/schema/list.rb +180 -0
- data/lib/google/cloud/pubsub/service.rb +147 -32
- data/lib/google/cloud/pubsub/snapshot.rb +5 -2
- data/lib/google/cloud/pubsub/snapshot/list.rb +2 -2
- data/lib/google/cloud/pubsub/subscriber.rb +43 -25
- data/lib/google/cloud/pubsub/subscriber/inventory.rb +10 -3
- data/lib/google/cloud/pubsub/subscriber/stream.rb +3 -3
- data/lib/google/cloud/pubsub/subscriber/timed_unary_buffer.rb +6 -7
- data/lib/google/cloud/pubsub/subscription.rb +110 -53
- data/lib/google/cloud/pubsub/subscription/list.rb +2 -2
- data/lib/google/cloud/pubsub/topic.rb +126 -19
- data/lib/google/cloud/pubsub/topic/list.rb +2 -2
- data/lib/google/cloud/pubsub/version.rb +1 -1
- metadata +23 -6
@@ -26,7 +26,10 @@ module Google
|
|
26
26
|
##
|
27
27
|
# @private Represents the Pub/Sub service API, including IAM mixins.
|
28
28
|
class Service
|
29
|
-
attr_accessor :project
|
29
|
+
attr_accessor :project
|
30
|
+
attr_accessor :credentials
|
31
|
+
attr_accessor :host
|
32
|
+
attr_accessor :timeout
|
30
33
|
###
|
31
34
|
# The same client_id is used across all streaming pull connections that are created by this client. This is
|
32
35
|
# intentional, as it indicates to the server that any guarantees, such as message ordering, made for a stream
|
@@ -46,49 +49,56 @@ module Google
|
|
46
49
|
|
47
50
|
def subscriber
|
48
51
|
return mocked_subscriber if mocked_subscriber
|
49
|
-
@subscriber ||=
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
config.metadata = { "google-cloud-resource-prefix": "projects/#{@project}" }
|
57
|
-
end
|
52
|
+
@subscriber ||= V1::Subscriber::Client.new do |config|
|
53
|
+
config.credentials = credentials if credentials
|
54
|
+
config.timeout = timeout if timeout
|
55
|
+
config.endpoint = host if host
|
56
|
+
config.lib_name = "gccl"
|
57
|
+
config.lib_version = Google::Cloud::PubSub::VERSION
|
58
|
+
config.metadata = { "google-cloud-resource-prefix": "projects/#{@project}" }
|
58
59
|
end
|
59
60
|
end
|
60
61
|
attr_accessor :mocked_subscriber
|
61
62
|
|
62
63
|
def publisher
|
63
64
|
return mocked_publisher if mocked_publisher
|
64
|
-
@publisher ||=
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
config.metadata = { "google-cloud-resource-prefix": "projects/#{@project}" }
|
72
|
-
end
|
65
|
+
@publisher ||= V1::Publisher::Client.new do |config|
|
66
|
+
config.credentials = credentials if credentials
|
67
|
+
config.timeout = timeout if timeout
|
68
|
+
config.endpoint = host if host
|
69
|
+
config.lib_name = "gccl"
|
70
|
+
config.lib_version = Google::Cloud::PubSub::VERSION
|
71
|
+
config.metadata = { "google-cloud-resource-prefix": "projects/#{@project}" }
|
73
72
|
end
|
74
73
|
end
|
75
74
|
attr_accessor :mocked_publisher
|
76
75
|
|
77
76
|
def iam
|
78
77
|
return mocked_iam if mocked_iam
|
79
|
-
@iam ||=
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
config.metadata = { "google-cloud-resource-prefix" => "projects/#{@project}" }
|
87
|
-
end
|
78
|
+
@iam ||= V1::IAMPolicy::Client.new do |config|
|
79
|
+
config.credentials = credentials if credentials
|
80
|
+
config.timeout = timeout if timeout
|
81
|
+
config.endpoint = host if host
|
82
|
+
config.lib_name = "gccl"
|
83
|
+
config.lib_version = Google::Cloud::PubSub::VERSION
|
84
|
+
config.metadata = { "google-cloud-resource-prefix" => "projects/#{@project}" }
|
88
85
|
end
|
89
86
|
end
|
90
87
|
attr_accessor :mocked_iam
|
91
88
|
|
89
|
+
def schemas
|
90
|
+
return mocked_schemas if mocked_schemas
|
91
|
+
@schemas ||= V1::SchemaService::Client.new do |config|
|
92
|
+
config.credentials = credentials if credentials
|
93
|
+
config.timeout = timeout if timeout
|
94
|
+
config.endpoint = host if host
|
95
|
+
config.lib_name = "gccl"
|
96
|
+
config.lib_version = Google::Cloud::PubSub::VERSION
|
97
|
+
config.metadata = { "google-cloud-resource-prefix" => "projects/#{@project}" }
|
98
|
+
end
|
99
|
+
end
|
100
|
+
attr_accessor :mocked_schemas
|
101
|
+
|
92
102
|
##
|
93
103
|
# Gets the configuration of a topic.
|
94
104
|
# Since the topic only has the name attribute,
|
@@ -111,18 +121,35 @@ module Google
|
|
111
121
|
|
112
122
|
##
|
113
123
|
# Creates the given topic with the given name.
|
114
|
-
def create_topic topic_name,
|
124
|
+
def create_topic topic_name,
|
125
|
+
labels: nil,
|
126
|
+
kms_key_name: nil,
|
127
|
+
persistence_regions: nil,
|
128
|
+
schema_name: nil,
|
129
|
+
message_encoding: nil,
|
130
|
+
options: {}
|
115
131
|
if persistence_regions
|
116
|
-
message_storage_policy =
|
132
|
+
message_storage_policy = Google::Cloud::PubSub::V1::MessageStoragePolicy.new(
|
117
133
|
allowed_persistence_regions: Array(persistence_regions)
|
118
|
-
|
134
|
+
)
|
135
|
+
end
|
136
|
+
|
137
|
+
if schema_name || message_encoding
|
138
|
+
unless schema_name && message_encoding
|
139
|
+
raise ArgumentError, "Schema settings must include both schema_name and message_encoding."
|
140
|
+
end
|
141
|
+
schema_settings = Google::Cloud::PubSub::V1::SchemaSettings.new(
|
142
|
+
schema: schema_path(schema_name),
|
143
|
+
encoding: message_encoding.to_s.upcase
|
144
|
+
)
|
119
145
|
end
|
120
146
|
|
121
147
|
publisher.create_topic \
|
122
148
|
name: topic_path(topic_name, options),
|
123
149
|
labels: labels,
|
124
150
|
kms_key_name: kms_key_name,
|
125
|
-
message_storage_policy: message_storage_policy
|
151
|
+
message_storage_policy: message_storage_policy,
|
152
|
+
schema_settings: schema_settings
|
126
153
|
end
|
127
154
|
|
128
155
|
def update_topic topic_obj, *fields
|
@@ -294,6 +321,89 @@ module Google
|
|
294
321
|
end
|
295
322
|
end
|
296
323
|
|
324
|
+
##
|
325
|
+
# Lists schemas in the current (or given) project.
|
326
|
+
# @param view [String, Symbol, nil] Possible values:
|
327
|
+
# * `BASIC` - Include the name and type of the schema, but not the definition.
|
328
|
+
# * `FULL` - Include all Schema object fields.
|
329
|
+
#
|
330
|
+
def list_schemas view, options = {}
|
331
|
+
schema_view = Google::Cloud::PubSub::V1::SchemaView.const_get view.to_s.upcase
|
332
|
+
paged_enum = schemas.list_schemas parent: project_path(options),
|
333
|
+
view: schema_view,
|
334
|
+
page_size: options[:max],
|
335
|
+
page_token: options[:token]
|
336
|
+
|
337
|
+
paged_enum.response
|
338
|
+
end
|
339
|
+
|
340
|
+
##
|
341
|
+
# Creates a schema in the current (or given) project.
|
342
|
+
def create_schema schema_id, type, definition, options = {}
|
343
|
+
schema = Google::Cloud::PubSub::V1::Schema.new(
|
344
|
+
type: type,
|
345
|
+
definition: definition
|
346
|
+
)
|
347
|
+
schemas.create_schema parent: project_path(options),
|
348
|
+
schema: schema,
|
349
|
+
schema_id: schema_id
|
350
|
+
end
|
351
|
+
|
352
|
+
##
|
353
|
+
# Gets the details of a schema.
|
354
|
+
# @param view [String, Symbol, nil] The set of fields to return in the response. Possible values:
|
355
|
+
# * `BASIC` - Include the name and type of the schema, but not the definition.
|
356
|
+
# * `FULL` - Include all Schema object fields.
|
357
|
+
#
|
358
|
+
def get_schema schema_name, view, options = {}
|
359
|
+
schema_view = Google::Cloud::PubSub::V1::SchemaView.const_get view.to_s.upcase
|
360
|
+
schemas.get_schema name: schema_path(schema_name, options),
|
361
|
+
view: schema_view
|
362
|
+
end
|
363
|
+
|
364
|
+
##
|
365
|
+
# Delete a schema.
|
366
|
+
def delete_schema schema_name
|
367
|
+
schemas.delete_schema name: schema_path(schema_name)
|
368
|
+
end
|
369
|
+
|
370
|
+
##
|
371
|
+
# Validate the definition string intended for a schema.
|
372
|
+
def validate_schema type, definition, options = {}
|
373
|
+
schema = Google::Cloud::PubSub::V1::Schema.new(
|
374
|
+
type: type,
|
375
|
+
definition: definition
|
376
|
+
)
|
377
|
+
schemas.validate_schema parent: project_path(options),
|
378
|
+
schema: schema
|
379
|
+
end
|
380
|
+
|
381
|
+
##
|
382
|
+
# Validates a message against a schema.
|
383
|
+
#
|
384
|
+
# @param message_data [String] Message to validate against the provided `schema_spec`.
|
385
|
+
# @param message_encoding [Google::Cloud::PubSub::V1::Encoding] The encoding expected for messages.
|
386
|
+
# @param schema_name [String] Name of the schema against which to validate.
|
387
|
+
# @param project [String] Name of the project if not the default project.
|
388
|
+
# @param type [String] Ad-hoc schema type against which to validate.
|
389
|
+
# @param definition [String] Ad-hoc schema definition against which to validate.
|
390
|
+
#
|
391
|
+
def validate_message message_data, message_encoding, schema_name: nil, project: nil, type: nil, definition: nil
|
392
|
+
if type && definition
|
393
|
+
schema = Google::Cloud::PubSub::V1::Schema.new(
|
394
|
+
type: type,
|
395
|
+
definition: definition
|
396
|
+
)
|
397
|
+
end
|
398
|
+
schemas.validate_message parent: project_path(project: project),
|
399
|
+
name: schema_path(schema_name),
|
400
|
+
schema: schema,
|
401
|
+
message: message_data,
|
402
|
+
encoding: message_encoding
|
403
|
+
end
|
404
|
+
|
405
|
+
# Helper methods
|
406
|
+
|
297
407
|
def get_topic_policy topic_name, options = {}
|
298
408
|
iam.get_iam_policy resource: topic_path(topic_name, options)
|
299
409
|
end
|
@@ -338,6 +448,11 @@ module Google
|
|
338
448
|
"#{project_path options}/snapshots/#{snapshot_name}"
|
339
449
|
end
|
340
450
|
|
451
|
+
def schema_path schema_name, options = {}
|
452
|
+
return schema_name if schema_name.nil? || schema_name.to_s.include?("/")
|
453
|
+
"#{project_path options}/schemas/#{schema_name}"
|
454
|
+
end
|
455
|
+
|
341
456
|
def inspect
|
342
457
|
"#<#{self.class.name} (#{@project})>"
|
343
458
|
end
|
@@ -15,6 +15,7 @@
|
|
15
15
|
|
16
16
|
require "google/cloud/errors"
|
17
17
|
require "google/cloud/pubsub/snapshot/list"
|
18
|
+
require "google/cloud/pubsub/v1"
|
18
19
|
|
19
20
|
module Google
|
20
21
|
module Cloud
|
@@ -58,8 +59,10 @@ module Google
|
|
58
59
|
end
|
59
60
|
|
60
61
|
##
|
61
|
-
# The name of the snapshot.
|
62
|
-
#
|
62
|
+
# The name of the snapshot.
|
63
|
+
#
|
64
|
+
# @return [String] A fully-qualified snapshot name in the form
|
65
|
+
# `projects/{project_id}/snapshots/{snapshot_id}`.
|
63
66
|
def name
|
64
67
|
@grpc.name
|
65
68
|
end
|
@@ -125,12 +125,12 @@ module Google
|
|
125
125
|
# puts snapshot.name
|
126
126
|
# end
|
127
127
|
#
|
128
|
-
def all request_limit: nil
|
128
|
+
def all request_limit: nil, &block
|
129
129
|
request_limit = request_limit.to_i if request_limit
|
130
130
|
return enum_for :all, request_limit: request_limit unless block_given?
|
131
131
|
results = self
|
132
132
|
loop do
|
133
|
-
results.each
|
133
|
+
results.each(&block)
|
134
134
|
if request_limit
|
135
135
|
request_limit -= 1
|
136
136
|
break if request_limit.negative?
|
@@ -64,8 +64,13 @@ module Google
|
|
64
64
|
class Subscriber
|
65
65
|
include MonitorMixin
|
66
66
|
|
67
|
-
attr_reader :subscription_name
|
68
|
-
|
67
|
+
attr_reader :subscription_name
|
68
|
+
attr_reader :callback
|
69
|
+
attr_reader :deadline
|
70
|
+
attr_reader :streams
|
71
|
+
attr_reader :message_ordering
|
72
|
+
attr_reader :callback_threads
|
73
|
+
attr_reader :push_threads
|
69
74
|
|
70
75
|
##
|
71
76
|
# @private Implementation attributes.
|
@@ -123,28 +128,21 @@ module Google
|
|
123
128
|
|
124
129
|
##
|
125
130
|
# Immediately stops the subscriber. No new messages will be pulled from
|
126
|
-
# the subscription.
|
127
|
-
#
|
128
|
-
#
|
129
|
-
#
|
130
|
-
# received messages have been processed or released.
|
131
|
+
# the subscription. Use {#wait!} to block until all received messages have
|
132
|
+
# been processed or released: All actions taken on received messages that
|
133
|
+
# have not yet been sent to the API will be sent to the API. All received
|
134
|
+
# but unprocessed messages will be released back to the API and redelivered.
|
131
135
|
#
|
132
136
|
# @return [Subscriber] returns self so calls can be chained.
|
133
137
|
#
|
134
138
|
def stop
|
135
|
-
|
139
|
+
synchronize do
|
136
140
|
@started = false
|
137
141
|
@stopped = true
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
end
|
142
|
+
@stream_pool.map(&:stop)
|
143
|
+
wait_stop_buffer_thread!
|
144
|
+
self
|
142
145
|
end
|
143
|
-
stop_pool.map(&:join)
|
144
|
-
# Stop the buffer after the streams are all stopped
|
145
|
-
synchronize { @buffer.stop }
|
146
|
-
|
147
|
-
self
|
148
146
|
end
|
149
147
|
|
150
148
|
##
|
@@ -162,13 +160,8 @@ module Google
|
|
162
160
|
# @return [Subscriber] returns self so calls can be chained.
|
163
161
|
#
|
164
162
|
def wait! timeout = nil
|
165
|
-
|
166
|
-
|
167
|
-
Thread.new { stream.wait! timeout }
|
168
|
-
end
|
169
|
-
end
|
170
|
-
wait_pool.map(&:join)
|
171
|
-
|
163
|
+
wait_stop_buffer_thread!
|
164
|
+
@wait_stop_buffer_thread.join timeout
|
172
165
|
self
|
173
166
|
end
|
174
167
|
|
@@ -306,6 +299,17 @@ module Google
|
|
306
299
|
# @deprecated Use {#max_outstanding_bytes}.
|
307
300
|
alias inventory_bytesize max_outstanding_bytes
|
308
301
|
|
302
|
+
##
|
303
|
+
# Whether to enforce flow control at the client side only or to enforce it at both the client and
|
304
|
+
# the server. For more details about flow control see https://cloud.google.com/pubsub/docs/pull#config.
|
305
|
+
#
|
306
|
+
# @return [Boolean] `true` when only client side flow control is enforced, `false` when both client and
|
307
|
+
# server side flow control are enforced.
|
308
|
+
#
|
309
|
+
def use_legacy_flow_control?
|
310
|
+
@inventory[:use_legacy_flow_control]
|
311
|
+
end
|
312
|
+
|
309
313
|
##
|
310
314
|
# The number of seconds that received messages can be held awaiting processing. Default is 3,600 (1 hour).
|
311
315
|
#
|
@@ -334,7 +338,8 @@ module Google
|
|
334
338
|
limit: @inventory[:max_outstanding_messages].fdiv(@streams).ceil,
|
335
339
|
bytesize: @inventory[:max_outstanding_bytes].fdiv(@streams).ceil,
|
336
340
|
extension: @inventory[:max_total_lease_duration],
|
337
|
-
max_duration_per_lease_extension: @inventory[:max_duration_per_lease_extension]
|
341
|
+
max_duration_per_lease_extension: @inventory[:max_duration_per_lease_extension],
|
342
|
+
use_legacy_flow_control: @inventory[:use_legacy_flow_control]
|
338
343
|
}
|
339
344
|
end
|
340
345
|
|
@@ -362,6 +367,18 @@ module Google
|
|
362
367
|
|
363
368
|
protected
|
364
369
|
|
370
|
+
##
|
371
|
+
# Starts a new thread to call wait! (blocking) on each Stream and then stop the TimedUnaryBuffer.
|
372
|
+
def wait_stop_buffer_thread!
|
373
|
+
synchronize do
|
374
|
+
@wait_stop_buffer_thread ||= Thread.new do
|
375
|
+
@stream_pool.map(&:wait!)
|
376
|
+
# Shutdown the buffer TimerTask (and flush the buffer) after the streams are all stopped.
|
377
|
+
@buffer.stop
|
378
|
+
end
|
379
|
+
end
|
380
|
+
end
|
381
|
+
|
365
382
|
def coerce_inventory inventory
|
366
383
|
@inventory = inventory
|
367
384
|
if @inventory.is_a? Hash
|
@@ -377,6 +394,7 @@ module Google
|
|
377
394
|
@inventory[:max_outstanding_bytes] = Integer(@inventory[:max_outstanding_bytes] || 100_000_000)
|
378
395
|
@inventory[:max_total_lease_duration] = Integer(@inventory[:max_total_lease_duration] || 3600)
|
379
396
|
@inventory[:max_duration_per_lease_extension] = Integer(@inventory[:max_duration_per_lease_extension] || 0)
|
397
|
+
@inventory[:use_legacy_flow_control] = @inventory[:use_legacy_flow_control] || false
|
380
398
|
end
|
381
399
|
|
382
400
|
def default_error_callbacks
|
@@ -30,15 +30,22 @@ module Google
|
|
30
30
|
|
31
31
|
include MonitorMixin
|
32
32
|
|
33
|
-
attr_reader :stream
|
34
|
-
|
35
|
-
|
33
|
+
attr_reader :stream
|
34
|
+
attr_reader :limit
|
35
|
+
attr_reader :bytesize
|
36
|
+
attr_reader :extension
|
37
|
+
attr_reader :max_duration_per_lease_extension
|
38
|
+
attr_reader :use_legacy_flow_control
|
39
|
+
|
40
|
+
def initialize stream, limit:, bytesize:, extension:, max_duration_per_lease_extension:,
|
41
|
+
use_legacy_flow_control:
|
36
42
|
super()
|
37
43
|
@stream = stream
|
38
44
|
@limit = limit
|
39
45
|
@bytesize = bytesize
|
40
46
|
@extension = extension
|
41
47
|
@max_duration_per_lease_extension = max_duration_per_lease_extension
|
48
|
+
@use_legacy_flow_control = use_legacy_flow_control
|
42
49
|
@inventory = {}
|
43
50
|
@wait_cond = new_cond
|
44
51
|
end
|
@@ -58,7 +58,7 @@ module Google
|
|
58
58
|
@paused = nil
|
59
59
|
@pause_cond = new_cond
|
60
60
|
|
61
|
-
@inventory = Inventory.new self,
|
61
|
+
@inventory = Inventory.new self, **@subscriber.stream_inventory
|
62
62
|
|
63
63
|
@sequencer = Sequencer.new(&method(:perform_callback_async)) if subscriber.message_ordering
|
64
64
|
|
@@ -363,8 +363,8 @@ module Google
|
|
363
363
|
req.modify_deadline_ack_ids += @inventory.ack_ids
|
364
364
|
req.modify_deadline_seconds += @inventory.ack_ids.map { @subscriber.deadline }
|
365
365
|
req.client_id = @subscriber.service.client_id
|
366
|
-
req.max_outstanding_messages = @inventory.limit
|
367
|
-
req.max_outstanding_bytes = @inventory.bytesize
|
366
|
+
req.max_outstanding_messages = @inventory.use_legacy_flow_control ? 0 : @inventory.limit
|
367
|
+
req.max_outstanding_bytes = @inventory.use_legacy_flow_control ? 0 : @inventory.bytesize
|
368
368
|
end
|
369
369
|
end
|
370
370
|
|
@@ -25,7 +25,8 @@ module Google
|
|
25
25
|
class TimedUnaryBuffer
|
26
26
|
include MonitorMixin
|
27
27
|
|
28
|
-
attr_reader :max_bytes
|
28
|
+
attr_reader :max_bytes
|
29
|
+
attr_reader :interval
|
29
30
|
|
30
31
|
def initialize subscriber, max_bytes: 500_000, interval: 1.0
|
31
32
|
super() # to init MonitorMixin
|
@@ -138,7 +139,7 @@ module Google
|
|
138
139
|
end
|
139
140
|
|
140
141
|
groups = prev_reg.each_pair.group_by { |_ack_id, delay| delay }
|
141
|
-
req_hash =
|
142
|
+
req_hash = groups.transform_values { |v| v.map(&:first) }
|
142
143
|
|
143
144
|
requests = { acknowledge: [] }
|
144
145
|
ack_ids = Array(req_hash.delete(:ack)) # ack has no deadline set
|
@@ -215,11 +216,9 @@ module Google
|
|
215
216
|
|
216
217
|
def add_future pool
|
217
218
|
Concurrent::Promises.future_on pool do
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
error! e
|
222
|
-
end
|
219
|
+
yield
|
220
|
+
rescue StandardError => e
|
221
|
+
error! e
|
223
222
|
end
|
224
223
|
end
|
225
224
|
end
|