google-cloud-pubsub 2.4.0 → 2.7.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/AUTHENTICATION.md +2 -1
- data/CHANGELOG.md +55 -0
- data/CONTRIBUTING.md +2 -3
- data/OVERVIEW.md +9 -6
- data/lib/google/cloud/pubsub.rb +0 -4
- data/lib/google/cloud/pubsub/async_publisher.rb +42 -6
- data/lib/google/cloud/pubsub/async_publisher/batch.rb +2 -6
- data/lib/google/cloud/pubsub/batch_publisher.rb +31 -30
- data/lib/google/cloud/pubsub/errors.rb +8 -0
- data/lib/google/cloud/pubsub/flow_controller.rb +139 -0
- data/lib/google/cloud/pubsub/project.rb +248 -11
- data/lib/google/cloud/pubsub/schema.rb +310 -0
- data/lib/google/cloud/pubsub/schema/list.rb +180 -0
- data/lib/google/cloud/pubsub/service.rb +143 -31
- data/lib/google/cloud/pubsub/snapshot.rb +1 -0
- data/lib/google/cloud/pubsub/subscriber.rb +22 -22
- data/lib/google/cloud/pubsub/subscription.rb +27 -25
- data/lib/google/cloud/pubsub/topic.rb +103 -3
- data/lib/google/cloud/pubsub/version.rb +1 -1
- metadata +20 -3
@@ -0,0 +1,180 @@
|
|
1
|
+
# Copyright 2021 Google LLC
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
|
16
|
+
require "delegate"
|
17
|
+
|
18
|
+
module Google
|
19
|
+
module Cloud
|
20
|
+
module PubSub
|
21
|
+
class Schema
|
22
|
+
##
|
23
|
+
# Schema::List is a special case Array with additional values.
|
24
|
+
class List < DelegateClass(::Array)
|
25
|
+
##
|
26
|
+
# If not empty, indicates that there are more schemas
|
27
|
+
# that match the request and this value should be passed to
|
28
|
+
# the next {Google::Cloud::PubSub::Project#schemas} to continue.
|
29
|
+
attr_accessor :token
|
30
|
+
|
31
|
+
##
|
32
|
+
# @private Create a new Schema::List with an array of values.
|
33
|
+
def initialize arr = []
|
34
|
+
@prefix = nil
|
35
|
+
@token = nil
|
36
|
+
@view = nil
|
37
|
+
@max = nil
|
38
|
+
super arr
|
39
|
+
end
|
40
|
+
|
41
|
+
##
|
42
|
+
# Whether there a next page of schemas.
|
43
|
+
#
|
44
|
+
# @return [Boolean]
|
45
|
+
#
|
46
|
+
# @example
|
47
|
+
# require "google/cloud/pubsub"
|
48
|
+
#
|
49
|
+
# pubsub = Google::Cloud::PubSub.new
|
50
|
+
#
|
51
|
+
# schemas = pubsub.schemas
|
52
|
+
# if schemas.next?
|
53
|
+
# next_schemas = schemas.next
|
54
|
+
# end
|
55
|
+
#
|
56
|
+
def next?
|
57
|
+
!token.nil?
|
58
|
+
end
|
59
|
+
|
60
|
+
##
|
61
|
+
# Retrieve the next page of schemas.
|
62
|
+
#
|
63
|
+
# @return [Schema::List]
|
64
|
+
#
|
65
|
+
# @example
|
66
|
+
# require "google/cloud/pubsub"
|
67
|
+
#
|
68
|
+
# pubsub = Google::Cloud::PubSub.new
|
69
|
+
#
|
70
|
+
# schemas = pubsub.schemas
|
71
|
+
# if schemas.next?
|
72
|
+
# next_schemas = schemas.next
|
73
|
+
# end
|
74
|
+
#
|
75
|
+
def next
|
76
|
+
return nil unless next?
|
77
|
+
ensure_service!
|
78
|
+
next_schemas
|
79
|
+
end
|
80
|
+
|
81
|
+
##
|
82
|
+
# Retrieves remaining results by repeatedly invoking {#next} until
|
83
|
+
# {#next?} returns `false`. Calls the given block once for each
|
84
|
+
# result, which is passed as the argument to the block.
|
85
|
+
#
|
86
|
+
# An Enumerator is returned if no block is given.
|
87
|
+
#
|
88
|
+
# This method will make repeated API calls until all remaining results
|
89
|
+
# are retrieved. (Unlike `#each`, for example, which merely iterates
|
90
|
+
# over the results returned by a single API call.) Use with caution.
|
91
|
+
#
|
92
|
+
# @param [Integer] request_limit The upper limit of API requests to
|
93
|
+
# make to load all schemas. Default is no limit.
|
94
|
+
# @yield [schema] The block for accessing each schema.
|
95
|
+
# @yieldparam [Schema] schema The schema object.
|
96
|
+
#
|
97
|
+
# @return [Enumerator]
|
98
|
+
#
|
99
|
+
# @example Iterating each schema by passing a block:
|
100
|
+
# require "google/cloud/pubsub"
|
101
|
+
#
|
102
|
+
# pubsub = Google::Cloud::PubSub.new
|
103
|
+
#
|
104
|
+
# schemas = pubsub.schemas
|
105
|
+
# schemas.all do |schema|
|
106
|
+
# puts schema.name
|
107
|
+
# end
|
108
|
+
#
|
109
|
+
# @example Using the enumerator by not passing a block:
|
110
|
+
# require "google/cloud/pubsub"
|
111
|
+
#
|
112
|
+
# pubsub = Google::Cloud::PubSub.new
|
113
|
+
#
|
114
|
+
# schemas = pubsub.schemas
|
115
|
+
# all_names = schemas.all.map do |schema|
|
116
|
+
# schema.name
|
117
|
+
# end
|
118
|
+
#
|
119
|
+
# @example Limit the number of API calls made:
|
120
|
+
# require "google/cloud/pubsub"
|
121
|
+
#
|
122
|
+
# pubsub = Google::Cloud::PubSub.new
|
123
|
+
#
|
124
|
+
# schemas = pubsub.schemas
|
125
|
+
# schemas.all(request_limit: 10) do |schema|
|
126
|
+
# puts schema.name
|
127
|
+
# end
|
128
|
+
#
|
129
|
+
def all request_limit: nil, &block
|
130
|
+
request_limit = request_limit.to_i if request_limit
|
131
|
+
return enum_for :all, request_limit: request_limit unless block_given?
|
132
|
+
results = self
|
133
|
+
loop do
|
134
|
+
results.each(&block)
|
135
|
+
if request_limit
|
136
|
+
request_limit -= 1
|
137
|
+
break if request_limit.negative?
|
138
|
+
end
|
139
|
+
break unless results.next?
|
140
|
+
results = results.next
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
##
|
145
|
+
# @private New Schemas::List from a
|
146
|
+
# Google::Cloud::PubSub::V1::ListSchemasRequest object.
|
147
|
+
def self.from_grpc grpc_list, service, view, max = nil
|
148
|
+
subs = new(Array(grpc_list.schemas).map do |grpc|
|
149
|
+
Schema.from_grpc grpc, service
|
150
|
+
end)
|
151
|
+
token = grpc_list.next_page_token
|
152
|
+
token = nil if token == "".freeze
|
153
|
+
subs.instance_variable_set :@token, token
|
154
|
+
subs.instance_variable_set :@service, service
|
155
|
+
subs.instance_variable_set :@view, view
|
156
|
+
subs.instance_variable_set :@max, max
|
157
|
+
subs
|
158
|
+
end
|
159
|
+
|
160
|
+
protected
|
161
|
+
|
162
|
+
##
|
163
|
+
# @private Raise an error unless an active connection to the service
|
164
|
+
# is available.
|
165
|
+
def ensure_service!
|
166
|
+
raise "Must have active connection to service" unless @service
|
167
|
+
end
|
168
|
+
|
169
|
+
def next_schemas
|
170
|
+
options = { prefix: @prefix, token: @token, max: @max }
|
171
|
+
grpc = @service.list_schemas @view, options
|
172
|
+
self.class.from_grpc grpc, @service, @view, @max
|
173
|
+
end
|
174
|
+
end
|
175
|
+
end
|
176
|
+
end
|
177
|
+
|
178
|
+
Pubsub = PubSub unless const_defined? :Pubsub
|
179
|
+
end
|
180
|
+
end
|
@@ -49,49 +49,56 @@ module Google
|
|
49
49
|
|
50
50
|
def subscriber
|
51
51
|
return mocked_subscriber if mocked_subscriber
|
52
|
-
@subscriber ||=
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
config.metadata = { "google-cloud-resource-prefix": "projects/#{@project}" }
|
60
|
-
end
|
52
|
+
@subscriber ||= V1::Subscriber::Client.new do |config|
|
53
|
+
config.credentials = credentials if credentials
|
54
|
+
config.timeout = timeout if timeout
|
55
|
+
config.endpoint = host if host
|
56
|
+
config.lib_name = "gccl"
|
57
|
+
config.lib_version = Google::Cloud::PubSub::VERSION
|
58
|
+
config.metadata = { "google-cloud-resource-prefix": "projects/#{@project}" }
|
61
59
|
end
|
62
60
|
end
|
63
61
|
attr_accessor :mocked_subscriber
|
64
62
|
|
65
63
|
def publisher
|
66
64
|
return mocked_publisher if mocked_publisher
|
67
|
-
@publisher ||=
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
config.metadata = { "google-cloud-resource-prefix": "projects/#{@project}" }
|
75
|
-
end
|
65
|
+
@publisher ||= V1::Publisher::Client.new do |config|
|
66
|
+
config.credentials = credentials if credentials
|
67
|
+
config.timeout = timeout if timeout
|
68
|
+
config.endpoint = host if host
|
69
|
+
config.lib_name = "gccl"
|
70
|
+
config.lib_version = Google::Cloud::PubSub::VERSION
|
71
|
+
config.metadata = { "google-cloud-resource-prefix": "projects/#{@project}" }
|
76
72
|
end
|
77
73
|
end
|
78
74
|
attr_accessor :mocked_publisher
|
79
75
|
|
80
76
|
def iam
|
81
77
|
return mocked_iam if mocked_iam
|
82
|
-
@iam ||=
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
config.metadata = { "google-cloud-resource-prefix" => "projects/#{@project}" }
|
90
|
-
end
|
78
|
+
@iam ||= V1::IAMPolicy::Client.new do |config|
|
79
|
+
config.credentials = credentials if credentials
|
80
|
+
config.timeout = timeout if timeout
|
81
|
+
config.endpoint = host if host
|
82
|
+
config.lib_name = "gccl"
|
83
|
+
config.lib_version = Google::Cloud::PubSub::VERSION
|
84
|
+
config.metadata = { "google-cloud-resource-prefix" => "projects/#{@project}" }
|
91
85
|
end
|
92
86
|
end
|
93
87
|
attr_accessor :mocked_iam
|
94
88
|
|
89
|
+
def schemas
|
90
|
+
return mocked_schemas if mocked_schemas
|
91
|
+
@schemas ||= V1::SchemaService::Client.new do |config|
|
92
|
+
config.credentials = credentials if credentials
|
93
|
+
config.timeout = timeout if timeout
|
94
|
+
config.endpoint = host if host
|
95
|
+
config.lib_name = "gccl"
|
96
|
+
config.lib_version = Google::Cloud::PubSub::VERSION
|
97
|
+
config.metadata = { "google-cloud-resource-prefix" => "projects/#{@project}" }
|
98
|
+
end
|
99
|
+
end
|
100
|
+
attr_accessor :mocked_schemas
|
101
|
+
|
95
102
|
##
|
96
103
|
# Gets the configuration of a topic.
|
97
104
|
# Since the topic only has the name attribute,
|
@@ -114,18 +121,35 @@ module Google
|
|
114
121
|
|
115
122
|
##
|
116
123
|
# Creates the given topic with the given name.
|
117
|
-
def create_topic topic_name,
|
124
|
+
def create_topic topic_name,
|
125
|
+
labels: nil,
|
126
|
+
kms_key_name: nil,
|
127
|
+
persistence_regions: nil,
|
128
|
+
schema_name: nil,
|
129
|
+
message_encoding: nil,
|
130
|
+
options: {}
|
118
131
|
if persistence_regions
|
119
|
-
message_storage_policy =
|
132
|
+
message_storage_policy = Google::Cloud::PubSub::V1::MessageStoragePolicy.new(
|
120
133
|
allowed_persistence_regions: Array(persistence_regions)
|
121
|
-
|
134
|
+
)
|
135
|
+
end
|
136
|
+
|
137
|
+
if schema_name || message_encoding
|
138
|
+
unless schema_name && message_encoding
|
139
|
+
raise ArgumentError, "Schema settings must include both schema_name and message_encoding."
|
140
|
+
end
|
141
|
+
schema_settings = Google::Cloud::PubSub::V1::SchemaSettings.new(
|
142
|
+
schema: schema_path(schema_name),
|
143
|
+
encoding: message_encoding.to_s.upcase
|
144
|
+
)
|
122
145
|
end
|
123
146
|
|
124
147
|
publisher.create_topic \
|
125
148
|
name: topic_path(topic_name, options),
|
126
149
|
labels: labels,
|
127
150
|
kms_key_name: kms_key_name,
|
128
|
-
message_storage_policy: message_storage_policy
|
151
|
+
message_storage_policy: message_storage_policy,
|
152
|
+
schema_settings: schema_settings
|
129
153
|
end
|
130
154
|
|
131
155
|
def update_topic topic_obj, *fields
|
@@ -297,6 +321,89 @@ module Google
|
|
297
321
|
end
|
298
322
|
end
|
299
323
|
|
324
|
+
##
|
325
|
+
# Lists schemas in the current (or given) project.
|
326
|
+
# @param view [String, Symbol, nil] Possible values:
|
327
|
+
# * `BASIC` - Include the name and type of the schema, but not the definition.
|
328
|
+
# * `FULL` - Include all Schema object fields.
|
329
|
+
#
|
330
|
+
def list_schemas view, options = {}
|
331
|
+
schema_view = Google::Cloud::PubSub::V1::SchemaView.const_get view.to_s.upcase
|
332
|
+
paged_enum = schemas.list_schemas parent: project_path(options),
|
333
|
+
view: schema_view,
|
334
|
+
page_size: options[:max],
|
335
|
+
page_token: options[:token]
|
336
|
+
|
337
|
+
paged_enum.response
|
338
|
+
end
|
339
|
+
|
340
|
+
##
|
341
|
+
# Creates a schema in the current (or given) project.
|
342
|
+
def create_schema schema_id, type, definition, options = {}
|
343
|
+
schema = Google::Cloud::PubSub::V1::Schema.new(
|
344
|
+
type: type,
|
345
|
+
definition: definition
|
346
|
+
)
|
347
|
+
schemas.create_schema parent: project_path(options),
|
348
|
+
schema: schema,
|
349
|
+
schema_id: schema_id
|
350
|
+
end
|
351
|
+
|
352
|
+
##
|
353
|
+
# Gets the details of a schema.
|
354
|
+
# @param view [String, Symbol, nil] The set of fields to return in the response. Possible values:
|
355
|
+
# * `BASIC` - Include the name and type of the schema, but not the definition.
|
356
|
+
# * `FULL` - Include all Schema object fields.
|
357
|
+
#
|
358
|
+
def get_schema schema_name, view, options = {}
|
359
|
+
schema_view = Google::Cloud::PubSub::V1::SchemaView.const_get view.to_s.upcase
|
360
|
+
schemas.get_schema name: schema_path(schema_name, options),
|
361
|
+
view: schema_view
|
362
|
+
end
|
363
|
+
|
364
|
+
##
|
365
|
+
# Delete a schema.
|
366
|
+
def delete_schema schema_name
|
367
|
+
schemas.delete_schema name: schema_path(schema_name)
|
368
|
+
end
|
369
|
+
|
370
|
+
##
|
371
|
+
# Validate the definition string intended for a schema.
|
372
|
+
def validate_schema type, definition, options = {}
|
373
|
+
schema = Google::Cloud::PubSub::V1::Schema.new(
|
374
|
+
type: type,
|
375
|
+
definition: definition
|
376
|
+
)
|
377
|
+
schemas.validate_schema parent: project_path(options),
|
378
|
+
schema: schema
|
379
|
+
end
|
380
|
+
|
381
|
+
##
|
382
|
+
# Validates a message against a schema.
|
383
|
+
#
|
384
|
+
# @param message_data [String] Message to validate against the provided `schema_spec`.
|
385
|
+
# @param message_encoding [Google::Cloud::PubSub::V1::Encoding] The encoding expected for messages.
|
386
|
+
# @param schema_name [String] Name of the schema against which to validate.
|
387
|
+
# @param project [String] Name of the project if not the default project.
|
388
|
+
# @param type [String] Ad-hoc schema type against which to validate.
|
389
|
+
# @param definition [String] Ad-hoc schema definition against which to validate.
|
390
|
+
#
|
391
|
+
def validate_message message_data, message_encoding, schema_name: nil, project: nil, type: nil, definition: nil
|
392
|
+
if type && definition
|
393
|
+
schema = Google::Cloud::PubSub::V1::Schema.new(
|
394
|
+
type: type,
|
395
|
+
definition: definition
|
396
|
+
)
|
397
|
+
end
|
398
|
+
schemas.validate_message parent: project_path(project: project),
|
399
|
+
name: schema_path(schema_name),
|
400
|
+
schema: schema,
|
401
|
+
message: message_data,
|
402
|
+
encoding: message_encoding
|
403
|
+
end
|
404
|
+
|
405
|
+
# Helper methods
|
406
|
+
|
300
407
|
def get_topic_policy topic_name, options = {}
|
301
408
|
iam.get_iam_policy resource: topic_path(topic_name, options)
|
302
409
|
end
|
@@ -341,6 +448,11 @@ module Google
|
|
341
448
|
"#{project_path options}/snapshots/#{snapshot_name}"
|
342
449
|
end
|
343
450
|
|
451
|
+
def schema_path schema_name, options = {}
|
452
|
+
return schema_name if schema_name.nil? || schema_name.to_s.include?("/")
|
453
|
+
"#{project_path options}/schemas/#{schema_name}"
|
454
|
+
end
|
455
|
+
|
344
456
|
def inspect
|
345
457
|
"#<#{self.class.name} (#{@project})>"
|
346
458
|
end
|
@@ -128,28 +128,21 @@ module Google
|
|
128
128
|
|
129
129
|
##
|
130
130
|
# Immediately stops the subscriber. No new messages will be pulled from
|
131
|
-
# the subscription.
|
132
|
-
#
|
133
|
-
#
|
134
|
-
#
|
135
|
-
# received messages have been processed or released.
|
131
|
+
# the subscription. Use {#wait!} to block until all received messages have
|
132
|
+
# been processed or released: All actions taken on received messages that
|
133
|
+
# have not yet been sent to the API will be sent to the API. All received
|
134
|
+
# but unprocessed messages will be released back to the API and redelivered.
|
136
135
|
#
|
137
136
|
# @return [Subscriber] returns self so calls can be chained.
|
138
137
|
#
|
139
138
|
def stop
|
140
|
-
|
139
|
+
synchronize do
|
141
140
|
@started = false
|
142
141
|
@stopped = true
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
end
|
142
|
+
@stream_pool.map(&:stop)
|
143
|
+
wait_stop_buffer_thread!
|
144
|
+
self
|
147
145
|
end
|
148
|
-
stop_pool.map(&:join)
|
149
|
-
# Stop the buffer after the streams are all stopped
|
150
|
-
synchronize { @buffer.stop }
|
151
|
-
|
152
|
-
self
|
153
146
|
end
|
154
147
|
|
155
148
|
##
|
@@ -167,13 +160,8 @@ module Google
|
|
167
160
|
# @return [Subscriber] returns self so calls can be chained.
|
168
161
|
#
|
169
162
|
def wait! timeout = nil
|
170
|
-
|
171
|
-
|
172
|
-
Thread.new { stream.wait! timeout }
|
173
|
-
end
|
174
|
-
end
|
175
|
-
wait_pool.map(&:join)
|
176
|
-
|
163
|
+
wait_stop_buffer_thread!
|
164
|
+
@wait_stop_buffer_thread.join timeout
|
177
165
|
self
|
178
166
|
end
|
179
167
|
|
@@ -379,6 +367,18 @@ module Google
|
|
379
367
|
|
380
368
|
protected
|
381
369
|
|
370
|
+
##
|
371
|
+
# Starts a new thread to call wait! (blocking) on each Stream and then stop the TimedUnaryBuffer.
|
372
|
+
def wait_stop_buffer_thread!
|
373
|
+
synchronize do
|
374
|
+
@wait_stop_buffer_thread ||= Thread.new do
|
375
|
+
@stream_pool.map(&:wait!)
|
376
|
+
# Shutdown the buffer TimerTask (and flush the buffer) after the streams are all stopped.
|
377
|
+
@buffer.stop
|
378
|
+
end
|
379
|
+
end
|
380
|
+
end
|
381
|
+
|
382
382
|
def coerce_inventory inventory
|
383
383
|
@inventory = inventory
|
384
384
|
if @inventory.is_a? Hash
|