google-cloud-pubsub 2.4.0 → 2.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AUTHENTICATION.md +2 -1
- data/CHANGELOG.md +55 -0
- data/CONTRIBUTING.md +2 -3
- data/OVERVIEW.md +9 -6
- data/lib/google/cloud/pubsub.rb +0 -4
- data/lib/google/cloud/pubsub/async_publisher.rb +42 -6
- data/lib/google/cloud/pubsub/async_publisher/batch.rb +2 -6
- data/lib/google/cloud/pubsub/batch_publisher.rb +31 -30
- data/lib/google/cloud/pubsub/errors.rb +8 -0
- data/lib/google/cloud/pubsub/flow_controller.rb +139 -0
- data/lib/google/cloud/pubsub/project.rb +248 -11
- data/lib/google/cloud/pubsub/schema.rb +310 -0
- data/lib/google/cloud/pubsub/schema/list.rb +180 -0
- data/lib/google/cloud/pubsub/service.rb +143 -31
- data/lib/google/cloud/pubsub/snapshot.rb +1 -0
- data/lib/google/cloud/pubsub/subscriber.rb +22 -22
- data/lib/google/cloud/pubsub/subscription.rb +27 -25
- data/lib/google/cloud/pubsub/topic.rb +103 -3
- data/lib/google/cloud/pubsub/version.rb +1 -1
- metadata +20 -3
@@ -0,0 +1,180 @@
|
|
1
|
+
# Copyright 2021 Google LLC
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
|
16
|
+
require "delegate"
|
17
|
+
|
18
|
+
module Google
|
19
|
+
module Cloud
|
20
|
+
module PubSub
|
21
|
+
class Schema
|
22
|
+
##
|
23
|
+
# Schema::List is a special case Array with additional values.
|
24
|
+
class List < DelegateClass(::Array)
|
25
|
+
##
|
26
|
+
# If not empty, indicates that there are more schemas
|
27
|
+
# that match the request and this value should be passed to
|
28
|
+
# the next {Google::Cloud::PubSub::Project#schemas} to continue.
|
29
|
+
attr_accessor :token
|
30
|
+
|
31
|
+
##
|
32
|
+
# @private Create a new Schema::List with an array of values.
|
33
|
+
def initialize arr = []
|
34
|
+
@prefix = nil
|
35
|
+
@token = nil
|
36
|
+
@view = nil
|
37
|
+
@max = nil
|
38
|
+
super arr
|
39
|
+
end
|
40
|
+
|
41
|
+
##
|
42
|
+
# Whether there a next page of schemas.
|
43
|
+
#
|
44
|
+
# @return [Boolean]
|
45
|
+
#
|
46
|
+
# @example
|
47
|
+
# require "google/cloud/pubsub"
|
48
|
+
#
|
49
|
+
# pubsub = Google::Cloud::PubSub.new
|
50
|
+
#
|
51
|
+
# schemas = pubsub.schemas
|
52
|
+
# if schemas.next?
|
53
|
+
# next_schemas = schemas.next
|
54
|
+
# end
|
55
|
+
#
|
56
|
+
def next?
|
57
|
+
!token.nil?
|
58
|
+
end
|
59
|
+
|
60
|
+
##
|
61
|
+
# Retrieve the next page of schemas.
|
62
|
+
#
|
63
|
+
# @return [Schema::List]
|
64
|
+
#
|
65
|
+
# @example
|
66
|
+
# require "google/cloud/pubsub"
|
67
|
+
#
|
68
|
+
# pubsub = Google::Cloud::PubSub.new
|
69
|
+
#
|
70
|
+
# schemas = pubsub.schemas
|
71
|
+
# if schemas.next?
|
72
|
+
# next_schemas = schemas.next
|
73
|
+
# end
|
74
|
+
#
|
75
|
+
def next
|
76
|
+
return nil unless next?
|
77
|
+
ensure_service!
|
78
|
+
next_schemas
|
79
|
+
end
|
80
|
+
|
81
|
+
##
|
82
|
+
# Retrieves remaining results by repeatedly invoking {#next} until
|
83
|
+
# {#next?} returns `false`. Calls the given block once for each
|
84
|
+
# result, which is passed as the argument to the block.
|
85
|
+
#
|
86
|
+
# An Enumerator is returned if no block is given.
|
87
|
+
#
|
88
|
+
# This method will make repeated API calls until all remaining results
|
89
|
+
# are retrieved. (Unlike `#each`, for example, which merely iterates
|
90
|
+
# over the results returned by a single API call.) Use with caution.
|
91
|
+
#
|
92
|
+
# @param [Integer] request_limit The upper limit of API requests to
|
93
|
+
# make to load all schemas. Default is no limit.
|
94
|
+
# @yield [schema] The block for accessing each schema.
|
95
|
+
# @yieldparam [Schema] schema The schema object.
|
96
|
+
#
|
97
|
+
# @return [Enumerator]
|
98
|
+
#
|
99
|
+
# @example Iterating each schema by passing a block:
|
100
|
+
# require "google/cloud/pubsub"
|
101
|
+
#
|
102
|
+
# pubsub = Google::Cloud::PubSub.new
|
103
|
+
#
|
104
|
+
# schemas = pubsub.schemas
|
105
|
+
# schemas.all do |schema|
|
106
|
+
# puts schema.name
|
107
|
+
# end
|
108
|
+
#
|
109
|
+
# @example Using the enumerator by not passing a block:
|
110
|
+
# require "google/cloud/pubsub"
|
111
|
+
#
|
112
|
+
# pubsub = Google::Cloud::PubSub.new
|
113
|
+
#
|
114
|
+
# schemas = pubsub.schemas
|
115
|
+
# all_names = schemas.all.map do |schema|
|
116
|
+
# schema.name
|
117
|
+
# end
|
118
|
+
#
|
119
|
+
# @example Limit the number of API calls made:
|
120
|
+
# require "google/cloud/pubsub"
|
121
|
+
#
|
122
|
+
# pubsub = Google::Cloud::PubSub.new
|
123
|
+
#
|
124
|
+
# schemas = pubsub.schemas
|
125
|
+
# schemas.all(request_limit: 10) do |schema|
|
126
|
+
# puts schema.name
|
127
|
+
# end
|
128
|
+
#
|
129
|
+
def all request_limit: nil, &block
|
130
|
+
request_limit = request_limit.to_i if request_limit
|
131
|
+
return enum_for :all, request_limit: request_limit unless block_given?
|
132
|
+
results = self
|
133
|
+
loop do
|
134
|
+
results.each(&block)
|
135
|
+
if request_limit
|
136
|
+
request_limit -= 1
|
137
|
+
break if request_limit.negative?
|
138
|
+
end
|
139
|
+
break unless results.next?
|
140
|
+
results = results.next
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
##
|
145
|
+
# @private New Schemas::List from a
|
146
|
+
# Google::Cloud::PubSub::V1::ListSchemasRequest object.
|
147
|
+
def self.from_grpc grpc_list, service, view, max = nil
|
148
|
+
subs = new(Array(grpc_list.schemas).map do |grpc|
|
149
|
+
Schema.from_grpc grpc, service
|
150
|
+
end)
|
151
|
+
token = grpc_list.next_page_token
|
152
|
+
token = nil if token == "".freeze
|
153
|
+
subs.instance_variable_set :@token, token
|
154
|
+
subs.instance_variable_set :@service, service
|
155
|
+
subs.instance_variable_set :@view, view
|
156
|
+
subs.instance_variable_set :@max, max
|
157
|
+
subs
|
158
|
+
end
|
159
|
+
|
160
|
+
protected
|
161
|
+
|
162
|
+
##
|
163
|
+
# @private Raise an error unless an active connection to the service
|
164
|
+
# is available.
|
165
|
+
def ensure_service!
|
166
|
+
raise "Must have active connection to service" unless @service
|
167
|
+
end
|
168
|
+
|
169
|
+
def next_schemas
|
170
|
+
options = { prefix: @prefix, token: @token, max: @max }
|
171
|
+
grpc = @service.list_schemas @view, options
|
172
|
+
self.class.from_grpc grpc, @service, @view, @max
|
173
|
+
end
|
174
|
+
end
|
175
|
+
end
|
176
|
+
end
|
177
|
+
|
178
|
+
Pubsub = PubSub unless const_defined? :Pubsub
|
179
|
+
end
|
180
|
+
end
|
@@ -49,49 +49,56 @@ module Google
|
|
49
49
|
|
50
50
|
def subscriber
|
51
51
|
return mocked_subscriber if mocked_subscriber
|
52
|
-
@subscriber ||=
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
config.metadata = { "google-cloud-resource-prefix": "projects/#{@project}" }
|
60
|
-
end
|
52
|
+
@subscriber ||= V1::Subscriber::Client.new do |config|
|
53
|
+
config.credentials = credentials if credentials
|
54
|
+
config.timeout = timeout if timeout
|
55
|
+
config.endpoint = host if host
|
56
|
+
config.lib_name = "gccl"
|
57
|
+
config.lib_version = Google::Cloud::PubSub::VERSION
|
58
|
+
config.metadata = { "google-cloud-resource-prefix": "projects/#{@project}" }
|
61
59
|
end
|
62
60
|
end
|
63
61
|
attr_accessor :mocked_subscriber
|
64
62
|
|
65
63
|
def publisher
|
66
64
|
return mocked_publisher if mocked_publisher
|
67
|
-
@publisher ||=
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
config.metadata = { "google-cloud-resource-prefix": "projects/#{@project}" }
|
75
|
-
end
|
65
|
+
@publisher ||= V1::Publisher::Client.new do |config|
|
66
|
+
config.credentials = credentials if credentials
|
67
|
+
config.timeout = timeout if timeout
|
68
|
+
config.endpoint = host if host
|
69
|
+
config.lib_name = "gccl"
|
70
|
+
config.lib_version = Google::Cloud::PubSub::VERSION
|
71
|
+
config.metadata = { "google-cloud-resource-prefix": "projects/#{@project}" }
|
76
72
|
end
|
77
73
|
end
|
78
74
|
attr_accessor :mocked_publisher
|
79
75
|
|
80
76
|
def iam
|
81
77
|
return mocked_iam if mocked_iam
|
82
|
-
@iam ||=
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
config.metadata = { "google-cloud-resource-prefix" => "projects/#{@project}" }
|
90
|
-
end
|
78
|
+
@iam ||= V1::IAMPolicy::Client.new do |config|
|
79
|
+
config.credentials = credentials if credentials
|
80
|
+
config.timeout = timeout if timeout
|
81
|
+
config.endpoint = host if host
|
82
|
+
config.lib_name = "gccl"
|
83
|
+
config.lib_version = Google::Cloud::PubSub::VERSION
|
84
|
+
config.metadata = { "google-cloud-resource-prefix" => "projects/#{@project}" }
|
91
85
|
end
|
92
86
|
end
|
93
87
|
attr_accessor :mocked_iam
|
94
88
|
|
89
|
+
def schemas
|
90
|
+
return mocked_schemas if mocked_schemas
|
91
|
+
@schemas ||= V1::SchemaService::Client.new do |config|
|
92
|
+
config.credentials = credentials if credentials
|
93
|
+
config.timeout = timeout if timeout
|
94
|
+
config.endpoint = host if host
|
95
|
+
config.lib_name = "gccl"
|
96
|
+
config.lib_version = Google::Cloud::PubSub::VERSION
|
97
|
+
config.metadata = { "google-cloud-resource-prefix" => "projects/#{@project}" }
|
98
|
+
end
|
99
|
+
end
|
100
|
+
attr_accessor :mocked_schemas
|
101
|
+
|
95
102
|
##
|
96
103
|
# Gets the configuration of a topic.
|
97
104
|
# Since the topic only has the name attribute,
|
@@ -114,18 +121,35 @@ module Google
|
|
114
121
|
|
115
122
|
##
|
116
123
|
# Creates the given topic with the given name.
|
117
|
-
def create_topic topic_name,
|
124
|
+
def create_topic topic_name,
|
125
|
+
labels: nil,
|
126
|
+
kms_key_name: nil,
|
127
|
+
persistence_regions: nil,
|
128
|
+
schema_name: nil,
|
129
|
+
message_encoding: nil,
|
130
|
+
options: {}
|
118
131
|
if persistence_regions
|
119
|
-
message_storage_policy =
|
132
|
+
message_storage_policy = Google::Cloud::PubSub::V1::MessageStoragePolicy.new(
|
120
133
|
allowed_persistence_regions: Array(persistence_regions)
|
121
|
-
|
134
|
+
)
|
135
|
+
end
|
136
|
+
|
137
|
+
if schema_name || message_encoding
|
138
|
+
unless schema_name && message_encoding
|
139
|
+
raise ArgumentError, "Schema settings must include both schema_name and message_encoding."
|
140
|
+
end
|
141
|
+
schema_settings = Google::Cloud::PubSub::V1::SchemaSettings.new(
|
142
|
+
schema: schema_path(schema_name),
|
143
|
+
encoding: message_encoding.to_s.upcase
|
144
|
+
)
|
122
145
|
end
|
123
146
|
|
124
147
|
publisher.create_topic \
|
125
148
|
name: topic_path(topic_name, options),
|
126
149
|
labels: labels,
|
127
150
|
kms_key_name: kms_key_name,
|
128
|
-
message_storage_policy: message_storage_policy
|
151
|
+
message_storage_policy: message_storage_policy,
|
152
|
+
schema_settings: schema_settings
|
129
153
|
end
|
130
154
|
|
131
155
|
def update_topic topic_obj, *fields
|
@@ -297,6 +321,89 @@ module Google
|
|
297
321
|
end
|
298
322
|
end
|
299
323
|
|
324
|
+
##
|
325
|
+
# Lists schemas in the current (or given) project.
|
326
|
+
# @param view [String, Symbol, nil] Possible values:
|
327
|
+
# * `BASIC` - Include the name and type of the schema, but not the definition.
|
328
|
+
# * `FULL` - Include all Schema object fields.
|
329
|
+
#
|
330
|
+
def list_schemas view, options = {}
|
331
|
+
schema_view = Google::Cloud::PubSub::V1::SchemaView.const_get view.to_s.upcase
|
332
|
+
paged_enum = schemas.list_schemas parent: project_path(options),
|
333
|
+
view: schema_view,
|
334
|
+
page_size: options[:max],
|
335
|
+
page_token: options[:token]
|
336
|
+
|
337
|
+
paged_enum.response
|
338
|
+
end
|
339
|
+
|
340
|
+
##
|
341
|
+
# Creates a schema in the current (or given) project.
|
342
|
+
def create_schema schema_id, type, definition, options = {}
|
343
|
+
schema = Google::Cloud::PubSub::V1::Schema.new(
|
344
|
+
type: type,
|
345
|
+
definition: definition
|
346
|
+
)
|
347
|
+
schemas.create_schema parent: project_path(options),
|
348
|
+
schema: schema,
|
349
|
+
schema_id: schema_id
|
350
|
+
end
|
351
|
+
|
352
|
+
##
|
353
|
+
# Gets the details of a schema.
|
354
|
+
# @param view [String, Symbol, nil] The set of fields to return in the response. Possible values:
|
355
|
+
# * `BASIC` - Include the name and type of the schema, but not the definition.
|
356
|
+
# * `FULL` - Include all Schema object fields.
|
357
|
+
#
|
358
|
+
def get_schema schema_name, view, options = {}
|
359
|
+
schema_view = Google::Cloud::PubSub::V1::SchemaView.const_get view.to_s.upcase
|
360
|
+
schemas.get_schema name: schema_path(schema_name, options),
|
361
|
+
view: schema_view
|
362
|
+
end
|
363
|
+
|
364
|
+
##
|
365
|
+
# Delete a schema.
|
366
|
+
def delete_schema schema_name
|
367
|
+
schemas.delete_schema name: schema_path(schema_name)
|
368
|
+
end
|
369
|
+
|
370
|
+
##
|
371
|
+
# Validate the definition string intended for a schema.
|
372
|
+
def validate_schema type, definition, options = {}
|
373
|
+
schema = Google::Cloud::PubSub::V1::Schema.new(
|
374
|
+
type: type,
|
375
|
+
definition: definition
|
376
|
+
)
|
377
|
+
schemas.validate_schema parent: project_path(options),
|
378
|
+
schema: schema
|
379
|
+
end
|
380
|
+
|
381
|
+
##
|
382
|
+
# Validates a message against a schema.
|
383
|
+
#
|
384
|
+
# @param message_data [String] Message to validate against the provided `schema_spec`.
|
385
|
+
# @param message_encoding [Google::Cloud::PubSub::V1::Encoding] The encoding expected for messages.
|
386
|
+
# @param schema_name [String] Name of the schema against which to validate.
|
387
|
+
# @param project [String] Name of the project if not the default project.
|
388
|
+
# @param type [String] Ad-hoc schema type against which to validate.
|
389
|
+
# @param definition [String] Ad-hoc schema definition against which to validate.
|
390
|
+
#
|
391
|
+
def validate_message message_data, message_encoding, schema_name: nil, project: nil, type: nil, definition: nil
|
392
|
+
if type && definition
|
393
|
+
schema = Google::Cloud::PubSub::V1::Schema.new(
|
394
|
+
type: type,
|
395
|
+
definition: definition
|
396
|
+
)
|
397
|
+
end
|
398
|
+
schemas.validate_message parent: project_path(project: project),
|
399
|
+
name: schema_path(schema_name),
|
400
|
+
schema: schema,
|
401
|
+
message: message_data,
|
402
|
+
encoding: message_encoding
|
403
|
+
end
|
404
|
+
|
405
|
+
# Helper methods
|
406
|
+
|
300
407
|
def get_topic_policy topic_name, options = {}
|
301
408
|
iam.get_iam_policy resource: topic_path(topic_name, options)
|
302
409
|
end
|
@@ -341,6 +448,11 @@ module Google
|
|
341
448
|
"#{project_path options}/snapshots/#{snapshot_name}"
|
342
449
|
end
|
343
450
|
|
451
|
+
def schema_path schema_name, options = {}
|
452
|
+
return schema_name if schema_name.nil? || schema_name.to_s.include?("/")
|
453
|
+
"#{project_path options}/schemas/#{schema_name}"
|
454
|
+
end
|
455
|
+
|
344
456
|
def inspect
|
345
457
|
"#<#{self.class.name} (#{@project})>"
|
346
458
|
end
|
@@ -128,28 +128,21 @@ module Google
|
|
128
128
|
|
129
129
|
##
|
130
130
|
# Immediately stops the subscriber. No new messages will be pulled from
|
131
|
-
# the subscription.
|
132
|
-
#
|
133
|
-
#
|
134
|
-
#
|
135
|
-
# received messages have been processed or released.
|
131
|
+
# the subscription. Use {#wait!} to block until all received messages have
|
132
|
+
# been processed or released: All actions taken on received messages that
|
133
|
+
# have not yet been sent to the API will be sent to the API. All received
|
134
|
+
# but unprocessed messages will be released back to the API and redelivered.
|
136
135
|
#
|
137
136
|
# @return [Subscriber] returns self so calls can be chained.
|
138
137
|
#
|
139
138
|
def stop
|
140
|
-
|
139
|
+
synchronize do
|
141
140
|
@started = false
|
142
141
|
@stopped = true
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
end
|
142
|
+
@stream_pool.map(&:stop)
|
143
|
+
wait_stop_buffer_thread!
|
144
|
+
self
|
147
145
|
end
|
148
|
-
stop_pool.map(&:join)
|
149
|
-
# Stop the buffer after the streams are all stopped
|
150
|
-
synchronize { @buffer.stop }
|
151
|
-
|
152
|
-
self
|
153
146
|
end
|
154
147
|
|
155
148
|
##
|
@@ -167,13 +160,8 @@ module Google
|
|
167
160
|
# @return [Subscriber] returns self so calls can be chained.
|
168
161
|
#
|
169
162
|
def wait! timeout = nil
|
170
|
-
|
171
|
-
|
172
|
-
Thread.new { stream.wait! timeout }
|
173
|
-
end
|
174
|
-
end
|
175
|
-
wait_pool.map(&:join)
|
176
|
-
|
163
|
+
wait_stop_buffer_thread!
|
164
|
+
@wait_stop_buffer_thread.join timeout
|
177
165
|
self
|
178
166
|
end
|
179
167
|
|
@@ -379,6 +367,18 @@ module Google
|
|
379
367
|
|
380
368
|
protected
|
381
369
|
|
370
|
+
##
|
371
|
+
# Starts a new thread to call wait! (blocking) on each Stream and then stop the TimedUnaryBuffer.
|
372
|
+
def wait_stop_buffer_thread!
|
373
|
+
synchronize do
|
374
|
+
@wait_stop_buffer_thread ||= Thread.new do
|
375
|
+
@stream_pool.map(&:wait!)
|
376
|
+
# Shutdown the buffer TimerTask (and flush the buffer) after the streams are all stopped.
|
377
|
+
@buffer.stop
|
378
|
+
end
|
379
|
+
end
|
380
|
+
end
|
381
|
+
|
382
382
|
def coerce_inventory inventory
|
383
383
|
@inventory = inventory
|
384
384
|
if @inventory.is_a? Hash
|