google-cloud-pubsub 0.33.1 → 2.15.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/AUTHENTICATION.md +41 -40
- data/CHANGELOG.md +610 -0
- data/CONTRIBUTING.md +328 -116
- data/EMULATOR.md +2 -2
- data/LOGGING.md +95 -3
- data/OVERVIEW.md +183 -90
- data/TROUBLESHOOTING.md +2 -8
- data/lib/google/cloud/pubsub/acknowledge_result.rb +79 -0
- data/lib/google/cloud/pubsub/async_publisher/batch.rb +306 -0
- data/lib/google/cloud/pubsub/async_publisher.rb +270 -161
- data/lib/google/cloud/pubsub/batch_publisher.rb +65 -33
- data/lib/google/cloud/pubsub/convert.rb +36 -8
- data/lib/google/cloud/pubsub/credentials.rb +7 -5
- data/lib/google/cloud/pubsub/errors.rb +93 -0
- data/lib/google/cloud/pubsub/flow_controller.rb +139 -0
- data/lib/google/cloud/pubsub/message.rb +52 -7
- data/lib/google/cloud/pubsub/policy.rb +15 -12
- data/lib/google/cloud/pubsub/project.rb +341 -75
- data/lib/google/cloud/pubsub/publish_result.rb +9 -2
- data/lib/google/cloud/pubsub/received_message.rb +182 -20
- data/lib/google/cloud/pubsub/retry_policy.rb +88 -0
- data/lib/google/cloud/pubsub/schema/list.rb +180 -0
- data/lib/google/cloud/pubsub/schema.rb +310 -0
- data/lib/google/cloud/pubsub/service.rb +285 -258
- data/lib/google/cloud/pubsub/snapshot/list.rb +14 -14
- data/lib/google/cloud/pubsub/snapshot.rb +17 -12
- data/lib/google/cloud/pubsub/subscriber/enumerator_queue.rb +4 -4
- data/lib/google/cloud/pubsub/subscriber/inventory.rb +74 -33
- data/lib/google/cloud/pubsub/subscriber/sequencer.rb +115 -0
- data/lib/google/cloud/pubsub/subscriber/stream.rb +138 -91
- data/lib/google/cloud/pubsub/subscriber/timed_unary_buffer.rb +397 -0
- data/lib/google/cloud/pubsub/subscriber.rb +213 -51
- data/lib/google/cloud/pubsub/subscription/list.rb +16 -16
- data/lib/google/cloud/pubsub/subscription/push_config.rb +268 -0
- data/lib/google/cloud/pubsub/subscription.rb +827 -137
- data/lib/google/cloud/pubsub/topic/list.rb +14 -14
- data/lib/google/cloud/pubsub/topic.rb +565 -93
- data/lib/google/cloud/pubsub/version.rb +4 -2
- data/lib/google/cloud/pubsub.rb +50 -41
- data/lib/google-cloud-pubsub.rb +26 -29
- metadata +59 -53
- data/lib/google/cloud/pubsub/subscriber/async_stream_pusher.rb +0 -222
- data/lib/google/cloud/pubsub/subscriber/async_unary_pusher.rb +0 -270
- data/lib/google/cloud/pubsub/v1/credentials.rb +0 -39
- data/lib/google/cloud/pubsub/v1/doc/google/iam/v1/iam_policy.rb +0 -63
- data/lib/google/cloud/pubsub/v1/doc/google/iam/v1/policy.rb +0 -128
- data/lib/google/cloud/pubsub/v1/doc/google/protobuf/duration.rb +0 -91
- data/lib/google/cloud/pubsub/v1/doc/google/protobuf/empty.rb +0 -29
- data/lib/google/cloud/pubsub/v1/doc/google/protobuf/field_mask.rb +0 -230
- data/lib/google/cloud/pubsub/v1/doc/google/protobuf/timestamp.rb +0 -109
- data/lib/google/cloud/pubsub/v1/doc/google/pubsub/v1/pubsub.rb +0 -628
- data/lib/google/cloud/pubsub/v1/publisher_client.rb +0 -734
- data/lib/google/cloud/pubsub/v1/publisher_client_config.json +0 -105
- data/lib/google/cloud/pubsub/v1/subscriber_client.rb +0 -1267
- data/lib/google/cloud/pubsub/v1/subscriber_client_config.json +0 -144
- data/lib/google/cloud/pubsub/v1.rb +0 -17
- data/lib/google/pubsub/v1/pubsub_pb.rb +0 -222
- data/lib/google/pubsub/v1/pubsub_services_pb.rb +0 -192
@@ -0,0 +1,306 @@
|
|
1
|
+
# Copyright 2019 Google LLC
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
|
16
|
+
require "monitor"
|
17
|
+
require "google/cloud/pubsub/errors"
|
18
|
+
|
19
|
+
module Google
|
20
|
+
module Cloud
|
21
|
+
module PubSub
|
22
|
+
class AsyncPublisher
|
23
|
+
##
|
24
|
+
# @private
|
25
|
+
class Batch
|
26
|
+
include MonitorMixin
|
27
|
+
|
28
|
+
attr_reader :items
|
29
|
+
attr_reader :ordering_key
|
30
|
+
|
31
|
+
def initialize publisher, ordering_key
|
32
|
+
# init MonitorMixin
|
33
|
+
super()
|
34
|
+
|
35
|
+
@publisher = publisher
|
36
|
+
@ordering_key = ordering_key
|
37
|
+
@items = []
|
38
|
+
@queue = []
|
39
|
+
@default_message_bytes = publisher.topic_name.bytesize + 2
|
40
|
+
@total_message_bytes = @default_message_bytes
|
41
|
+
@publishing = false
|
42
|
+
@stopping = false
|
43
|
+
@canceled = false
|
44
|
+
end
|
45
|
+
|
46
|
+
##
|
47
|
+
# Adds a message and callback to the batch.
|
48
|
+
#
|
49
|
+
# The method will indicate how the message is added. It will either be
|
50
|
+
# added to the active list of items, it will be queued to be picked up
|
51
|
+
# once the active publishing job has been completed, or it will
|
52
|
+
# indicate that the batch is full and a publishing job should be
|
53
|
+
# created.
|
54
|
+
#
|
55
|
+
# @param [Google::Cloud::PubSub::V1::PubsubMessage] msg The message.
|
56
|
+
# @param [Proc, nil] callback The callback.
|
57
|
+
#
|
58
|
+
# @return [Symbol] The state of the batch.
|
59
|
+
#
|
60
|
+
# * `:added` - Added to the active list of items to be published.
|
61
|
+
# * `:queued` - Batch is publishing, and the messsage is queued.
|
62
|
+
# * `:full` - Batch is full and ready to be published, and the
|
63
|
+
# message is queued.
|
64
|
+
#
|
65
|
+
def add msg, callback
|
66
|
+
synchronize do
|
67
|
+
raise AsyncPublisherStopped if @stopping
|
68
|
+
raise OrderingKeyError, @ordering_key if @canceled
|
69
|
+
|
70
|
+
if @publishing
|
71
|
+
queue_add msg, callback
|
72
|
+
:queued
|
73
|
+
elsif try_add msg, callback
|
74
|
+
:added
|
75
|
+
else
|
76
|
+
queue_add msg, callback
|
77
|
+
:full
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
##
|
83
|
+
# Marks the batch to be published.
|
84
|
+
#
|
85
|
+
# The method will indicate whether a new publishing job should be
|
86
|
+
# started to publish the batch. See {publishing?}.
|
87
|
+
#
|
88
|
+
# @param [Boolean] stop Indicates whether the batch should also be
|
89
|
+
# marked for stopping, and any existing publish job should publish
|
90
|
+
# all items until the batch is empty.
|
91
|
+
#
|
92
|
+
# @return [Boolean] Returns whether a new publishing job should be
|
93
|
+
# started to publish the batch. If the batch is already being
|
94
|
+
# published then this will return `false`.
|
95
|
+
#
|
96
|
+
def publish! stop: nil
|
97
|
+
synchronize do
|
98
|
+
@stopping = true if stop
|
99
|
+
|
100
|
+
return false if @canceled
|
101
|
+
|
102
|
+
# If we are already publishing, do not indicate a new job needs to
|
103
|
+
# be started.
|
104
|
+
return false if @publishing
|
105
|
+
|
106
|
+
@publishing = !(@items.empty? && @queue.empty?)
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
##
|
111
|
+
# Indicates whether the batch has an active publishing job.
|
112
|
+
#
|
113
|
+
# @return [Boolean]
|
114
|
+
#
|
115
|
+
def publishing?
|
116
|
+
# This probably does not need to be synchronized
|
117
|
+
@publishing
|
118
|
+
end
|
119
|
+
|
120
|
+
##
|
121
|
+
# Indicates whether the batch has been stopped and all items will be
|
122
|
+
# published until the batch is empty.
|
123
|
+
#
|
124
|
+
# @return [Boolean]
|
125
|
+
#
|
126
|
+
def stopping?
|
127
|
+
# This does not need to be synchronized because nothing un-stops
|
128
|
+
@stopping
|
129
|
+
end
|
130
|
+
|
131
|
+
##
|
132
|
+
# Fills the batch by sequentially moving the queued items that will
|
133
|
+
# fit into the active item list.
|
134
|
+
#
|
135
|
+
# This method is only intended to be used by the active publishing
|
136
|
+
# job.
|
137
|
+
#
|
138
|
+
def rebalance!
|
139
|
+
synchronize do
|
140
|
+
return [] if @canceled
|
141
|
+
|
142
|
+
until @queue.empty?
|
143
|
+
item = @queue.first
|
144
|
+
if try_add item.msg, item.callback
|
145
|
+
@queue.shift
|
146
|
+
next
|
147
|
+
end
|
148
|
+
break
|
149
|
+
end
|
150
|
+
|
151
|
+
@items
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
##
|
156
|
+
# Resets the batch after a successful publish. This clears the active
|
157
|
+
# item list and moves the queued items that will fit into the active
|
158
|
+
# item list.
|
159
|
+
#
|
160
|
+
# If the batch has enough queued items to fill the batch again, the
|
161
|
+
# publishing job should continue to publish the reset batch until the
|
162
|
+
# batch indicated it should stop.
|
163
|
+
#
|
164
|
+
# This method is only intended to be used by the active publishing
|
165
|
+
# job.
|
166
|
+
#
|
167
|
+
# @return [Boolean] Whether the active publishing job should continue
|
168
|
+
# publishing after the reset.
|
169
|
+
#
|
170
|
+
def reset!
|
171
|
+
synchronize do
|
172
|
+
@items = []
|
173
|
+
@total_message_bytes = @default_message_bytes
|
174
|
+
|
175
|
+
if @canceled
|
176
|
+
@queue = []
|
177
|
+
@publishing = false
|
178
|
+
return false
|
179
|
+
end
|
180
|
+
|
181
|
+
until @queue.empty?
|
182
|
+
item = @queue.first
|
183
|
+
added = try_add item.msg, item.callback
|
184
|
+
break unless added
|
185
|
+
@queue.shift
|
186
|
+
end
|
187
|
+
|
188
|
+
return false unless @publishing
|
189
|
+
if @items.empty?
|
190
|
+
@publishing = false
|
191
|
+
return false
|
192
|
+
else
|
193
|
+
return true if stopping?
|
194
|
+
if @queue.empty?
|
195
|
+
@publishing = false
|
196
|
+
return false
|
197
|
+
end
|
198
|
+
end
|
199
|
+
end
|
200
|
+
true
|
201
|
+
end
|
202
|
+
|
203
|
+
##
|
204
|
+
# Cancel the batch and hault futher batches until resumed. See
|
205
|
+
# {#resume!} and {#canceled?}.
|
206
|
+
#
|
207
|
+
# @return [Array<Item}] All items, including queued items
|
208
|
+
#
|
209
|
+
def cancel!
|
210
|
+
synchronize do
|
211
|
+
@canceled = true
|
212
|
+
@items + @queue
|
213
|
+
end
|
214
|
+
end
|
215
|
+
|
216
|
+
##
|
217
|
+
# Resume the batch and proceed to publish messages. See {#cancel!} and
|
218
|
+
# {#canceled?}.
|
219
|
+
#
|
220
|
+
# @return [Boolean] Whether the batch was resumed.
|
221
|
+
#
|
222
|
+
def resume!
|
223
|
+
synchronize do
|
224
|
+
# Return false if the batch is not canceled
|
225
|
+
return false unless @canceled
|
226
|
+
|
227
|
+
@items = []
|
228
|
+
@queue = []
|
229
|
+
@total_message_bytes = @default_message_bytes
|
230
|
+
@publishing = false
|
231
|
+
@canceled = false
|
232
|
+
end
|
233
|
+
true
|
234
|
+
end
|
235
|
+
|
236
|
+
##
|
237
|
+
# Indicates whether the batch has been canceled due to an error while
|
238
|
+
# publishing. See {#cancel!} and {#resume!}.
|
239
|
+
#
|
240
|
+
# @return [Boolean]
|
241
|
+
#
|
242
|
+
def canceled?
|
243
|
+
# This does not need to be synchronized because nothing un-stops
|
244
|
+
synchronize { @canceled }
|
245
|
+
end
|
246
|
+
|
247
|
+
##
|
248
|
+
# Determines whether the batch is empty and ready to be culled.
|
249
|
+
#
|
250
|
+
def empty?
|
251
|
+
synchronize do
|
252
|
+
return false if @publishing || @canceled || @stopping
|
253
|
+
|
254
|
+
@items.empty? && @queue.empty?
|
255
|
+
end
|
256
|
+
end
|
257
|
+
|
258
|
+
def total_message_bytes
|
259
|
+
@total_message_bytes
|
260
|
+
end
|
261
|
+
|
262
|
+
protected
|
263
|
+
|
264
|
+
def items_add msg, callback
|
265
|
+
item = Item.new msg, callback
|
266
|
+
@items << item
|
267
|
+
@total_message_bytes += item.bytesize + 2
|
268
|
+
end
|
269
|
+
|
270
|
+
def try_add msg, callback
|
271
|
+
if @items.empty?
|
272
|
+
# Always add when empty, even if bytesize is bigger than total
|
273
|
+
items_add msg, callback
|
274
|
+
return true
|
275
|
+
end
|
276
|
+
new_message_count = total_message_count + 1
|
277
|
+
new_message_bytes = total_message_bytes + msg.to_proto.bytesize + 2
|
278
|
+
if new_message_count > @publisher.max_messages ||
|
279
|
+
new_message_bytes >= @publisher.max_bytes
|
280
|
+
return false
|
281
|
+
end
|
282
|
+
items_add msg, callback
|
283
|
+
true
|
284
|
+
end
|
285
|
+
|
286
|
+
def queue_add msg, callback
|
287
|
+
item = Item.new msg, callback
|
288
|
+
@queue << item
|
289
|
+
end
|
290
|
+
|
291
|
+
def total_message_count
|
292
|
+
@items.count
|
293
|
+
end
|
294
|
+
|
295
|
+
Item = Struct.new :msg, :callback do
|
296
|
+
def bytesize
|
297
|
+
msg.to_proto.bytesize
|
298
|
+
end
|
299
|
+
end
|
300
|
+
end
|
301
|
+
end
|
302
|
+
end
|
303
|
+
|
304
|
+
Pubsub = PubSub unless const_defined? :Pubsub
|
305
|
+
end
|
306
|
+
end
|