google-cloud-pubsub 1.0.2 → 2.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. checksums.yaml +4 -4
  2. data/AUTHENTICATION.md +16 -54
  3. data/CHANGELOG.md +464 -0
  4. data/CONTRIBUTING.md +328 -116
  5. data/EMULATOR.md +1 -1
  6. data/LOGGING.md +94 -2
  7. data/OVERVIEW.md +121 -68
  8. data/TROUBLESHOOTING.md +2 -8
  9. data/lib/google/cloud/pubsub/acknowledge_result.rb +79 -0
  10. data/lib/google/cloud/pubsub/async_publisher/batch.rb +319 -0
  11. data/lib/google/cloud/pubsub/async_publisher.rb +231 -156
  12. data/lib/google/cloud/pubsub/batch_publisher.rb +60 -30
  13. data/lib/google/cloud/pubsub/convert.rb +33 -7
  14. data/lib/google/cloud/pubsub/credentials.rb +2 -2
  15. data/lib/google/cloud/pubsub/errors.rb +93 -0
  16. data/lib/google/cloud/pubsub/flow_controller.rb +137 -0
  17. data/lib/google/cloud/pubsub/message.rb +45 -4
  18. data/lib/google/cloud/pubsub/policy.rb +3 -2
  19. data/lib/google/cloud/pubsub/project.rb +316 -49
  20. data/lib/google/cloud/pubsub/publish_result.rb +6 -1
  21. data/lib/google/cloud/pubsub/received_message.rb +171 -10
  22. data/lib/google/cloud/pubsub/retry_policy.rb +88 -0
  23. data/lib/google/cloud/pubsub/schema/list.rb +180 -0
  24. data/lib/google/cloud/pubsub/schema.rb +310 -0
  25. data/lib/google/cloud/pubsub/service.rb +285 -269
  26. data/lib/google/cloud/pubsub/snapshot/list.rb +4 -6
  27. data/lib/google/cloud/pubsub/snapshot.rb +5 -2
  28. data/lib/google/cloud/pubsub/subscriber/inventory.rb +69 -32
  29. data/lib/google/cloud/pubsub/subscriber/sequencer.rb +115 -0
  30. data/lib/google/cloud/pubsub/subscriber/stream.rb +108 -49
  31. data/lib/google/cloud/pubsub/subscriber/timed_unary_buffer.rb +191 -30
  32. data/lib/google/cloud/pubsub/subscriber.rb +155 -45
  33. data/lib/google/cloud/pubsub/subscription/list.rb +4 -6
  34. data/lib/google/cloud/pubsub/subscription/push_config.rb +55 -31
  35. data/lib/google/cloud/pubsub/subscription.rb +561 -77
  36. data/lib/google/cloud/pubsub/topic/list.rb +4 -6
  37. data/lib/google/cloud/pubsub/topic.rb +372 -52
  38. data/lib/google/cloud/pubsub/version.rb +1 -1
  39. data/lib/google/cloud/pubsub.rb +35 -46
  40. data/lib/google-cloud-pubsub.rb +21 -27
  41. metadata +26 -189
  42. data/lib/google/cloud/pubsub/v1/credentials.rb +0 -41
  43. data/lib/google/cloud/pubsub/v1/doc/google/iam/v1/iam_policy.rb +0 -21
  44. data/lib/google/cloud/pubsub/v1/doc/google/iam/v1/options.rb +0 -21
  45. data/lib/google/cloud/pubsub/v1/doc/google/iam/v1/policy.rb +0 -21
  46. data/lib/google/cloud/pubsub/v1/doc/google/protobuf/duration.rb +0 -91
  47. data/lib/google/cloud/pubsub/v1/doc/google/protobuf/empty.rb +0 -29
  48. data/lib/google/cloud/pubsub/v1/doc/google/protobuf/field_mask.rb +0 -222
  49. data/lib/google/cloud/pubsub/v1/doc/google/protobuf/timestamp.rb +0 -113
  50. data/lib/google/cloud/pubsub/v1/doc/google/pubsub/v1/pubsub.rb +0 -744
  51. data/lib/google/cloud/pubsub/v1/doc/google/type/expr.rb +0 -19
  52. data/lib/google/cloud/pubsub/v1/publisher_client.rb +0 -786
  53. data/lib/google/cloud/pubsub/v1/publisher_client_config.json +0 -105
  54. data/lib/google/cloud/pubsub/v1/subscriber_client.rb +0 -1385
  55. data/lib/google/cloud/pubsub/v1/subscriber_client_config.json +0 -138
  56. data/lib/google/cloud/pubsub/v1.rb +0 -17
  57. data/lib/google/pubsub/v1/pubsub_pb.rb +0 -249
  58. data/lib/google/pubsub/v1/pubsub_services_pb.rb +0 -211
@@ -0,0 +1,319 @@
1
+ # Copyright 2019 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+
16
+ require "monitor"
17
+ require "google/cloud/pubsub/errors"
18
+
19
+ module Google
20
+ module Cloud
21
+ module PubSub
22
+ class AsyncPublisher
23
+ ##
24
+ # @private
25
+ class Batch
26
+ include MonitorMixin
27
+
28
+ attr_reader :items
29
+ attr_reader :ordering_key
30
+
31
+ def initialize publisher, ordering_key
32
+ # init MonitorMixin
33
+ super()
34
+
35
+ @publisher = publisher
36
+ @ordering_key = ordering_key
37
+ @items = []
38
+ @queue = []
39
+ @default_message_bytes = publisher.topic_name.bytesize + 2
40
+ @total_message_bytes = @default_message_bytes
41
+ @publishing = false
42
+ @stopping = false
43
+ @canceled = false
44
+ end
45
+
46
+ ##
47
+ # Adds a message and callback to the batch.
48
+ #
49
+ # The method will indicate how the message is added. It will either be
50
+ # added to the active list of items, it will be queued to be picked up
51
+ # once the active publishing job has been completed, or it will
52
+ # indicate that the batch is full and a publishing job should be
53
+ # created.
54
+ #
55
+ # @param [Google::Cloud::PubSub::V1::PubsubMessage] msg The message.
56
+ # @param [Proc, nil] callback The callback.
57
+ #
58
+ # @return [Symbol] The state of the batch.
59
+ #
60
+ # * `:added` - Added to the active list of items to be published.
61
+ # * `:queued` - Batch is publishing, and the messsage is queued.
62
+ # * `:full` - Batch is full and ready to be published, and the
63
+ # message is queued.
64
+ #
65
+ def add msg, callback
66
+ synchronize do
67
+ raise AsyncPublisherStopped if @stopping
68
+ raise OrderingKeyError, @ordering_key if @canceled
69
+
70
+ if @publishing
71
+ queue_add msg, callback
72
+ :queued
73
+ elsif try_add msg, callback
74
+ :added
75
+ else
76
+ queue_add msg, callback
77
+ :full
78
+ end
79
+ end
80
+ end
81
+
82
+ ##
83
+ # Marks the batch to be published.
84
+ #
85
+ # The method will indicate whether a new publishing job should be
86
+ # started to publish the batch. See {publishing?}.
87
+ #
88
+ # @param [Boolean] stop Indicates whether the batch should also be
89
+ # marked for stopping, and any existing publish job should publish
90
+ # all items until the batch is empty.
91
+ #
92
+ # @return [Boolean] Returns whether a new publishing job should be
93
+ # started to publish the batch. If the batch is already being
94
+ # published then this will return `false`.
95
+ #
96
+ def publish! stop: nil
97
+ synchronize do
98
+ @stopping = true if stop
99
+
100
+ return false if @canceled
101
+
102
+ # If we are already publishing, do not indicate a new job needs to
103
+ # be started.
104
+ return false if @publishing
105
+
106
+ @publishing = !(@items.empty? && @queue.empty?)
107
+ end
108
+ end
109
+
110
+ ##
111
+ # Indicates whether the batch has an active publishing job.
112
+ #
113
+ # @return [Boolean]
114
+ #
115
+ def publishing?
116
+ # This probably does not need to be synchronized
117
+ @publishing
118
+ end
119
+
120
+ ##
121
+ # Indicates whether the batch has been stopped and all items will be
122
+ # published until the batch is empty.
123
+ #
124
+ # @return [Boolean]
125
+ #
126
+ def stopping?
127
+ # This does not need to be synchronized because nothing un-stops
128
+ @stopping
129
+ end
130
+
131
+ ##
132
+ # Fills the batch by sequentially moving the queued items that will
133
+ # fit into the active item list.
134
+ #
135
+ # This method is only intended to be used by the active publishing
136
+ # job.
137
+ #
138
+ def rebalance!
139
+ synchronize do
140
+ return [] if @canceled
141
+
142
+ until @queue.empty?
143
+ item = @queue.first
144
+ if try_add item.msg, item.callback, item.create_time
145
+ @queue.shift
146
+ next
147
+ end
148
+ break
149
+ end
150
+
151
+ @items
152
+ end
153
+ end
154
+
155
+ ##
156
+ # Resets the batch after a successful publish. This clears the active
157
+ # item list and moves the queued items that will fit into the active
158
+ # item list.
159
+ #
160
+ # If the batch has enough queued items to fill the batch again, the
161
+ # publishing job should continue to publish the reset batch until the
162
+ # batch indicated it should stop.
163
+ #
164
+ # This method is only intended to be used by the active publishing
165
+ # job.
166
+ #
167
+ # @return [Boolean] Whether the active publishing job should continue
168
+ # publishing after the reset.
169
+ #
170
+ def reset!
171
+ synchronize do
172
+ @items = []
173
+ @total_message_bytes = @default_message_bytes
174
+
175
+ if @canceled
176
+ @queue = []
177
+ @publishing = false
178
+ return false
179
+ end
180
+
181
+ refill_items
182
+
183
+ return false unless @publishing
184
+ if @items.empty?
185
+ @publishing = false
186
+ return false
187
+ else
188
+ return true if stopping?
189
+ return true if interval_met?(@items.first.create_time) || batch_full_by_count?
190
+ if @queue.empty?
191
+ @publishing = false
192
+ return false
193
+ end
194
+ end
195
+ end
196
+ true
197
+ end
198
+
199
+ ##
200
+ # Cancel the batch and hault futher batches until resumed. See
201
+ # {#resume!} and {#canceled?}.
202
+ #
203
+ # @return [Array<Item}] All items, including queued items
204
+ #
205
+ def cancel!
206
+ synchronize do
207
+ @canceled = true
208
+ @items + @queue
209
+ end
210
+ end
211
+
212
+ ##
213
+ # Resume the batch and proceed to publish messages. See {#cancel!} and
214
+ # {#canceled?}.
215
+ #
216
+ # @return [Boolean] Whether the batch was resumed.
217
+ #
218
+ def resume!
219
+ synchronize do
220
+ # Return false if the batch is not canceled
221
+ return false unless @canceled
222
+
223
+ @items = []
224
+ @queue = []
225
+ @total_message_bytes = @default_message_bytes
226
+ @publishing = false
227
+ @canceled = false
228
+ end
229
+ true
230
+ end
231
+
232
+ ##
233
+ # Indicates whether the batch has been canceled due to an error while
234
+ # publishing. See {#cancel!} and {#resume!}.
235
+ #
236
+ # @return [Boolean]
237
+ #
238
+ def canceled?
239
+ # This does not need to be synchronized because nothing un-stops
240
+ synchronize { @canceled }
241
+ end
242
+
243
+ ##
244
+ # Determines whether the batch is empty and ready to be culled.
245
+ #
246
+ def empty?
247
+ synchronize do
248
+ return false if @publishing || @canceled || @stopping
249
+
250
+ @items.empty? && @queue.empty?
251
+ end
252
+ end
253
+
254
+ def total_message_bytes
255
+ @total_message_bytes
256
+ end
257
+
258
+ protected
259
+
260
+ def interval_met? create_time
261
+ Time.now - create_time > @publisher.interval
262
+ end
263
+
264
+ def batch_full_by_count?
265
+ total_message_count == @publisher.max_messages
266
+ end
267
+
268
+ def refill_items
269
+ until @queue.empty?
270
+ item = @queue.first
271
+ added = try_add item.msg, item.callback, item.create_time
272
+ break unless added
273
+ @queue.shift
274
+ end
275
+ end
276
+
277
+ def items_add msg, callback, create_time
278
+ item = Item.new msg, callback, (create_time || Time.now)
279
+ @items << item
280
+ @total_message_bytes += item.bytesize + 2
281
+ end
282
+
283
+ def try_add msg, callback, create_time = nil
284
+ if @items.empty?
285
+ # Always add when empty, even if bytesize is bigger than total
286
+ items_add msg, callback, create_time
287
+ return true
288
+ end
289
+ new_message_count = total_message_count + 1
290
+ new_message_bytes = total_message_bytes + msg.to_proto.bytesize + 2
291
+ if new_message_count > @publisher.max_messages ||
292
+ new_message_bytes >= @publisher.max_bytes
293
+ return false
294
+ end
295
+ items_add msg, callback, create_time
296
+ true
297
+ end
298
+
299
+ def queue_add msg, callback
300
+ item = Item.new msg, callback, Time.now
301
+ @queue << item
302
+ end
303
+
304
+ def total_message_count
305
+ @items.count
306
+ end
307
+
308
+ Item = Struct.new :msg, :callback, :create_time do
309
+ def bytesize
310
+ msg.to_proto.bytesize
311
+ end
312
+ end
313
+ end
314
+ end
315
+ end
316
+
317
+ Pubsub = PubSub unless const_defined? :Pubsub
318
+ end
319
+ end