google-cloud-pubsub 2.6.0 → 2.8.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 3ad1e392a28fae7f97e78f0e5a73fcccb4cef83462b7ff6caac53c4e450cbf91
4
- data.tar.gz: 0d71eca6ba2b40ddb7d1855b2ea9bd4c75422aed74cc2f008cee1881fc850c43
3
+ metadata.gz: 5d702b4f68714fc0deb23b437d1df3f6a9f7d1d13c0e7b4f4fe40dcf7a48cae2
4
+ data.tar.gz: 1cd4c417d8e44532b10a69c1f6cb824b402ede9ba38731a29315c9b1a2743e16
5
5
  SHA512:
6
- metadata.gz: 4ff13cb0796a1fff747785b078ead8f2c0e71d56cfabdaed4156b83eb5bb800c8e35fbc84a4e722f51df8602ff6ca8b81ddfbeac51ada0eb0c7029e5258467d8
7
- data.tar.gz: '058a56e42f9f42366f709c86f1efe2cec8d1fe909ae2097ac2e7dc9a1e061376636bb7d109711ded9d84284c48b05bf739707b592e500ae8671fda76d22bbf88'
6
+ metadata.gz: 6a4b139e6a3170baeb92a8aea0e93196ebc39843deede789ad33545ef149aa2c789e49d5976b98b70954073f29dbfd29079067b6924797e3228f7b5a6fdfc77f
7
+ data.tar.gz: ca7cdcf23e8121cc27a0f1876dcccc74013adeec27995a95b342b821fc94cae09586f444b7f80e69991ebb01461f4f3a43364b15e54907972f5cde09582388e8
data/AUTHENTICATION.md CHANGED
@@ -96,7 +96,8 @@ client = Google::Cloud::PubSub.new
96
96
 
97
97
  ### Configuration
98
98
 
99
- The **Project ID** and **Credentials JSON** can be configured instead of placing them in environment variables or providing them as arguments.
99
+ The **Project ID** and the path to the **Credentials JSON** file can be configured
100
+ instead of placing them in environment variables or providing them as arguments.
100
101
 
101
102
  ```ruby
102
103
  require "google/cloud/pubsub"
data/CHANGELOG.md CHANGED
@@ -1,5 +1,42 @@
1
1
  # Release History
2
2
 
3
+ ### 2.8.0 / 2021-08-30
4
+
5
+ #### Features
6
+
7
+ * Add Pub/Sub topic retention fields
8
+ * Add retention to Project#create_topic
9
+ * Add Topic#retention
10
+ * Add Topic#retention=
11
+ * Add Subscription#topic_retention
12
+
13
+ ### 2.7.1 / 2021-07-08
14
+
15
+ #### Documentation
16
+
17
+ * Update AUTHENTICATION.md in handwritten packages
18
+
19
+ ### 2.7.0 / 2021-06-15
20
+
21
+ #### Features
22
+
23
+ * Add Publisher Flow Control
24
+ * Add flow_control to async options in Project#create_topic and Project#topic
25
+ * Add FlowControlLimitError
26
+
27
+ #### Bug Fixes
28
+
29
+ * Fix Project#schema and #schemas to return full resource
30
+ * Include schema definition in default return values.
31
+ * Fix Schema#definition to return nil instead of empty string when not present.
32
+
33
+ ### 2.6.1 / 2021-04-28
34
+
35
+ #### Bug Fixes
36
+
37
+ * Add final flush of pending requests to Subscriber#wait!
38
+ * fix(pubsub): Add final flush of pending requests to Subscriber#wait!
39
+
3
40
  ### 2.6.0 / 2021-04-19
4
41
 
5
42
  #### Features
@@ -129,8 +129,8 @@ module Google
129
129
  end
130
130
 
131
131
  ##
132
- # Rebalances the batch by moving any queued items that will fit into
133
- # the active item list.
132
+ # Fills the batch by sequentially moving the queued items that will
133
+ # fit into the active item list.
134
134
  #
135
135
  # This method is only intended to be used by the active publishing
136
136
  # job.
@@ -152,8 +152,6 @@ module Google
152
152
  end
153
153
  end
154
154
 
155
- # rubocop:disable Metrics/MethodLength
156
-
157
155
  ##
158
156
  # Resets the batch after a successful publish. This clears the active
159
157
  # item list and moves the queued items that will fit into the active
@@ -202,8 +200,6 @@ module Google
202
200
  true
203
201
  end
204
202
 
205
- # rubocop:enable Metrics/MethodLength
206
-
207
203
  ##
208
204
  # Cancel the batch and hault futher batches until resumed. See
209
205
  # {#resume!} and {#canceled?}.
@@ -16,6 +16,7 @@
16
16
  require "monitor"
17
17
  require "concurrent"
18
18
  require "google/cloud/pubsub/errors"
19
+ require "google/cloud/pubsub/flow_controller"
19
20
  require "google/cloud/pubsub/async_publisher/batch"
20
21
  require "google/cloud/pubsub/publish_result"
21
22
  require "google/cloud/pubsub/service"
@@ -65,14 +66,21 @@ module Google
65
66
  attr_reader :interval
66
67
  attr_reader :publish_threads
67
68
  attr_reader :callback_threads
69
+ attr_reader :flow_control
68
70
  ##
69
71
  # @private Implementation accessors
70
72
  attr_reader :service, :batch, :publish_thread_pool,
71
- :callback_thread_pool
73
+ :callback_thread_pool, :flow_controller
72
74
 
73
75
  ##
74
76
  # @private Create a new instance of the object.
75
- def initialize topic_name, service, max_bytes: 1_000_000, max_messages: 100, interval: 0.01, threads: {}
77
+ def initialize topic_name,
78
+ service,
79
+ max_bytes: 1_000_000,
80
+ max_messages: 100,
81
+ interval: 0.01,
82
+ threads: {},
83
+ flow_control: {}
76
84
  # init MonitorMixin
77
85
  super()
78
86
  @topic_name = service.topic_path topic_name
@@ -83,6 +91,10 @@ module Google
83
91
  @interval = interval
84
92
  @publish_threads = (threads[:publish] || 2).to_i
85
93
  @callback_threads = (threads[:callback] || 4).to_i
94
+ @flow_control = {
95
+ message_limit: 10 * @max_messages,
96
+ byte_limit: 10 * @max_bytes
97
+ }.merge(flow_control).freeze
86
98
 
87
99
  @published_at = nil
88
100
  @publish_thread_pool = Concurrent::ThreadPoolExecutor.new max_threads: @publish_threads
@@ -91,7 +103,7 @@ module Google
91
103
  @ordered = false
92
104
  @batches = {}
93
105
  @cond = new_cond
94
-
106
+ @flow_controller = FlowController.new(**@flow_control)
95
107
  @thread = Thread.new { run_background }
96
108
  end
97
109
 
@@ -121,13 +133,22 @@ module Google
121
133
  #
122
134
  def publish data = nil, attributes = nil, ordering_key: nil, **extra_attrs, &callback
123
135
  msg = Convert.pubsub_message data, attributes, ordering_key, extra_attrs
136
+ begin
137
+ @flow_controller.acquire msg.to_proto.bytesize
138
+ rescue FlowControlLimitError => e
139
+ stop_publish ordering_key, e if ordering_key
140
+ raise
141
+ end
124
142
 
125
143
  synchronize do
126
144
  raise AsyncPublisherStopped if @stopped
127
145
  raise OrderedMessagesDisabled if !@ordered && !msg.ordering_key.empty? # default is empty string
128
146
 
129
147
  batch = resolve_batch_for_message msg
130
- raise OrderingKeyError, batch.ordering_key if batch.canceled?
148
+ if batch.canceled?
149
+ @flow_controller.release msg.to_proto.bytesize
150
+ raise OrderingKeyError, batch.ordering_key
151
+ end
131
152
  batch_action = batch.add msg, callback
132
153
  if batch_action == :full
133
154
  publish_batches!
@@ -305,6 +326,21 @@ module Google
305
326
  @batches[ordering_key]
306
327
  end
307
328
 
329
+ def stop_publish ordering_key, err
330
+ synchronize do
331
+ batch = resolve_batch_for_ordering_key ordering_key
332
+ return if batch.nil?
333
+ items = batch.cancel!
334
+ items.each do |item|
335
+ @flow_controller.release item.bytesize
336
+ next unless item.callback
337
+
338
+ publish_result = PublishResult.from_error item.msg, err
339
+ execute_callback_async item.callback, publish_result
340
+ end
341
+ end
342
+ end
343
+
308
344
  def publish_batches! stop: nil
309
345
  @batches.reject! { |_ordering_key, batch| batch.empty? }
310
346
  @batches.each_value do |batch|
@@ -325,7 +361,6 @@ module Google
325
361
  end
326
362
 
327
363
  # rubocop:disable Metrics/AbcSize
328
- # rubocop:disable Metrics/MethodLength
329
364
 
330
365
  def publish_batch_sync topic_name, batch
331
366
  # The only batch methods that are safe to call from the loop are
@@ -337,6 +372,7 @@ module Google
337
372
  unless items.empty?
338
373
  grpc = @service.publish topic_name, items.map(&:msg)
339
374
  items.zip Array(grpc.message_ids) do |item, id|
375
+ @flow_controller.release item.bytesize
340
376
  next unless item.callback
341
377
 
342
378
  item.msg.message_id = id
@@ -363,6 +399,7 @@ module Google
363
399
  end
364
400
 
365
401
  items.each do |item|
402
+ @flow_controller.release item.bytesize
366
403
  next unless item.callback
367
404
 
368
405
  publish_result = PublishResult.from_error item.msg, e
@@ -374,7 +411,6 @@ module Google
374
411
  end
375
412
 
376
413
  # rubocop:enable Metrics/AbcSize
377
- # rubocop:enable Metrics/MethodLength
378
414
 
379
415
  PUBLISH_RETRY_ERRORS = [
380
416
  GRPC::Cancelled, GRPC::DeadlineExceeded, GRPC::Internal,
@@ -78,6 +78,14 @@ module Google
78
78
  super "Can't publish message using #{ordering_key}."
79
79
  end
80
80
  end
81
+
82
+ ##
83
+ # Raised when the desired action is `error` and the message would exceed
84
+ # flow control limits, or when the desired action is `block` and the
85
+ # message would block forever against the flow control limits.
86
+ #
87
+ class FlowControlLimitError < Google::Cloud::Error
88
+ end
81
89
  end
82
90
 
83
91
  Pubsub = PubSub unless const_defined? :Pubsub
@@ -0,0 +1,139 @@
1
+ # Copyright 2021 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+
16
+ require "google/cloud/pubsub/errors"
17
+ require "concurrent/atomics"
18
+
19
+ module Google
20
+ module Cloud
21
+ module PubSub
22
+ ##
23
+ # @private
24
+ #
25
+ # Used to control the flow of messages passing through it.
26
+ #
27
+ class FlowController
28
+ attr_reader :message_limit
29
+ attr_reader :byte_limit
30
+ attr_reader :limit_exceeded_behavior
31
+ ##
32
+ # @private Implementation accessors
33
+ attr_reader :outstanding_messages, :outstanding_bytes, :awaiting
34
+
35
+ def initialize message_limit: 1000, byte_limit: 10_000_000, limit_exceeded_behavior: :ignore
36
+ unless [:ignore, :error, :block].include? limit_exceeded_behavior
37
+ raise ArgumentError, "limit_exceeded_behavior must be one of :ignore, :error, :block"
38
+ end
39
+ if [:error, :block].include?(limit_exceeded_behavior) && message_limit < 1
40
+ raise ArgumentError,
41
+ "Flow control message limit (#{message_limit}) exceeded by a single message, would block forever"
42
+ end
43
+ @mutex = Mutex.new
44
+ @message_limit = message_limit
45
+ @byte_limit = byte_limit
46
+ @limit_exceeded_behavior = limit_exceeded_behavior
47
+ @outstanding_messages = 0
48
+ @outstanding_bytes = 0
49
+
50
+ @awaiting = []
51
+ end
52
+
53
+ def acquire message_size
54
+ return if limit_exceeded_behavior == :ignore
55
+ @mutex.lock
56
+ if limit_exceeded_behavior == :error && would_exceed_message_limit?
57
+ raise FlowControlLimitError, "Flow control message limit (#{message_limit}) would be exceeded"
58
+ end
59
+ if limit_exceeded_behavior == :error && would_exceed_byte_limit?(message_size)
60
+ raise FlowControlLimitError,
61
+ "Flow control byte limit (#{byte_limit}) would be exceeded, message_size: #{message_size}"
62
+ end
63
+ if limit_exceeded_behavior == :block && message_size > byte_limit
64
+ raise FlowControlLimitError,
65
+ "Flow control byte limit (#{byte_limit}) exceeded by a single message, would block forever"
66
+ end
67
+
68
+ acquire_or_wait message_size
69
+ ensure
70
+ @mutex.unlock if @mutex.owned?
71
+ end
72
+
73
+ def release message_size
74
+ return if limit_exceeded_behavior == :ignore
75
+ @mutex.synchronize do
76
+ raise "Flow control messages count would be negative" if (@outstanding_messages - 1).negative?
77
+ raise "Flow control bytes count would be negative" if (@outstanding_bytes - message_size).negative?
78
+
79
+ @outstanding_messages -= 1
80
+ @outstanding_bytes -= message_size
81
+ @awaiting.first.set unless @awaiting.empty?
82
+ end
83
+ end
84
+
85
+ protected
86
+
87
+ # rubocop:disable Style/IdenticalConditionalBranches
88
+ # rubocop:disable Style/GuardClause
89
+
90
+ def acquire_or_wait message_size
91
+ waiter = nil
92
+ while is_new_and_others_wait?(waiter) ||
93
+ would_exceed_byte_limit?(message_size) ||
94
+ would_exceed_message_limit?
95
+
96
+ if waiter.nil?
97
+ waiter = Concurrent::Event.new
98
+ # This waiter gets added to the back of the line.
99
+ @awaiting << waiter
100
+ else
101
+ waiter = Concurrent::Event.new
102
+ # This waiter already in line stays at the head of the line.
103
+ @awaiting[0] = waiter
104
+ end
105
+ @mutex.unlock
106
+ waiter.wait
107
+ @mutex.lock
108
+ end
109
+ @outstanding_messages += 1
110
+ @outstanding_bytes += message_size
111
+
112
+ @awaiting.shift if waiter # Remove the newly released waiter from the head of the queue.
113
+
114
+ # There may be some surplus left; let the next message waiting try to acquire a permit.
115
+ if !@awaiting.empty? && @outstanding_bytes < byte_limit && @outstanding_messages < message_limit
116
+ @awaiting.first.set
117
+ end
118
+ end
119
+
120
+ # rubocop:enable Style/IdenticalConditionalBranches
121
+ # rubocop:enable Style/GuardClause
122
+
123
+ def is_new_and_others_wait? waiter
124
+ waiter.nil? && !@awaiting.empty?
125
+ end
126
+
127
+ def would_exceed_message_limit?
128
+ @outstanding_messages + 1 > message_limit
129
+ end
130
+
131
+ def would_exceed_byte_limit? bytes_requested
132
+ @outstanding_bytes + bytes_requested > byte_limit
133
+ end
134
+ end
135
+ end
136
+
137
+ Pubsub = PubSub unless const_defined? :Pubsub
138
+ end
139
+ end
@@ -103,6 +103,16 @@ module Google
103
103
  # * `:threads` (Hash) The number of threads to create to handle concurrent calls by the publisher:
104
104
  # * `:publish` (Integer) The number of threads used to publish messages. Default is 2.
105
105
  # * `:callback` (Integer) The number of threads to handle the published messages' callbacks. Default is 4.
106
+ # * `:flow_control` (Hash) The client flow control settings for message publishing:
107
+ # * `:message_limit` (Integer) The maximum number of messages allowed to wait to be published. Default is
108
+ # `10 * max_messages`.
109
+ # * `:byte_limit` (Integer) The maximum total size of messages allowed to wait to be published. Default is
110
+ # `10 * max_bytes`.
111
+ # * `:limit_exceeded_behavior` (Symbol) The action to take when publish flow control limits are exceeded.
112
+ # Possible values include: `:ignore` - Flow control is disabled. `:error` - Calls to {Topic#publish_async}
113
+ # will raise {FlowControlLimitError} when publish flow control limits are exceeded. `:block` - Calls to
114
+ # {Topic#publish_async} will block until capacity is available when publish flow control limits are
115
+ # exceeded. The default value is `:ignore`.
106
116
  #
107
117
  # @return [Google::Cloud::PubSub::Topic, nil] Returns `nil` if topic
108
118
  # does not exist.
@@ -206,6 +216,16 @@ module Google
206
216
  # Default is 2.
207
217
  # * `:callback` (Integer) The number of threads to handle the published
208
218
  # messages' callbacks. Default is 4.
219
+ # * `:flow_control` (Hash) The client flow control settings for message publishing:
220
+ # * `:message_limit` (Integer) The maximum number of messages allowed to wait to be published. Default is
221
+ # `10 * max_messages`.
222
+ # * `:byte_limit` (Integer) The maximum total size of messages allowed to wait to be published. Default is
223
+ # `10 * max_bytes`.
224
+ # * `:limit_exceeded_behavior` (Symbol) The action to take when publish flow control limits are exceeded.
225
+ # Possible values include: `:ignore` - Flow control is disabled. `:error` - Calls to {Topic#publish_async}
226
+ # will raise {FlowControlLimitError} when publish flow control limits are exceeded. `:block` - Calls to
227
+ # {Topic#publish_async} will block until capacity is available when publish flow control limits are
228
+ # exceeded. The default value is `:ignore`.
209
229
  # @param [String] schema_name The name of the schema that messages
210
230
  # published should be validated against. Optional. The value can be a
211
231
  # simple schema ID (relative name), in which case the current project
@@ -218,6 +238,14 @@ module Google
218
238
  # * `JSON` - JSON encoding.
219
239
  # * `BINARY` - Binary encoding, as defined by the schema type. For some
220
240
  # schema types, binary encoding may not be available.
241
+ # @param [Numeric] retention Indicates the minimum number of seconds to retain a message
242
+ # after it is published to the topic. If this field is set, messages published
243
+ # to the topic within the `retention` number of seconds are always available to
244
+ # subscribers. For instance, it allows any attached subscription to [seek to a
245
+ # timestamp](https://cloud.google.com/pubsub/docs/replay-overview#seek_to_a_time)
246
+ # that is up to `retention` number of seconds in the past. If this field is
247
+ # not set, message retention is controlled by settings on individual
248
+ # subscriptions. Cannot be less than 600 (10 minutes) or more than 604,800 (7 days).
221
249
  #
222
250
  # @return [Google::Cloud::PubSub::Topic]
223
251
  #
@@ -233,14 +261,16 @@ module Google
233
261
  persistence_regions: nil,
234
262
  async: nil,
235
263
  schema_name: nil,
236
- message_encoding: nil
264
+ message_encoding: nil,
265
+ retention: nil
237
266
  ensure_service!
238
267
  grpc = service.create_topic topic_name,
239
268
  labels: labels,
240
269
  kms_key_name: kms_key,
241
270
  persistence_regions: persistence_regions,
242
271
  schema_name: schema_name,
243
- message_encoding: message_encoding
272
+ message_encoding: message_encoding,
273
+ retention: retention
244
274
  Topic.from_grpc grpc, service, async: async
245
275
  end
246
276
  alias new_topic create_topic
@@ -423,7 +453,7 @@ module Google
423
453
  # * `BASIC` - Include the `name` and `type` of the schema, but not the `definition`.
424
454
  # * `FULL` - Include all Schema object fields.
425
455
  #
426
- # The default value is `BASIC`.
456
+ # The default value is `FULL`.
427
457
  # @param [String] project If the schema belongs to a project other
428
458
  # than the one currently connected to, the alternate project ID can be
429
459
  # specified here. Not used if a fully-qualified schema name is
@@ -444,7 +474,7 @@ module Google
444
474
  # schema = pubsub.schema "my-schema"
445
475
  # schema.name #=> "projects/my-project/schemas/my-schema"
446
476
  # schema.type #=> :PROTOCOL_BUFFER
447
- # # schema.definition # nil - Use view: :full to load complete resource.
477
+ # schema.definition # The schema definition
448
478
  #
449
479
  # @example Skip the lookup against the service with `skip_lookup`:
450
480
  # require "google/cloud/pubsub"
@@ -458,21 +488,21 @@ module Google
458
488
  # schema.type #=> nil
459
489
  # schema.definition #=> nil
460
490
  #
461
- # @example Get the schema definition with `view: :full`:
491
+ # @example Omit the schema definition with `view: :basic`:
462
492
  # require "google/cloud/pubsub"
463
493
  #
464
494
  # pubsub = Google::Cloud::PubSub.new
465
495
  #
466
- # schema = pubsub.schema "my-schema", view: :full
496
+ # schema = pubsub.schema "my-schema", view: :basic
467
497
  # schema.name #=> "projects/my-project/schemas/my-schema"
468
498
  # schema.type #=> :PROTOCOL_BUFFER
469
- # schema.definition # The schema definition
499
+ # schema.definition #=> nil
470
500
  #
471
501
  def schema schema_name, view: nil, project: nil, skip_lookup: nil
472
502
  ensure_service!
473
503
  options = { project: project }
474
504
  return Schema.from_name schema_name, view, service, options if skip_lookup
475
- view ||= :BASIC
505
+ view ||= :FULL
476
506
  grpc = service.get_schema schema_name, view, options
477
507
  Schema.from_grpc grpc, service
478
508
  rescue Google::Cloud::NotFoundError
@@ -531,7 +561,7 @@ module Google
531
561
  # * `BASIC` - Include the `name` and `type` of the schema, but not the `definition`.
532
562
  # * `FULL` - Include all Schema object fields.
533
563
  #
534
- # The default value is `BASIC`.
564
+ # The default value is `FULL`.
535
565
  # @param [String] token A previously-returned page token representing
536
566
  # part of the larger set of results to view.
537
567
  # @param [Integer] max Maximum number of schemas to return.
@@ -561,7 +591,7 @@ module Google
561
591
  #
562
592
  def schemas view: nil, token: nil, max: nil
563
593
  ensure_service!
564
- view ||= :BASIC
594
+ view ||= :FULL
565
595
  options = { token: token, max: max }
566
596
  grpc = service.list_schemas view, options
567
597
  Schema::List.from_grpc grpc, service, view, max
@@ -48,7 +48,7 @@ module Google
48
48
  @grpc = grpc
49
49
  @service = service
50
50
  @exists = nil
51
- @view = view || :BASIC
51
+ @view = view || :FULL
52
52
  end
53
53
 
54
54
  ##
@@ -81,7 +81,7 @@ module Google
81
81
  #
82
82
  def definition
83
83
  return nil if reference?
84
- @grpc.definition
84
+ @grpc.definition if @grpc.definition && !@grpc.definition.empty?
85
85
  end
86
86
 
87
87
  ##
@@ -141,7 +141,7 @@ module Google
141
141
  # * `FULL` - Include all Schema object fields.
142
142
  #
143
143
  # Optional. If not provided or `nil`, the last non-nil `view` argument to this method will be used if one has
144
- # been given, othewise `BASIC` will be used.
144
+ # been given, othewise `FULL` will be used.
145
145
  #
146
146
  # @return [Google::Cloud::PubSub::Schema] Returns the reloaded schema.
147
147
  #
@@ -153,7 +153,7 @@ module Google
153
153
  #
154
154
  # schema.reload!
155
155
  #
156
- # @example Use the `view` option to load the full resource:
156
+ # @example Use the `view` option to load the basic or full resource:
157
157
  # require "google/cloud/pubsub"
158
158
  #
159
159
  # pubsub = Google::Cloud::PubSub.new
@@ -267,7 +267,7 @@ module Google
267
267
  # require "google/cloud/pubsub"
268
268
  #
269
269
  # pubsub = Google::Cloud::PubSub.new
270
- # schema = pubsub.schema "my-schema", view: :full
270
+ # schema = pubsub.schema "my-schema"
271
271
  #
272
272
  # schema.resource_full? #=> true
273
273
  #
@@ -127,6 +127,7 @@ module Google
127
127
  persistence_regions: nil,
128
128
  schema_name: nil,
129
129
  message_encoding: nil,
130
+ retention: nil,
130
131
  options: {}
131
132
  if persistence_regions
132
133
  message_storage_policy = Google::Cloud::PubSub::V1::MessageStoragePolicy.new(
@@ -145,11 +146,12 @@ module Google
145
146
  end
146
147
 
147
148
  publisher.create_topic \
148
- name: topic_path(topic_name, options),
149
- labels: labels,
150
- kms_key_name: kms_key_name,
151
- message_storage_policy: message_storage_policy,
152
- schema_settings: schema_settings
149
+ name: topic_path(topic_name, options),
150
+ labels: labels,
151
+ kms_key_name: kms_key_name,
152
+ message_storage_policy: message_storage_policy,
153
+ schema_settings: schema_settings,
154
+ message_retention_duration: Convert.number_to_duration(retention)
153
155
  end
154
156
 
155
157
  def update_topic topic_obj, *fields
@@ -128,28 +128,21 @@ module Google
128
128
 
129
129
  ##
130
130
  # Immediately stops the subscriber. No new messages will be pulled from
131
- # the subscription. All actions taken on received messages that have not
132
- # yet been sent to the API will be sent to the API. All received but
133
- # unprocessed messages will be released back to the API and redelivered.
134
- # Use {#wait!} to block until the subscriber is fully stopped and all
135
- # received messages have been processed or released.
131
+ # the subscription. Use {#wait!} to block until all received messages have
132
+ # been processed or released: All actions taken on received messages that
133
+ # have not yet been sent to the API will be sent to the API. All received
134
+ # but unprocessed messages will be released back to the API and redelivered.
136
135
  #
137
136
  # @return [Subscriber] returns self so calls can be chained.
138
137
  #
139
138
  def stop
140
- stop_pool = synchronize do
139
+ synchronize do
141
140
  @started = false
142
141
  @stopped = true
143
-
144
- @stream_pool.map do |stream|
145
- Thread.new { stream.stop }
146
- end
142
+ @stream_pool.map(&:stop)
143
+ wait_stop_buffer_thread!
144
+ self
147
145
  end
148
- stop_pool.map(&:join)
149
- # Stop the buffer after the streams are all stopped
150
- synchronize { @buffer.stop }
151
-
152
- self
153
146
  end
154
147
 
155
148
  ##
@@ -167,13 +160,8 @@ module Google
167
160
  # @return [Subscriber] returns self so calls can be chained.
168
161
  #
169
162
  def wait! timeout = nil
170
- wait_pool = synchronize do
171
- @stream_pool.map do |stream|
172
- Thread.new { stream.wait! timeout }
173
- end
174
- end
175
- wait_pool.map(&:join)
176
-
163
+ wait_stop_buffer_thread!
164
+ @wait_stop_buffer_thread.join timeout
177
165
  self
178
166
  end
179
167
 
@@ -379,6 +367,18 @@ module Google
379
367
 
380
368
  protected
381
369
 
370
+ ##
371
+ # Starts a new thread to call wait! (blocking) on each Stream and then stop the TimedUnaryBuffer.
372
+ def wait_stop_buffer_thread!
373
+ synchronize do
374
+ @wait_stop_buffer_thread ||= Thread.new do
375
+ @stream_pool.map(&:wait!)
376
+ # Shutdown the buffer TimerTask (and flush the buffer) after the streams are all stopped.
377
+ @buffer.stop
378
+ end
379
+ end
380
+ end
381
+
382
382
  def coerce_inventory inventory
383
383
  @inventory = inventory
384
384
  if @inventory.is_a? Hash
@@ -168,9 +168,8 @@ module Google
168
168
  # backlog, from the moment a message is published. If
169
169
  # {#retain_acked} is `true`, then this also configures the retention of
170
170
  # acknowledged messages, and thus configures how far back in time a
171
- # {#seek} can be done. Cannot be more than 604,800 seconds (7 days) or
172
- # less than 600 seconds (10 minutes). Default is 604,800 seconds (7
173
- # days).
171
+ # {#seek} can be done. Cannot be less than 600 (10 minutes) or more
172
+ # than 604,800 (7 days). Default is 604,800 seconds (7 days).
174
173
  #
175
174
  # Makes an API call to retrieve the retention value when called on a
176
175
  # reference object. See {#reference?}.
@@ -195,6 +194,24 @@ module Google
195
194
  @resource_name = nil
196
195
  end
197
196
 
197
+ ##
198
+ # Indicates the minimum duration for which a message is retained after
199
+ # it is published to the subscription's topic. If this field is set,
200
+ # messages published to the subscription's topic in the last
201
+ # `topic_message_retention_duration` are always available to subscribers.
202
+ # Output only. See {Topic#retention}.
203
+ #
204
+ # Makes an API call to retrieve the retention value when called on a
205
+ # reference object. See {#reference?}.
206
+ #
207
+ # @return [Numeric, nil] The topic message retention duration in seconds,
208
+ # or `nil` if not set.
209
+ #
210
+ def topic_retention
211
+ ensure_grpc!
212
+ Convert.duration_to_number @grpc.topic_message_retention_duration
213
+ end
214
+
198
215
  ##
199
216
  # Returns the URL locating the endpoint to which messages should be
200
217
  # pushed. For example, a Webhook endpoint might use
@@ -305,6 +305,43 @@ module Google
305
305
  message_encoding.to_s.upcase == "JSON"
306
306
  end
307
307
 
308
+ ##
309
+ # Indicates the minimum number of seconds to retain a message after it is
310
+ # published to the topic. If this field is set, messages published to the topic
311
+ # within the `retention` number of seconds are always available to subscribers.
312
+ # For instance, it allows any attached subscription to [seek to a
313
+ # timestamp](https://cloud.google.com/pubsub/docs/replay-overview#seek_to_a_time)
314
+ # that is up to `retention` number of seconds in the past. If this field is
315
+ # not set, message retention is controlled by settings on individual
316
+ # subscriptions. Cannot be less than 600 (10 minutes) or more than 604,800 (7 days).
317
+ # See {#retention=}.
318
+ #
319
+ # Makes an API call to retrieve the retention value when called on a
320
+ # reference object. See {#reference?}.
321
+ #
322
+ # @return [Numeric, nil] The message retention duration in seconds, or `nil` if not set.
323
+ #
324
+ def retention
325
+ ensure_grpc!
326
+ Convert.duration_to_number @grpc.message_retention_duration
327
+ end
328
+
329
+ ##
330
+ # Sets the message retention duration in seconds. If set to a positive duration
331
+ # between 600 (10 minutes) and 604,800 (7 days), inclusive, the message retention
332
+ # duration is changed. If set to `nil`, this clears message retention duration
333
+ # from the topic. See {#retention}.
334
+ #
335
+ # @param [Numeric, nil] new_retention The new message retention duration value.
336
+ #
337
+ def retention= new_retention
338
+ new_retention_duration = Convert.number_to_duration new_retention
339
+ update_grpc = Google::Cloud::PubSub::V1::Topic.new name: name,
340
+ message_retention_duration: new_retention_duration
341
+ @grpc = service.update_topic update_grpc, :message_retention_duration
342
+ @resource_name = nil
343
+ end
344
+
308
345
  ##
309
346
  # Permanently deletes the topic.
310
347
  #
@@ -684,6 +721,11 @@ module Google
684
721
  # @note At the time of this release, ordering keys are not yet publicly
685
722
  # enabled and requires special project enablements.
686
723
  #
724
+ # Publisher flow control limits the number of outstanding messages that
725
+ # are allowed to wait to be published. See the `flow_control` key in the
726
+ # `async` parameter in {Project#topic} for more information about publisher
727
+ # flow control settings.
728
+ #
687
729
  # @param [String, File] data The message payload. This will be converted
688
730
  # to bytes encoded as ASCII-8BIT.
689
731
  # @param [Hash] attributes Optional attributes for the message.
@@ -703,6 +745,13 @@ module Google
703
745
  # message with an `ordering_key` that has already failed when
704
746
  # publishing. Use {#resume_publish} to allow this `ordering_key` to be
705
747
  # published again.
748
+ # @raise [Google::Cloud::PubSub::FlowControlLimitError] when publish flow
749
+ # control limits are exceeded, and the `async` parameter key
750
+ # `flow_control.limit_exceeded_behavior` is set to `:error` or `:block`.
751
+ # If `flow_control.limit_exceeded_behavior` is set to `:block`, this error
752
+ # will be raised only when a limit would be exceeded by a single message.
753
+ # See the `async` parameter in {Project#topic} for more information about
754
+ # `flow_control` settings.
706
755
  #
707
756
  # @example
708
757
  # require "google/cloud/pubsub"
@@ -16,7 +16,7 @@
16
16
  module Google
17
17
  module Cloud
18
18
  module PubSub
19
- VERSION = "2.6.0".freeze
19
+ VERSION = "2.8.0".freeze
20
20
  end
21
21
 
22
22
  Pubsub = PubSub unless const_defined? :Pubsub
@@ -33,8 +33,6 @@ module Google
33
33
  # See {file:OVERVIEW.md Google Cloud Pub/Sub Overview}.
34
34
  #
35
35
  module PubSub
36
- # rubocop:disable Metrics/AbcSize
37
-
38
36
  ##
39
37
  # Creates a new object for connecting to the Pub/Sub service.
40
38
  # Each call creates a new connection.
@@ -110,8 +108,6 @@ module Google
110
108
  PubSub::Project.new service
111
109
  end
112
110
 
113
- # rubocop:enable Metrics/AbcSize
114
-
115
111
  ##
116
112
  # Configure the Google Cloud PubSub library.
117
113
  #
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: google-cloud-pubsub
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.6.0
4
+ version: 2.8.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Mike Moore
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2021-04-19 00:00:00.000000000 Z
12
+ date: 2021-08-31 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: concurrent-ruby
@@ -233,6 +233,7 @@ files:
233
233
  - lib/google/cloud/pubsub/convert.rb
234
234
  - lib/google/cloud/pubsub/credentials.rb
235
235
  - lib/google/cloud/pubsub/errors.rb
236
+ - lib/google/cloud/pubsub/flow_controller.rb
236
237
  - lib/google/cloud/pubsub/message.rb
237
238
  - lib/google/cloud/pubsub/policy.rb
238
239
  - lib/google/cloud/pubsub/project.rb
@@ -275,7 +276,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
275
276
  - !ruby/object:Gem::Version
276
277
  version: '0'
277
278
  requirements: []
278
- rubygems_version: 3.2.13
279
+ rubygems_version: 3.2.17
279
280
  signing_key:
280
281
  specification_version: 4
281
282
  summary: API Client library for Google Cloud Pub/Sub