google-cloud-pubsub 3.1.1 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/lib/google/cloud/pubsub/async_publisher.rb +11 -10
- data/lib/google/cloud/pubsub/batch_publisher.rb +3 -1
- data/lib/google/cloud/pubsub/internal_logger.rb +76 -0
- data/lib/google/cloud/pubsub/message_listener/inventory.rb +11 -6
- data/lib/google/cloud/pubsub/message_listener/stream.rb +51 -8
- data/lib/google/cloud/pubsub/message_listener/timed_unary_buffer.rb +28 -8
- data/lib/google/cloud/pubsub/message_listener.rb +1 -0
- data/lib/google/cloud/pubsub/publisher.rb +2 -1
- data/lib/google/cloud/pubsub/service.rb +12 -1
- data/lib/google/cloud/pubsub/version.rb +1 -1
- data/lib/google/cloud/pubsub.rb +13 -3
- data/lib/google-cloud-pubsub.rb +4 -0
- metadata +2 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 25a5cb88a31551bb6a0ebed4625038b802ec27efcdd8e239291a3ad240bb1713
|
|
4
|
+
data.tar.gz: 63f3c9c3c1662df084e34c519d669ecd6bfca0d44c99000cfc7c7ea417494e6b
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 6b8101c156e3199b5b4081f86c713f7d51f0de7034293afc37a569aade9ea4c83566ba2958f53bc28bf60f13bc7cac8ca06f6e3a0e7efb178b3862fe5c0635f4
|
|
7
|
+
data.tar.gz: 720cfeb029a5e21de019959b54653daecf05d96e0a1e1239a2f2c22b7d9d71b12287904144884d1555fd20f9218036ee95475e10db117dd845f428fcad56bf0d
|
data/CHANGELOG.md
CHANGED
|
@@ -157,7 +157,7 @@ module Google
|
|
|
157
157
|
end
|
|
158
158
|
batch_action = batch.add msg, callback
|
|
159
159
|
if batch_action == :full
|
|
160
|
-
publish_batches!
|
|
160
|
+
publish_batches! reason: "batch full"
|
|
161
161
|
elsif @published_at.nil?
|
|
162
162
|
# Set initial time to now to start the background counter
|
|
163
163
|
@published_at = Time.now
|
|
@@ -180,7 +180,7 @@ module Google
|
|
|
180
180
|
break if @stopped
|
|
181
181
|
|
|
182
182
|
@stopped = true
|
|
183
|
-
publish_batches! stop: true
|
|
183
|
+
publish_batches! stop: true, reason: "shutdown"
|
|
184
184
|
@cond.signal
|
|
185
185
|
@publish_thread_pool.shutdown
|
|
186
186
|
end
|
|
@@ -234,7 +234,7 @@ module Google
|
|
|
234
234
|
# @return [AsyncPublisher] returns self so calls can be chained.
|
|
235
235
|
def flush
|
|
236
236
|
synchronize do
|
|
237
|
-
publish_batches!
|
|
237
|
+
publish_batches! reason: "manual flush"
|
|
238
238
|
@cond.signal
|
|
239
239
|
end
|
|
240
240
|
|
|
@@ -313,7 +313,7 @@ module Google
|
|
|
313
313
|
time_since_first_publish = Time.now - @published_at
|
|
314
314
|
if time_since_first_publish > @interval
|
|
315
315
|
# interval met, flush the batches...
|
|
316
|
-
publish_batches!
|
|
316
|
+
publish_batches! reason: "interval timeout"
|
|
317
317
|
@cond.wait
|
|
318
318
|
else
|
|
319
319
|
# still waiting for the interval to publish the batch...
|
|
@@ -347,28 +347,28 @@ module Google
|
|
|
347
347
|
end
|
|
348
348
|
end
|
|
349
349
|
|
|
350
|
-
def publish_batches! stop: nil
|
|
350
|
+
def publish_batches! stop: nil, reason: "unknown"
|
|
351
351
|
@batches.reject! { |_ordering_key, batch| batch.empty? }
|
|
352
352
|
@batches.each_value do |batch|
|
|
353
353
|
ready = batch.publish! stop: stop
|
|
354
|
-
publish_batch_async @topic_name, batch if ready
|
|
354
|
+
publish_batch_async @topic_name, batch, reason: reason if ready
|
|
355
355
|
end
|
|
356
356
|
# Set published_at to nil to wait indefinitely
|
|
357
357
|
@published_at = nil
|
|
358
358
|
end
|
|
359
359
|
|
|
360
|
-
def publish_batch_async topic_name, batch
|
|
360
|
+
def publish_batch_async topic_name, batch, reason: "unknown"
|
|
361
361
|
# TODO: raise unless @publish_thread_pool.running?
|
|
362
362
|
return unless @publish_thread_pool.running?
|
|
363
363
|
|
|
364
364
|
Concurrent::Promises.future_on(
|
|
365
|
-
@publish_thread_pool, topic_name, batch
|
|
366
|
-
) { |t, b| publish_batch_sync t, b }
|
|
365
|
+
@publish_thread_pool, topic_name, batch, reason
|
|
366
|
+
) { |t, b, r| publish_batch_sync t, b, reason: r }
|
|
367
367
|
end
|
|
368
368
|
|
|
369
369
|
# rubocop:disable Metrics/AbcSize
|
|
370
370
|
|
|
371
|
-
def publish_batch_sync topic_name, batch
|
|
371
|
+
def publish_batch_sync topic_name, batch, reason: "unknown"
|
|
372
372
|
# The only batch methods that are safe to call from the loop are
|
|
373
373
|
# rebalance! and reset! because they are the only methods that are
|
|
374
374
|
# synchronized.
|
|
@@ -379,6 +379,7 @@ module Google
|
|
|
379
379
|
grpc = @service.publish topic_name,
|
|
380
380
|
items.map(&:msg),
|
|
381
381
|
compress: compress && batch.total_message_bytes >= compression_bytes_threshold
|
|
382
|
+
service.logger.log_batch "publish-batch", reason, "publish", items.count, items.sum(&:bytesize)
|
|
382
383
|
items.zip Array(grpc.message_ids) do |item, id|
|
|
383
384
|
@flow_controller.release item.bytesize
|
|
384
385
|
next unless item.callback
|
|
@@ -35,6 +35,7 @@ module Google
|
|
|
35
35
|
# end
|
|
36
36
|
#
|
|
37
37
|
class BatchPublisher
|
|
38
|
+
|
|
38
39
|
##
|
|
39
40
|
# @private The messages to publish
|
|
40
41
|
attr_reader :messages
|
|
@@ -117,10 +118,11 @@ module Google
|
|
|
117
118
|
|
|
118
119
|
##
|
|
119
120
|
# @private Call the publish API with arrays of data and attrs.
|
|
120
|
-
def publish_batch_messages topic_name, service
|
|
121
|
+
def publish_batch_messages topic_name, service, reason: "unknown"
|
|
121
122
|
grpc = service.publish topic_name,
|
|
122
123
|
messages,
|
|
123
124
|
compress: compress && total_message_bytes >= compression_bytes_threshold
|
|
125
|
+
service.logger.log_batch "publish-batch", reason, "publish", messages.count, @total_message_bytes
|
|
124
126
|
to_gcloud_messages Array(grpc.message_ids)
|
|
125
127
|
end
|
|
126
128
|
end
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# Copyright 2025 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
require "logger"
|
|
15
|
+
|
|
16
|
+
require "google/cloud/config"
|
|
17
|
+
|
|
18
|
+
module Google
|
|
19
|
+
module Cloud
|
|
20
|
+
module PubSub
|
|
21
|
+
##
|
|
22
|
+
# @private
|
|
23
|
+
class InternalLogger
|
|
24
|
+
LOG_NAME = "pubsub".freeze
|
|
25
|
+
VALID_LOG_LEVELS = [:debug, :info, :warn, :error, :fatal].freeze
|
|
26
|
+
private_constant :VALID_LOG_LEVELS, :LOG_NAME
|
|
27
|
+
|
|
28
|
+
##
|
|
29
|
+
# @private
|
|
30
|
+
# rubocop:disable Naming/BlockForwarding
|
|
31
|
+
def log level, subtag, &message_block
|
|
32
|
+
return unless VALID_LOG_LEVELS.include?(level) && block_given?
|
|
33
|
+
# Only log if the logger is explicitly tagged for 'pubsub'.
|
|
34
|
+
return unless @logger && @logger.progname == LOG_NAME
|
|
35
|
+
|
|
36
|
+
@logger.public_send(level, "#{LOG_NAME}:#{subtag}", &message_block)
|
|
37
|
+
end
|
|
38
|
+
# rubocop:enable Naming/BlockForwarding
|
|
39
|
+
|
|
40
|
+
##
|
|
41
|
+
# @private
|
|
42
|
+
def log_batch logger_name, reason, type, num_messages, total_bytes
|
|
43
|
+
log :info, logger_name do
|
|
44
|
+
"#{reason} triggered #{type} batch of #{num_messages} messages, a total of #{total_bytes} bytes"
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
##
|
|
49
|
+
# @private
|
|
50
|
+
def log_ack_nack ack_ids, type
|
|
51
|
+
ack_ids.each do |ack_id|
|
|
52
|
+
log :info, "ack-nack" do
|
|
53
|
+
"message (ackID #{ack_id}) #{type}"
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
##
|
|
59
|
+
# @private
|
|
60
|
+
def log_expiry expired
|
|
61
|
+
expired.each do |ack_id, item|
|
|
62
|
+
log :info, "expiry" do
|
|
63
|
+
"message (ID #{item.message_id}, ackID #{ack_id}) has been dropped from leasing due to a timeout"
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
private
|
|
69
|
+
|
|
70
|
+
def initialize logger
|
|
71
|
+
@logger = logger || Logger.new(nil)
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
@@ -12,7 +12,6 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
-
|
|
16
15
|
require "monitor"
|
|
17
16
|
|
|
18
17
|
module Google
|
|
@@ -22,9 +21,9 @@ module Google
|
|
|
22
21
|
##
|
|
23
22
|
# @private
|
|
24
23
|
class Inventory
|
|
25
|
-
InventoryItem = Struct.new :bytesize, :pulled_at do
|
|
24
|
+
InventoryItem = Struct.new :message_id, :bytesize, :pulled_at do
|
|
26
25
|
def self.from rec_msg
|
|
27
|
-
new rec_msg.to_proto.bytesize, Time.now
|
|
26
|
+
new rec_msg.message.message_id, rec_msg.to_proto.bytesize, Time.now
|
|
28
27
|
end
|
|
29
28
|
end
|
|
30
29
|
|
|
@@ -70,18 +69,24 @@ module Google
|
|
|
70
69
|
def remove *ack_ids
|
|
71
70
|
ack_ids.flatten!
|
|
72
71
|
ack_ids.compact!
|
|
73
|
-
return if ack_ids.empty?
|
|
72
|
+
return {} if ack_ids.empty?
|
|
74
73
|
|
|
74
|
+
removed_items = {}
|
|
75
75
|
synchronize do
|
|
76
|
-
@inventory.
|
|
76
|
+
removed, keep = @inventory.partition { |ack_id, _| ack_ids.include? ack_id }
|
|
77
|
+
@inventory = keep.to_h
|
|
78
|
+
removed_items = removed.to_h
|
|
77
79
|
@wait_cond.broadcast
|
|
78
80
|
end
|
|
81
|
+
removed_items
|
|
79
82
|
end
|
|
80
83
|
|
|
81
84
|
def remove_expired!
|
|
82
85
|
synchronize do
|
|
83
86
|
extension_time = Time.new - extension
|
|
84
|
-
@inventory.
|
|
87
|
+
expired, keep = @inventory.partition { |_ack_id, item| item.pulled_at < extension_time }
|
|
88
|
+
@inventory = keep.to_h
|
|
89
|
+
stream.subscriber.service.logger.log_expiry expired
|
|
85
90
|
@wait_cond.broadcast
|
|
86
91
|
end
|
|
87
92
|
end
|
|
@@ -12,7 +12,6 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
-
|
|
16
15
|
require "google/cloud/pubsub/message_listener/sequencer"
|
|
17
16
|
require "google/cloud/pubsub/message_listener/enumerator_queue"
|
|
18
17
|
require "google/cloud/pubsub/message_listener/inventory"
|
|
@@ -73,7 +72,12 @@ module Google
|
|
|
73
72
|
execution_interval: 30
|
|
74
73
|
) do
|
|
75
74
|
# push empty request every 30 seconds to keep stream alive
|
|
76
|
-
|
|
75
|
+
unless inventory.empty?
|
|
76
|
+
subscriber.service.logger.log :info, "subscriber-streams" do
|
|
77
|
+
"sending keepAlive to stream for subscription #{@subscriber.subscription_name}"
|
|
78
|
+
end
|
|
79
|
+
push Google::Cloud::PubSub::V1::StreamingPullRequest.new
|
|
80
|
+
end
|
|
77
81
|
end.execute
|
|
78
82
|
end
|
|
79
83
|
|
|
@@ -93,6 +97,9 @@ module Google
|
|
|
93
97
|
synchronize do
|
|
94
98
|
break if @stopped
|
|
95
99
|
|
|
100
|
+
subscriber.service.logger.log :info, "subscriber-streams" do
|
|
101
|
+
"stopping stream for subscription #{@subscriber.subscription_name}"
|
|
102
|
+
end
|
|
96
103
|
# Close the stream by pushing the sentinel value.
|
|
97
104
|
# The unary pusher does not use the stream, so it can close here.
|
|
98
105
|
@request_queue&.push self
|
|
@@ -138,8 +145,9 @@ module Google
|
|
|
138
145
|
ack_ids = coerce_ack_ids messages
|
|
139
146
|
return true if ack_ids.empty?
|
|
140
147
|
|
|
148
|
+
removed_items = {}
|
|
141
149
|
synchronize do
|
|
142
|
-
@inventory.remove ack_ids
|
|
150
|
+
removed_items = @inventory.remove ack_ids
|
|
143
151
|
@subscriber.buffer.acknowledge ack_ids, callback
|
|
144
152
|
end
|
|
145
153
|
|
|
@@ -152,8 +160,9 @@ module Google
|
|
|
152
160
|
mod_ack_ids = coerce_ack_ids messages
|
|
153
161
|
return true if mod_ack_ids.empty?
|
|
154
162
|
|
|
163
|
+
removed_items = {}
|
|
155
164
|
synchronize do
|
|
156
|
-
@inventory.remove mod_ack_ids
|
|
165
|
+
removed_items = @inventory.remove mod_ack_ids
|
|
157
166
|
@subscriber.buffer.modify_ack_deadline deadline, mod_ack_ids, callback
|
|
158
167
|
end
|
|
159
168
|
|
|
@@ -215,7 +224,13 @@ module Google
|
|
|
215
224
|
def background_run
|
|
216
225
|
synchronize do
|
|
217
226
|
# Don't allow a stream to restart if already stopped
|
|
218
|
-
|
|
227
|
+
if @stopped
|
|
228
|
+
subscriber.service.logger.log :debug, "subscriber-streams" do
|
|
229
|
+
"not filling stream for subscription #{@subscriber.subscription_name} because stream is already" \
|
|
230
|
+
" stopped"
|
|
231
|
+
end
|
|
232
|
+
return
|
|
233
|
+
end
|
|
219
234
|
|
|
220
235
|
@stopped = false
|
|
221
236
|
@paused = false
|
|
@@ -233,6 +248,9 @@ module Google
|
|
|
233
248
|
# Call the StreamingPull API to get the response enumerator
|
|
234
249
|
options = { :"metadata" => { :"x-goog-request-params" => @subscriber.subscription_name } }
|
|
235
250
|
enum = @subscriber.service.streaming_pull @request_queue.each, options
|
|
251
|
+
subscriber.service.logger.log :info, "subscriber-streams" do
|
|
252
|
+
"rpc: streamingPull, subscription: #{@subscriber.subscription_name}, stream opened"
|
|
253
|
+
end
|
|
236
254
|
|
|
237
255
|
loop do
|
|
238
256
|
synchronize do
|
|
@@ -287,13 +305,23 @@ module Google
|
|
|
287
305
|
stop
|
|
288
306
|
rescue GRPC::Cancelled, GRPC::DeadlineExceeded, GRPC::Internal,
|
|
289
307
|
GRPC::ResourceExhausted, GRPC::Unauthenticated,
|
|
290
|
-
GRPC::Unavailable
|
|
308
|
+
GRPC::Unavailable => e
|
|
309
|
+
status_code = e.respond_to?(:code) ? e.code : e.class.name
|
|
310
|
+
subscriber.service.logger.log :error, "subscriber-streams" do
|
|
311
|
+
"Subscriber stream for subscription #{@subscriber.subscription_name} has ended with status " \
|
|
312
|
+
"#{status_code}; will be retried."
|
|
313
|
+
end
|
|
291
314
|
# Restart the stream with an incremental back for a retriable error.
|
|
292
|
-
|
|
293
315
|
retry
|
|
294
316
|
rescue RestartStream
|
|
317
|
+
subscriber.service.logger.log :info, "subscriber-streams" do
|
|
318
|
+
"Subscriber stream for subscription #{@subscriber.subscription_name} has ended; will be retried."
|
|
319
|
+
end
|
|
295
320
|
retry
|
|
296
321
|
rescue StandardError => e
|
|
322
|
+
subscriber.service.logger.log :error, "subscriber-streams" do
|
|
323
|
+
"error on stream for subscription #{@subscriber.subscription_name}: #{e.inspect}"
|
|
324
|
+
end
|
|
297
325
|
@subscriber.error! e
|
|
298
326
|
|
|
299
327
|
retry
|
|
@@ -336,13 +364,22 @@ module Google
|
|
|
336
364
|
return unless callback_thread_pool.running?
|
|
337
365
|
|
|
338
366
|
Concurrent::Promises.future_on(
|
|
339
|
-
callback_thread_pool,
|
|
367
|
+
callback_thread_pool,
|
|
368
|
+
rec_msg,
|
|
369
|
+
&method(:perform_callback_sync)
|
|
340
370
|
)
|
|
341
371
|
end
|
|
342
372
|
|
|
343
373
|
def perform_callback_sync rec_msg
|
|
374
|
+
subscriber.service.logger.log :info, "callback-delivery" do
|
|
375
|
+
"message (ID #{rec_msg.message_id}, ackID #{rec_msg.ack_id}) delivery to user callbacks"
|
|
376
|
+
end
|
|
344
377
|
@subscriber.callback.call rec_msg unless stopped?
|
|
345
378
|
rescue StandardError => e
|
|
379
|
+
subscriber.service.logger.log :info, "callback-exceptions" do
|
|
380
|
+
"message (ID #{rec_msg.message_id}, ackID #{rec_msg.ack_id}) caused a user callback exception: " \
|
|
381
|
+
"#{e.inspect}"
|
|
382
|
+
end
|
|
346
383
|
@subscriber.error! e
|
|
347
384
|
ensure
|
|
348
385
|
release rec_msg
|
|
@@ -369,6 +406,9 @@ module Google
|
|
|
369
406
|
return unless pause_streaming?
|
|
370
407
|
|
|
371
408
|
@paused = true
|
|
409
|
+
subscriber.service.logger.log :info, "subscriber-flow-control" do
|
|
410
|
+
"subscriber for #{@subscriber.subscription_name} is client-side flow control blocked"
|
|
411
|
+
end
|
|
372
412
|
end
|
|
373
413
|
|
|
374
414
|
def pause_streaming?
|
|
@@ -382,6 +422,9 @@ module Google
|
|
|
382
422
|
return unless unpause_streaming?
|
|
383
423
|
|
|
384
424
|
@paused = nil
|
|
425
|
+
subscriber.service.logger.log :info, "subscriber-flow-control" do
|
|
426
|
+
"subscriber for #{@subscriber.subscription_name} is unblocking client-side flow control"
|
|
427
|
+
end
|
|
385
428
|
# signal to the background thread that we are unpaused
|
|
386
429
|
@pause_cond.broadcast
|
|
387
430
|
end
|
|
@@ -64,7 +64,7 @@ module Google
|
|
|
64
64
|
@retry_thread_pool = Concurrent::ThreadPoolExecutor.new max_threads: @subscriber.callback_threads
|
|
65
65
|
@callback_thread_pool = Concurrent::ThreadPoolExecutor.new max_threads: @subscriber.callback_threads
|
|
66
66
|
@task = Concurrent::TimerTask.new execution_interval: interval do
|
|
67
|
-
flush!
|
|
67
|
+
flush! reason: "interval timeout"
|
|
68
68
|
end
|
|
69
69
|
end
|
|
70
70
|
|
|
@@ -108,9 +108,9 @@ module Google
|
|
|
108
108
|
true
|
|
109
109
|
end
|
|
110
110
|
|
|
111
|
-
def flush!
|
|
111
|
+
def flush! reason: "manual flush"
|
|
112
112
|
# Grab requests from the buffer and release synchronize ASAP
|
|
113
|
-
requests = flush_requests!
|
|
113
|
+
requests = flush_requests! reason
|
|
114
114
|
return if requests.empty?
|
|
115
115
|
|
|
116
116
|
# Perform the RCP calls concurrently
|
|
@@ -167,7 +167,7 @@ module Google
|
|
|
167
167
|
@task.shutdown
|
|
168
168
|
@retry_thread_pool.shutdown
|
|
169
169
|
@callback_thread_pool.shutdown
|
|
170
|
-
flush!
|
|
170
|
+
flush! reason: "shutdown"
|
|
171
171
|
self
|
|
172
172
|
end
|
|
173
173
|
|
|
@@ -296,7 +296,9 @@ module Google
|
|
|
296
296
|
end
|
|
297
297
|
end
|
|
298
298
|
|
|
299
|
-
|
|
299
|
+
# rubocop:disable Metrics/AbcSize
|
|
300
|
+
|
|
301
|
+
def flush_requests! reason
|
|
300
302
|
prev_reg =
|
|
301
303
|
synchronize do
|
|
302
304
|
return {} if @register.empty?
|
|
@@ -308,16 +310,34 @@ module Google
|
|
|
308
310
|
groups = prev_reg.each_pair.group_by { |_ack_id, delay| delay }
|
|
309
311
|
req_hash = groups.transform_values { |v| v.map(&:first) }
|
|
310
312
|
|
|
311
|
-
requests = { acknowledge: [] }
|
|
313
|
+
requests = { acknowledge: [], modify_ack_deadline: [] }
|
|
312
314
|
ack_ids = Array(req_hash.delete(:ack)) # ack has no deadline set
|
|
313
|
-
|
|
315
|
+
if ack_ids.any?
|
|
316
|
+
requests[:acknowledge] = create_acknowledge_requests ack_ids
|
|
317
|
+
new_reason = if requests[:acknowledge].length > 1
|
|
318
|
+
"#{reason} and partitioned for exceeding max bytes"
|
|
319
|
+
else
|
|
320
|
+
reason
|
|
321
|
+
end
|
|
322
|
+
requests[:acknowledge].each do |req|
|
|
323
|
+
@subscriber.service.logger.log_batch "ack-batch", new_reason, "ack", req.ack_ids.length, req.to_proto.bytesize
|
|
324
|
+
end
|
|
325
|
+
end
|
|
314
326
|
requests[:modify_ack_deadline] =
|
|
315
327
|
req_hash.map do |mod_deadline, mod_ack_ids|
|
|
316
|
-
create_modify_ack_deadline_requests mod_deadline, mod_ack_ids
|
|
328
|
+
mod_ack_reqs = create_modify_ack_deadline_requests mod_deadline, mod_ack_ids
|
|
329
|
+
type = mod_deadline.zero? ? "nack" : "modack"
|
|
330
|
+
new_reason = mod_ack_reqs.length > 1 ? "#{reason} and partitioned for exceeding max bytes" : reason
|
|
331
|
+
mod_ack_reqs.each do |req|
|
|
332
|
+
@subscriber.service.logger.log_batch "ack-batch", new_reason, type, req.ack_ids.length, req.to_proto.bytesize
|
|
333
|
+
end
|
|
334
|
+
mod_ack_reqs
|
|
317
335
|
end.flatten
|
|
318
336
|
requests
|
|
319
337
|
end
|
|
320
338
|
|
|
339
|
+
# rubocop:enable Metrics/AbcSize
|
|
340
|
+
|
|
321
341
|
def create_acknowledge_requests ack_ids
|
|
322
342
|
req = Google::Cloud::PubSub::V1::AcknowledgeRequest.new(
|
|
323
343
|
subscription: subscription_name,
|
|
@@ -223,7 +223,8 @@ module Google
|
|
|
223
223
|
|
|
224
224
|
block&.call batch
|
|
225
225
|
return nil if batch.messages.count.zero?
|
|
226
|
-
|
|
226
|
+
reason = block_given? ? "synchronous publish multiple" : "synchronous publish single"
|
|
227
|
+
batch.publish_batch_messages name, service, reason: reason
|
|
227
228
|
end
|
|
228
229
|
|
|
229
230
|
##
|
|
@@ -19,6 +19,7 @@ require "google/cloud/pubsub/convert"
|
|
|
19
19
|
require "google/cloud/pubsub/version"
|
|
20
20
|
require "google/cloud/pubsub/v1"
|
|
21
21
|
require "google/cloud/pubsub/admin_clients"
|
|
22
|
+
require "google/cloud/pubsub/internal_logger"
|
|
22
23
|
require "securerandom"
|
|
23
24
|
|
|
24
25
|
module Google
|
|
@@ -40,15 +41,20 @@ module Google
|
|
|
40
41
|
|
|
41
42
|
attr_reader :universe_domain
|
|
42
43
|
|
|
44
|
+
##
|
|
45
|
+
# @private The InternalLogger object.
|
|
46
|
+
attr_reader :logger
|
|
47
|
+
|
|
43
48
|
##
|
|
44
49
|
# Creates a new Service instance.
|
|
45
|
-
def initialize project, credentials, host: nil, timeout: nil, universe_domain: nil
|
|
50
|
+
def initialize project, credentials, host: nil, timeout: nil, universe_domain: nil, logger: nil
|
|
46
51
|
@project = project
|
|
47
52
|
@credentials = credentials
|
|
48
53
|
@host = host
|
|
49
54
|
@timeout = timeout
|
|
50
55
|
@client_id = SecureRandom.uuid.freeze
|
|
51
56
|
@universe_domain = universe_domain || ENV["GOOGLE_CLOUD_UNIVERSE_DOMAIN"] || "googleapis.com"
|
|
57
|
+
@logger = logger
|
|
52
58
|
end
|
|
53
59
|
|
|
54
60
|
def subscription_admin
|
|
@@ -141,6 +147,7 @@ module Google
|
|
|
141
147
|
##
|
|
142
148
|
# Acknowledges receipt of a message.
|
|
143
149
|
def acknowledge subscription, *ack_ids
|
|
150
|
+
logger.log_ack_nack ack_ids, "ack"
|
|
144
151
|
subscription_admin.acknowledge_internal subscription: subscription_path(subscription),
|
|
145
152
|
ack_ids: ack_ids
|
|
146
153
|
end
|
|
@@ -148,6 +155,9 @@ module Google
|
|
|
148
155
|
##
|
|
149
156
|
# Modifies the ack deadline for a specific message.
|
|
150
157
|
def modify_ack_deadline subscription, ids, deadline
|
|
158
|
+
if deadline.zero?
|
|
159
|
+
logger.log_ack_nack Array(ids), "nack"
|
|
160
|
+
end
|
|
151
161
|
subscription_admin.modify_ack_deadline_internal subscription: subscription_path(subscription),
|
|
152
162
|
ack_ids: Array(ids),
|
|
153
163
|
ack_deadline_seconds: deadline
|
|
@@ -193,6 +203,7 @@ module Google
|
|
|
193
203
|
rpc.timeout = timeout if rpc.respond_to? :timeout=
|
|
194
204
|
end
|
|
195
205
|
end
|
|
206
|
+
|
|
196
207
|
end
|
|
197
208
|
end
|
|
198
209
|
Pubsub = PubSub unless const_defined? :Pubsub
|
data/lib/google/cloud/pubsub.rb
CHANGED
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
require "google-cloud-pubsub"
|
|
17
|
+
require "google/cloud/pubsub/internal_logger"
|
|
17
18
|
require "google/cloud/pubsub/project"
|
|
18
19
|
require "google/cloud/config"
|
|
19
20
|
require "google/cloud/env"
|
|
@@ -70,11 +71,16 @@ module Google
|
|
|
70
71
|
#
|
|
71
72
|
# * `https://www.googleapis.com/auth/pubsub`
|
|
72
73
|
# @param [Numeric] timeout Default timeout to use in requests. Optional.
|
|
74
|
+
# @param [String] universe_domain A custom universe domain. Optional.
|
|
73
75
|
# @param [String] endpoint Override of the endpoint host name. Optional.
|
|
74
76
|
# If the param is nil, uses the default endpoint.
|
|
75
77
|
# @param [String] emulator_host Pub/Sub emulator host. Optional.
|
|
76
78
|
# If the param is nil, uses the value of the `emulator_host` config.
|
|
77
|
-
# @param
|
|
79
|
+
# @param [Logger] logger Optional Logger instance for emitting
|
|
80
|
+
# library-level debug logs. If not provided, it will default to
|
|
81
|
+
# configure.logger, which defaults to Logger.new STDOUT if not set. To
|
|
82
|
+
# enable logging, set environment variable GOOGLE_SDK_RUBY_LOGGING_GEMS
|
|
83
|
+
# to "all" or a comma separated list of gem names, including "pubsub".
|
|
78
84
|
#
|
|
79
85
|
# @return [Google::Cloud::PubSub::Project]
|
|
80
86
|
#
|
|
@@ -92,13 +98,15 @@ module Google
|
|
|
92
98
|
timeout: nil,
|
|
93
99
|
universe_domain: nil,
|
|
94
100
|
endpoint: nil,
|
|
95
|
-
emulator_host: nil
|
|
101
|
+
emulator_host: nil,
|
|
102
|
+
logger: nil
|
|
96
103
|
project_id ||= default_project_id
|
|
97
104
|
scope ||= configure.scope
|
|
98
105
|
timeout ||= configure.timeout
|
|
99
106
|
endpoint ||= configure.endpoint
|
|
100
107
|
universe_domain ||= configure.universe_domain
|
|
101
108
|
emulator_host ||= configure.emulator_host
|
|
109
|
+
logger ||= configure.logger
|
|
102
110
|
|
|
103
111
|
if emulator_host
|
|
104
112
|
credentials = :this_channel_is_insecure
|
|
@@ -114,10 +122,12 @@ module Google
|
|
|
114
122
|
project_id = project_id.to_s # Always cast to a string
|
|
115
123
|
raise ArgumentError, "project_id is missing" if project_id.empty?
|
|
116
124
|
|
|
125
|
+
logger = Google::Cloud::PubSub::InternalLogger.new logger
|
|
117
126
|
service = PubSub::Service.new project_id, credentials,
|
|
118
127
|
host: endpoint,
|
|
119
128
|
timeout: timeout,
|
|
120
|
-
universe_domain: universe_domain
|
|
129
|
+
universe_domain: universe_domain,
|
|
130
|
+
logger: logger
|
|
121
131
|
PubSub::Project.new service
|
|
122
132
|
end
|
|
123
133
|
|
data/lib/google-cloud-pubsub.rb
CHANGED
|
@@ -23,6 +23,7 @@ gem "google-cloud-core"
|
|
|
23
23
|
require "google/cloud" unless defined? Google::Cloud.new
|
|
24
24
|
require "google/cloud/config"
|
|
25
25
|
require "googleauth"
|
|
26
|
+
require "logger"
|
|
26
27
|
|
|
27
28
|
module Google
|
|
28
29
|
module Cloud
|
|
@@ -142,6 +143,8 @@ Google::Cloud.configure.add_config! :pubsub do |config| # rubocop:disable Metric
|
|
|
142
143
|
"https://www.googleapis.com/auth/pubsub"
|
|
143
144
|
]
|
|
144
145
|
|
|
146
|
+
default_logger = Logger.new $stdout
|
|
147
|
+
|
|
145
148
|
config.add_field! :project_id, default_project, match: String, allow_nil: true
|
|
146
149
|
config.add_alias! :project, :project_id
|
|
147
150
|
config.add_field! :credentials, default_creds, match: [String, Hash, Google::Auth::Credentials], allow_nil: true
|
|
@@ -153,4 +156,5 @@ Google::Cloud.configure.add_config! :pubsub do |config| # rubocop:disable Metric
|
|
|
153
156
|
config.add_field! :on_error, nil, match: Proc
|
|
154
157
|
config.add_field! :endpoint, nil, match: String
|
|
155
158
|
config.add_field! :universe_domain, nil, match: String
|
|
159
|
+
config.add_field! :logger, default_logger, match: Logger, allow_nil: true
|
|
156
160
|
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: google-cloud-pubsub
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 3.
|
|
4
|
+
version: 3.2.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Mike Moore
|
|
@@ -95,6 +95,7 @@ files:
|
|
|
95
95
|
- lib/google/cloud/pubsub/credentials.rb
|
|
96
96
|
- lib/google/cloud/pubsub/errors.rb
|
|
97
97
|
- lib/google/cloud/pubsub/flow_controller.rb
|
|
98
|
+
- lib/google/cloud/pubsub/internal_logger.rb
|
|
98
99
|
- lib/google/cloud/pubsub/message.rb
|
|
99
100
|
- lib/google/cloud/pubsub/message_listener.rb
|
|
100
101
|
- lib/google/cloud/pubsub/message_listener/enumerator_queue.rb
|