google-cloud-pubsub 1.0.2 → 2.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AUTHENTICATION.md +16 -54
- data/CHANGELOG.md +464 -0
- data/CONTRIBUTING.md +328 -116
- data/EMULATOR.md +1 -1
- data/LOGGING.md +94 -2
- data/OVERVIEW.md +121 -68
- data/TROUBLESHOOTING.md +2 -8
- data/lib/google/cloud/pubsub/acknowledge_result.rb +79 -0
- data/lib/google/cloud/pubsub/async_publisher/batch.rb +319 -0
- data/lib/google/cloud/pubsub/async_publisher.rb +231 -156
- data/lib/google/cloud/pubsub/batch_publisher.rb +60 -30
- data/lib/google/cloud/pubsub/convert.rb +33 -7
- data/lib/google/cloud/pubsub/credentials.rb +2 -2
- data/lib/google/cloud/pubsub/errors.rb +93 -0
- data/lib/google/cloud/pubsub/flow_controller.rb +137 -0
- data/lib/google/cloud/pubsub/message.rb +45 -4
- data/lib/google/cloud/pubsub/policy.rb +3 -2
- data/lib/google/cloud/pubsub/project.rb +316 -49
- data/lib/google/cloud/pubsub/publish_result.rb +6 -1
- data/lib/google/cloud/pubsub/received_message.rb +171 -10
- data/lib/google/cloud/pubsub/retry_policy.rb +88 -0
- data/lib/google/cloud/pubsub/schema/list.rb +180 -0
- data/lib/google/cloud/pubsub/schema.rb +310 -0
- data/lib/google/cloud/pubsub/service.rb +285 -269
- data/lib/google/cloud/pubsub/snapshot/list.rb +4 -6
- data/lib/google/cloud/pubsub/snapshot.rb +5 -2
- data/lib/google/cloud/pubsub/subscriber/inventory.rb +69 -32
- data/lib/google/cloud/pubsub/subscriber/sequencer.rb +115 -0
- data/lib/google/cloud/pubsub/subscriber/stream.rb +108 -49
- data/lib/google/cloud/pubsub/subscriber/timed_unary_buffer.rb +191 -30
- data/lib/google/cloud/pubsub/subscriber.rb +155 -45
- data/lib/google/cloud/pubsub/subscription/list.rb +4 -6
- data/lib/google/cloud/pubsub/subscription/push_config.rb +55 -31
- data/lib/google/cloud/pubsub/subscription.rb +561 -77
- data/lib/google/cloud/pubsub/topic/list.rb +4 -6
- data/lib/google/cloud/pubsub/topic.rb +372 -52
- data/lib/google/cloud/pubsub/version.rb +1 -1
- data/lib/google/cloud/pubsub.rb +35 -46
- data/lib/google-cloud-pubsub.rb +21 -27
- metadata +26 -189
- data/lib/google/cloud/pubsub/v1/credentials.rb +0 -41
- data/lib/google/cloud/pubsub/v1/doc/google/iam/v1/iam_policy.rb +0 -21
- data/lib/google/cloud/pubsub/v1/doc/google/iam/v1/options.rb +0 -21
- data/lib/google/cloud/pubsub/v1/doc/google/iam/v1/policy.rb +0 -21
- data/lib/google/cloud/pubsub/v1/doc/google/protobuf/duration.rb +0 -91
- data/lib/google/cloud/pubsub/v1/doc/google/protobuf/empty.rb +0 -29
- data/lib/google/cloud/pubsub/v1/doc/google/protobuf/field_mask.rb +0 -222
- data/lib/google/cloud/pubsub/v1/doc/google/protobuf/timestamp.rb +0 -113
- data/lib/google/cloud/pubsub/v1/doc/google/pubsub/v1/pubsub.rb +0 -744
- data/lib/google/cloud/pubsub/v1/doc/google/type/expr.rb +0 -19
- data/lib/google/cloud/pubsub/v1/publisher_client.rb +0 -786
- data/lib/google/cloud/pubsub/v1/publisher_client_config.json +0 -105
- data/lib/google/cloud/pubsub/v1/subscriber_client.rb +0 -1385
- data/lib/google/cloud/pubsub/v1/subscriber_client_config.json +0 -138
- data/lib/google/cloud/pubsub/v1.rb +0 -17
- data/lib/google/pubsub/v1/pubsub_pb.rb +0 -249
- data/lib/google/pubsub/v1/pubsub_services_pb.rb +0 -211
@@ -125,17 +125,15 @@ module Google
|
|
125
125
|
# puts snapshot.name
|
126
126
|
# end
|
127
127
|
#
|
128
|
-
def all request_limit: nil
|
128
|
+
def all request_limit: nil, &block
|
129
129
|
request_limit = request_limit.to_i if request_limit
|
130
|
-
unless block_given?
|
131
|
-
return enum_for :all, request_limit: request_limit
|
132
|
-
end
|
130
|
+
return enum_for :all, request_limit: request_limit unless block_given?
|
133
131
|
results = self
|
134
132
|
loop do
|
135
|
-
results.each
|
133
|
+
results.each(&block)
|
136
134
|
if request_limit
|
137
135
|
request_limit -= 1
|
138
|
-
break if request_limit
|
136
|
+
break if request_limit.negative?
|
139
137
|
end
|
140
138
|
break unless results.next?
|
141
139
|
results = results.next
|
@@ -15,6 +15,7 @@
|
|
15
15
|
|
16
16
|
require "google/cloud/errors"
|
17
17
|
require "google/cloud/pubsub/snapshot/list"
|
18
|
+
require "google/cloud/pubsub/v1"
|
18
19
|
|
19
20
|
module Google
|
20
21
|
module Cloud
|
@@ -58,8 +59,10 @@ module Google
|
|
58
59
|
end
|
59
60
|
|
60
61
|
##
|
61
|
-
# The name of the snapshot.
|
62
|
-
#
|
62
|
+
# The name of the snapshot.
|
63
|
+
#
|
64
|
+
# @return [String] A fully-qualified snapshot name in the form
|
65
|
+
# `projects/{project_id}/snapshots/{snapshot_id}`.
|
63
66
|
def name
|
64
67
|
@grpc.name
|
65
68
|
end
|
@@ -22,30 +22,49 @@ module Google
|
|
22
22
|
##
|
23
23
|
# @private
|
24
24
|
class Inventory
|
25
|
+
InventoryItem = Struct.new :bytesize, :pulled_at do
|
26
|
+
def self.from rec_msg
|
27
|
+
new rec_msg.to_proto.bytesize, Time.now
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
25
31
|
include MonitorMixin
|
26
32
|
|
27
|
-
attr_reader :stream
|
33
|
+
attr_reader :stream
|
34
|
+
attr_reader :limit
|
35
|
+
attr_reader :bytesize
|
36
|
+
attr_reader :extension
|
37
|
+
attr_reader :max_duration_per_lease_extension
|
38
|
+
attr_accessor :min_duration_per_lease_extension
|
39
|
+
attr_reader :use_legacy_flow_control
|
28
40
|
|
29
|
-
def initialize stream, limit
|
41
|
+
def initialize stream, limit:, bytesize:, extension:, max_duration_per_lease_extension:,
|
42
|
+
min_duration_per_lease_extension:, use_legacy_flow_control:
|
43
|
+
super()
|
30
44
|
@stream = stream
|
31
45
|
@limit = limit
|
32
|
-
@
|
46
|
+
@bytesize = bytesize
|
47
|
+
@extension = extension
|
48
|
+
@max_duration_per_lease_extension = max_duration_per_lease_extension
|
49
|
+
@min_duration_per_lease_extension = min_duration_per_lease_extension
|
50
|
+
@use_legacy_flow_control = use_legacy_flow_control
|
51
|
+
@inventory = {}
|
33
52
|
@wait_cond = new_cond
|
34
|
-
|
35
|
-
super()
|
36
53
|
end
|
37
54
|
|
38
55
|
def ack_ids
|
39
|
-
@
|
56
|
+
@inventory.keys
|
40
57
|
end
|
41
58
|
|
42
|
-
def add *
|
43
|
-
|
44
|
-
|
45
|
-
return if
|
59
|
+
def add *rec_msgs
|
60
|
+
rec_msgs.flatten!
|
61
|
+
rec_msgs.compact!
|
62
|
+
return if rec_msgs.empty?
|
46
63
|
|
47
64
|
synchronize do
|
48
|
-
|
65
|
+
rec_msgs.each do |rec_msg|
|
66
|
+
@inventory[rec_msg.ack_id] = InventoryItem.from rec_msg
|
67
|
+
end
|
49
68
|
@wait_cond.broadcast
|
50
69
|
end
|
51
70
|
end
|
@@ -56,20 +75,34 @@ module Google
|
|
56
75
|
return if ack_ids.empty?
|
57
76
|
|
58
77
|
synchronize do
|
59
|
-
@
|
78
|
+
@inventory.delete_if { |ack_id, _| ack_ids.include? ack_id }
|
79
|
+
@wait_cond.broadcast
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
def remove_expired!
|
84
|
+
synchronize do
|
85
|
+
extension_time = Time.new - extension
|
86
|
+
@inventory.delete_if { |_ack_id, item| item.pulled_at < extension_time }
|
60
87
|
@wait_cond.broadcast
|
61
88
|
end
|
62
89
|
end
|
63
90
|
|
64
91
|
def count
|
65
92
|
synchronize do
|
66
|
-
@
|
93
|
+
@inventory.count
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
def total_bytesize
|
98
|
+
synchronize do
|
99
|
+
@inventory.values.sum(&:bytesize)
|
67
100
|
end
|
68
101
|
end
|
69
102
|
|
70
103
|
def empty?
|
71
104
|
synchronize do
|
72
|
-
@
|
105
|
+
@inventory.empty?
|
73
106
|
end
|
74
107
|
end
|
75
108
|
|
@@ -93,7 +126,9 @@ module Google
|
|
93
126
|
end
|
94
127
|
|
95
128
|
def full?
|
96
|
-
|
129
|
+
synchronize do
|
130
|
+
@inventory.count >= limit || @inventory.values.sum(&:bytesize) >= bytesize
|
131
|
+
end
|
97
132
|
end
|
98
133
|
|
99
134
|
protected
|
@@ -101,26 +136,24 @@ module Google
|
|
101
136
|
def background_run
|
102
137
|
delay_target = nil
|
103
138
|
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
delay_target = nil
|
108
|
-
|
109
|
-
@wait_cond.wait # wait until broadcast
|
110
|
-
next
|
111
|
-
end
|
139
|
+
until stopped?
|
140
|
+
if empty?
|
141
|
+
delay_target = nil
|
112
142
|
|
113
|
-
|
114
|
-
|
143
|
+
synchronize { @wait_cond.wait } # wait until broadcast
|
144
|
+
next
|
145
|
+
end
|
115
146
|
|
116
|
-
|
117
|
-
|
118
|
-
stream.renew_lease!
|
119
|
-
next
|
120
|
-
end
|
147
|
+
delay_target ||= calc_target
|
148
|
+
delay_gap = delay_target - Time.now
|
121
149
|
|
122
|
-
|
150
|
+
unless delay_gap.positive?
|
151
|
+
delay_target = calc_target
|
152
|
+
stream.renew_lease!
|
153
|
+
next
|
123
154
|
end
|
155
|
+
|
156
|
+
synchronize { @wait_cond.wait delay_gap }
|
124
157
|
end
|
125
158
|
end
|
126
159
|
|
@@ -129,7 +162,11 @@ module Google
|
|
129
162
|
end
|
130
163
|
|
131
164
|
def calc_delay
|
132
|
-
(stream.subscriber.deadline - 3) * rand(0.8..0.9)
|
165
|
+
delay = (stream.subscriber.deadline - 3) * rand(0.8..0.9)
|
166
|
+
delay = [delay, max_duration_per_lease_extension].min if max_duration_per_lease_extension.positive?
|
167
|
+
delay = [delay, min_duration_per_lease_extension].max if min_duration_per_lease_extension.positive? &&
|
168
|
+
stream.exactly_once_delivery_enabled
|
169
|
+
delay
|
133
170
|
end
|
134
171
|
end
|
135
172
|
end
|
@@ -0,0 +1,115 @@
|
|
1
|
+
# Copyright 2019 Google LLC
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
|
16
|
+
require "monitor"
|
17
|
+
|
18
|
+
module Google
|
19
|
+
module Cloud
|
20
|
+
module PubSub
|
21
|
+
class Subscriber
|
22
|
+
##
|
23
|
+
# @private The sequencer's job is simple, keep track of all the
|
24
|
+
# streams's recieved message and deliver the messages with an
|
25
|
+
# ordering_key in the order they were recieved. The sequencer ensures
|
26
|
+
# only one callback can be performed at a time per ordering_key.
|
27
|
+
class Sequencer
|
28
|
+
include MonitorMixin
|
29
|
+
|
30
|
+
##
|
31
|
+
# @private Create an empty Subscriber::Sequencer object.
|
32
|
+
def initialize &block
|
33
|
+
raise ArgumentError if block.nil?
|
34
|
+
|
35
|
+
super() # to init MonitorMixin
|
36
|
+
|
37
|
+
@seq_hash = Hash.new { |hash, key| hash[key] = [] }
|
38
|
+
@process_callback = block
|
39
|
+
end
|
40
|
+
|
41
|
+
##
|
42
|
+
# @private Add a ReceivedMessage to the sequencer.
|
43
|
+
def add message
|
44
|
+
# Messages without ordering_key are not managed by the sequencer
|
45
|
+
if message.ordering_key.empty?
|
46
|
+
@process_callback.call message
|
47
|
+
return
|
48
|
+
end
|
49
|
+
|
50
|
+
perform_callback = synchronize do
|
51
|
+
# The purpose of this block is to add the message to the
|
52
|
+
# sequencer, and to return whether the message should be processed
|
53
|
+
# immediately, or whether it will be processed later by #next. We
|
54
|
+
# want to ensure that these operations happen atomically.
|
55
|
+
|
56
|
+
@seq_hash[message.ordering_key].push message
|
57
|
+
@seq_hash[message.ordering_key].count == 1
|
58
|
+
end
|
59
|
+
|
60
|
+
@process_callback.call message if perform_callback
|
61
|
+
end
|
62
|
+
|
63
|
+
##
|
64
|
+
# @private Indicate a ReceivedMessage was processed, and the next in
|
65
|
+
# the queue can now be processed.
|
66
|
+
def next message
|
67
|
+
# Messages without ordering_key are not managed by the sequencer
|
68
|
+
return if message.ordering_key.empty?
|
69
|
+
|
70
|
+
next_message = synchronize do
|
71
|
+
# The purpose of this block is to remove the message that was
|
72
|
+
# processed from the sequencer, and to return the next message to
|
73
|
+
# be processed. We want to ensure that these operations happen
|
74
|
+
# atomically.
|
75
|
+
|
76
|
+
# The message should be at index 0, so this should be a very quick
|
77
|
+
# operation.
|
78
|
+
if @seq_hash[message.ordering_key].first != message
|
79
|
+
# Raising this error will stop the other messages with this
|
80
|
+
# ordering key from being processed by the callback (delivered).
|
81
|
+
raise OrderedMessageDeliveryError, message
|
82
|
+
end
|
83
|
+
|
84
|
+
# Remove the message
|
85
|
+
@seq_hash[message.ordering_key].shift
|
86
|
+
|
87
|
+
# Retrieve the next message to be processed, or nil if empty
|
88
|
+
next_msg = @seq_hash[message.ordering_key].first
|
89
|
+
|
90
|
+
# Remove the ordering_key from hash when empty
|
91
|
+
@seq_hash.delete message.ordering_key if next_msg.nil?
|
92
|
+
|
93
|
+
# Return the next message to be processed, or nil if empty
|
94
|
+
next_msg
|
95
|
+
end
|
96
|
+
|
97
|
+
@process_callback.call next_message unless next_message.nil?
|
98
|
+
end
|
99
|
+
|
100
|
+
# @private
|
101
|
+
def to_s
|
102
|
+
"#{@seq_hash.count}/#{@seq_hash.values.sum(&:count)}"
|
103
|
+
end
|
104
|
+
|
105
|
+
# @private
|
106
|
+
def inspect
|
107
|
+
"#<#{self.class.name} (#{self})>"
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
Pubsub = PubSub unless const_defined? :Pubsub
|
114
|
+
end
|
115
|
+
end
|
@@ -13,6 +13,7 @@
|
|
13
13
|
# limitations under the License.
|
14
14
|
|
15
15
|
|
16
|
+
require "google/cloud/pubsub/subscriber/sequencer"
|
16
17
|
require "google/cloud/pubsub/subscriber/enumerator_queue"
|
17
18
|
require "google/cloud/pubsub/subscriber/inventory"
|
18
19
|
require "google/cloud/pubsub/service"
|
@@ -34,33 +35,46 @@ module Google
|
|
34
35
|
attr_reader :callback_thread_pool
|
35
36
|
|
36
37
|
##
|
37
|
-
# Subscriber attributes.
|
38
|
+
# @private Subscriber attributes.
|
38
39
|
attr_reader :subscriber
|
39
40
|
|
41
|
+
##
|
42
|
+
# @private Inventory.
|
43
|
+
attr_reader :inventory
|
44
|
+
|
45
|
+
##
|
46
|
+
# @private Sequencer.
|
47
|
+
attr_reader :sequencer
|
48
|
+
|
49
|
+
##
|
50
|
+
# @private exactly_once_delivery_enabled.
|
51
|
+
attr_reader :exactly_once_delivery_enabled
|
52
|
+
|
40
53
|
##
|
41
54
|
# @private Create an empty Subscriber::Stream object.
|
42
55
|
def initialize subscriber
|
56
|
+
super() # to init MonitorMixin
|
57
|
+
|
43
58
|
@subscriber = subscriber
|
44
59
|
|
45
60
|
@request_queue = nil
|
46
61
|
@stopped = nil
|
47
62
|
@paused = nil
|
48
63
|
@pause_cond = new_cond
|
64
|
+
@exactly_once_delivery_enabled = false
|
65
|
+
|
66
|
+
@inventory = Inventory.new self, **@subscriber.stream_inventory
|
67
|
+
|
68
|
+
@sequencer = Sequencer.new(&method(:perform_callback_async)) if subscriber.message_ordering
|
49
69
|
|
50
|
-
@
|
51
|
-
@callback_thread_pool = Concurrent::ThreadPoolExecutor.new \
|
52
|
-
max_threads: @subscriber.callback_threads
|
70
|
+
@callback_thread_pool = Concurrent::ThreadPoolExecutor.new max_threads: @subscriber.callback_threads
|
53
71
|
|
54
72
|
@stream_keepalive_task = Concurrent::TimerTask.new(
|
55
73
|
execution_interval: 30
|
56
74
|
) do
|
57
75
|
# push empty request every 30 seconds to keep stream alive
|
58
|
-
unless inventory.empty?
|
59
|
-
push Google::Cloud::PubSub::V1::StreamingPullRequest.new
|
60
|
-
end
|
76
|
+
push Google::Cloud::PubSub::V1::StreamingPullRequest.new unless inventory.empty?
|
61
77
|
end.execute
|
62
|
-
|
63
|
-
super() # to init MonitorMixin
|
64
78
|
end
|
65
79
|
|
66
80
|
def start
|
@@ -81,7 +95,7 @@ module Google
|
|
81
95
|
|
82
96
|
# Close the stream by pushing the sentinel value.
|
83
97
|
# The unary pusher does not use the stream, so it can close here.
|
84
|
-
@request_queue
|
98
|
+
@request_queue&.push self
|
85
99
|
|
86
100
|
# Signal to the background thread that we are stopped.
|
87
101
|
@stopped = true
|
@@ -107,6 +121,10 @@ module Google
|
|
107
121
|
synchronize { @paused }
|
108
122
|
end
|
109
123
|
|
124
|
+
def running?
|
125
|
+
!stopped?
|
126
|
+
end
|
127
|
+
|
110
128
|
def wait! timeout = nil
|
111
129
|
# Wait for all queued callbacks to be processed.
|
112
130
|
@callback_thread_pool.wait_for_termination timeout
|
@@ -116,13 +134,13 @@ module Google
|
|
116
134
|
|
117
135
|
##
|
118
136
|
# @private
|
119
|
-
def acknowledge *messages
|
137
|
+
def acknowledge *messages, &callback
|
120
138
|
ack_ids = coerce_ack_ids messages
|
121
139
|
return true if ack_ids.empty?
|
122
140
|
|
123
141
|
synchronize do
|
124
142
|
@inventory.remove ack_ids
|
125
|
-
@subscriber.buffer.acknowledge ack_ids
|
143
|
+
@subscriber.buffer.acknowledge ack_ids, callback
|
126
144
|
end
|
127
145
|
|
128
146
|
true
|
@@ -130,13 +148,13 @@ module Google
|
|
130
148
|
|
131
149
|
##
|
132
150
|
# @private
|
133
|
-
def modify_ack_deadline deadline, *messages
|
151
|
+
def modify_ack_deadline deadline, *messages, &callback
|
134
152
|
mod_ack_ids = coerce_ack_ids messages
|
135
153
|
return true if mod_ack_ids.empty?
|
136
154
|
|
137
155
|
synchronize do
|
138
156
|
@inventory.remove mod_ack_ids
|
139
|
-
@subscriber.buffer.modify_ack_deadline deadline, mod_ack_ids
|
157
|
+
@subscriber.buffer.modify_ack_deadline deadline, mod_ack_ids, callback
|
140
158
|
end
|
141
159
|
|
142
160
|
true
|
@@ -162,18 +180,14 @@ module Google
|
|
162
180
|
synchronize { @request_queue.push request }
|
163
181
|
end
|
164
182
|
|
165
|
-
def inventory
|
166
|
-
synchronize { @inventory }
|
167
|
-
end
|
168
|
-
|
169
183
|
##
|
170
184
|
# @private
|
171
185
|
def renew_lease!
|
172
186
|
synchronize do
|
173
187
|
return true if @inventory.empty?
|
174
188
|
|
175
|
-
@
|
176
|
-
|
189
|
+
@inventory.remove_expired!
|
190
|
+
@subscriber.buffer.renew_lease @subscriber.deadline, @inventory.ack_ids
|
177
191
|
unpause_streaming!
|
178
192
|
end
|
179
193
|
|
@@ -182,8 +196,8 @@ module Google
|
|
182
196
|
|
183
197
|
# @private
|
184
198
|
def to_s
|
185
|
-
"
|
186
|
-
|
199
|
+
seq_str = "sequenced: #{sequencer}, " if sequencer
|
200
|
+
"(inventory: #{@inventory.count}, #{seq_str}status: #{status}, thread: #{thread_status})"
|
187
201
|
end
|
188
202
|
|
189
203
|
# @private
|
@@ -217,7 +231,8 @@ module Google
|
|
217
231
|
end
|
218
232
|
|
219
233
|
# Call the StreamingPull API to get the response enumerator
|
220
|
-
|
234
|
+
options = { :"metadata" => { :"x-goog-request-params" => @subscriber.subscription_name } }
|
235
|
+
enum = @subscriber.service.streaming_pull @request_queue.each, options
|
221
236
|
|
222
237
|
loop do
|
223
238
|
synchronize do
|
@@ -231,28 +246,33 @@ module Google
|
|
231
246
|
break if synchronize { @stopped }
|
232
247
|
|
233
248
|
begin
|
234
|
-
# Cannot
|
249
|
+
# Cannot synchronize the enumerator, causes deadlock
|
235
250
|
response = enum.next
|
251
|
+
new_exactly_once_delivery_enabled = response&.subscription_properties&.exactly_once_delivery_enabled
|
252
|
+
received_messages = response.received_messages
|
236
253
|
|
237
|
-
#
|
238
|
-
received_ack_ids = response.received_messages.map(&:ack_id)
|
239
|
-
|
254
|
+
# Use synchronize so changes happen atomically
|
240
255
|
synchronize do
|
241
|
-
|
242
|
-
@
|
243
|
-
|
256
|
+
update_min_duration_per_lease_extension new_exactly_once_delivery_enabled
|
257
|
+
@exactly_once_delivery_enabled = new_exactly_once_delivery_enabled unless new_exactly_once_delivery_enabled.nil?
|
258
|
+
@subscriber.exactly_once_delivery_enabled = @exactly_once_delivery_enabled
|
244
259
|
|
245
|
-
#
|
246
|
-
@
|
260
|
+
# Create receipt of received messages reception
|
261
|
+
if @exactly_once_delivery_enabled
|
262
|
+
create_receipt_modack_for_eos received_messages
|
263
|
+
else
|
264
|
+
@subscriber.buffer.modify_ack_deadline @subscriber.deadline, received_messages.map(&:ack_id)
|
265
|
+
# Add received messages to inventory
|
266
|
+
@inventory.add received_messages
|
267
|
+
end
|
247
268
|
end
|
248
269
|
|
249
|
-
|
270
|
+
received_messages.each do |rec_msg_grpc|
|
250
271
|
rec_msg = ReceivedMessage.from_grpc(rec_msg_grpc, self)
|
251
|
-
synchronize
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
end
|
272
|
+
# No need to synchronize the callback future
|
273
|
+
register_callback rec_msg
|
274
|
+
end if !@exactly_once_delivery_enabled # Exactly once delivery scenario is handled by callback
|
275
|
+
|
256
276
|
synchronize { pause_streaming! }
|
257
277
|
rescue StopIteration
|
258
278
|
break
|
@@ -267,9 +287,8 @@ module Google
|
|
267
287
|
stop
|
268
288
|
rescue GRPC::Cancelled, GRPC::DeadlineExceeded, GRPC::Internal,
|
269
289
|
GRPC::ResourceExhausted, GRPC::Unauthenticated,
|
270
|
-
GRPC::Unavailable
|
290
|
+
GRPC::Unavailable
|
271
291
|
# Restart the stream with an incremental back for a retriable error.
|
272
|
-
# Also when GRPC raises the internal CallError.
|
273
292
|
|
274
293
|
retry
|
275
294
|
rescue RestartStream
|
@@ -282,18 +301,56 @@ module Google
|
|
282
301
|
|
283
302
|
# rubocop:enable all
|
284
303
|
|
304
|
+
def create_receipt_modack_for_eos received_messages
|
305
|
+
received_messages.each do |rec_msg_grpc|
|
306
|
+
callback = proc do |result|
|
307
|
+
if result.succeeded?
|
308
|
+
synchronize { @inventory.add rec_msg_grpc }
|
309
|
+
rec_msg = ReceivedMessage.from_grpc rec_msg_grpc, self
|
310
|
+
register_callback rec_msg
|
311
|
+
end
|
312
|
+
end
|
313
|
+
@subscriber.buffer.modify_ack_deadline @subscriber.deadline, [rec_msg_grpc.ack_id], callback
|
314
|
+
end
|
315
|
+
end
|
316
|
+
|
317
|
+
# Updates min_duration_per_lease_extension to 60 when exactly_once_delivery_enabled
|
318
|
+
# and reverts back to default 0 when disabled.
|
319
|
+
# Skips if exactly_once_enabled is not modified.
|
320
|
+
def update_min_duration_per_lease_extension new_exactly_once_delivery_enabled
|
321
|
+
return if new_exactly_once_delivery_enabled == @exactly_once_delivery_enabled
|
322
|
+
@inventory.min_duration_per_lease_extension = new_exactly_once_delivery_enabled ? 60 : 0
|
323
|
+
end
|
324
|
+
|
325
|
+
def register_callback rec_msg
|
326
|
+
if @sequencer
|
327
|
+
# Add the message to the sequencer to invoke the callback.
|
328
|
+
@sequencer.add rec_msg
|
329
|
+
else
|
330
|
+
# Call user provided code for received message
|
331
|
+
perform_callback_async rec_msg
|
332
|
+
end
|
333
|
+
end
|
334
|
+
|
285
335
|
def perform_callback_async rec_msg
|
286
336
|
return unless callback_thread_pool.running?
|
287
337
|
|
288
338
|
Concurrent::Promises.future_on(
|
289
|
-
callback_thread_pool,
|
290
|
-
)
|
339
|
+
callback_thread_pool, rec_msg, &method(:perform_callback_sync)
|
340
|
+
)
|
341
|
+
end
|
342
|
+
|
343
|
+
def perform_callback_sync rec_msg
|
344
|
+
@subscriber.callback.call rec_msg unless stopped?
|
345
|
+
rescue StandardError => e
|
346
|
+
@subscriber.error! e
|
347
|
+
ensure
|
348
|
+
release rec_msg
|
349
|
+
if @sequencer && running?
|
291
350
|
begin
|
292
|
-
|
293
|
-
rescue
|
294
|
-
|
295
|
-
ensure
|
296
|
-
stream.release msg
|
351
|
+
@sequencer.next rec_msg
|
352
|
+
rescue OrderedMessageDeliveryError => e
|
353
|
+
@subscriber.error! e
|
297
354
|
end
|
298
355
|
end
|
299
356
|
end
|
@@ -341,8 +398,10 @@ module Google
|
|
341
398
|
req.subscription = @subscriber.subscription_name
|
342
399
|
req.stream_ack_deadline_seconds = @subscriber.deadline
|
343
400
|
req.modify_deadline_ack_ids += @inventory.ack_ids
|
344
|
-
req.modify_deadline_seconds +=
|
345
|
-
|
401
|
+
req.modify_deadline_seconds += @inventory.ack_ids.map { @subscriber.deadline }
|
402
|
+
req.client_id = @subscriber.service.client_id
|
403
|
+
req.max_outstanding_messages = @inventory.use_legacy_flow_control ? 0 : @inventory.limit
|
404
|
+
req.max_outstanding_bytes = @inventory.use_legacy_flow_control ? 0 : @inventory.bytesize
|
346
405
|
end
|
347
406
|
end
|
348
407
|
|