command_tower 0.19.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/models/command_tower/messaging/delivery_attempt.rb +5 -0
- data/app/services/command_tower/messaging/delivery_retry_policy.rb +97 -0
- data/app/services/command_tower/messaging/execution/adapters/expo/adapter.rb +71 -23
- data/app/services/command_tower/messaging/execution/adapters/expo/http_client.rb +4 -0
- data/app/services/command_tower/messaging/execution/attempt_ledger.rb +42 -2
- data/app/services/command_tower/messaging/execution/operation_logger.rb +10 -0
- data/app/services/command_tower/messaging/execution/recovery.rb +64 -8
- data/app/workflows/command_tower/workflows/messaging/execution/deliver_workflow.rb +19 -0
- data/docs/upgrades/0.20.0.md +29 -0
- data/docs/upgrades/README.md +1 -0
- data/lib/command_tower/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 8bf05b7d610367efda3b4dce6349bfbd002891563a5f16a5688bbd2660717896
|
|
4
|
+
data.tar.gz: 5a31b97e28d791fbda2ad4899c121fe54c2f96e944c387c716d5541d7449c8f6
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 6c1a8b5cf203cb35b4697f593324b94eb1a0e0a7c4d4b171c752b669a700862679a28cbbe494b54a632294181fa6ddf7d78aa117cf5a71e1deb7e9fc036b122e
|
|
7
|
+
data.tar.gz: 5232cb631564969ef269b82fd1b9c38ef820ee76523640a09b2846b380348b64e5c4dc62288b0a391edde0586b4e3d28559d26614788d26e7ba26ab8a96c7bd0
|
|
@@ -21,6 +21,11 @@ module CommandTower
|
|
|
21
21
|
class_name: "CommandTower::Messaging::ChannelDelivery",
|
|
22
22
|
inverse_of: :delivery_attempts
|
|
23
23
|
|
|
24
|
+
# MariaDB reports JSON columns as LONGTEXT. The text caster would write
|
|
25
|
+
# a Ruby hash string and fail `json_valid`. Forcing the JSON caster
|
|
26
|
+
# writes one JSON document on MySQL and on MariaDB.
|
|
27
|
+
attribute :provider_metadata, :json
|
|
28
|
+
|
|
24
29
|
validates :status, presence: true, inclusion: { in: STATUSES }
|
|
25
30
|
validates :started_at, presence: true
|
|
26
31
|
end
|
|
@@ -8,6 +8,10 @@ module CommandTower
|
|
|
8
8
|
MAX_ATTEMPTS = 5
|
|
9
9
|
GRACE_WINDOW = 1.minute
|
|
10
10
|
|
|
11
|
+
PUSH_RETRY_LIMIT = 2
|
|
12
|
+
SUBMISSION_NOT_SUBMITTED = "not_submitted"
|
|
13
|
+
SUBMISSION_UNCERTAIN = "submission_uncertain"
|
|
14
|
+
|
|
11
15
|
class << self
|
|
12
16
|
def retries_exhausted?(delivery)
|
|
13
17
|
delivery.status == ChannelDelivery::STATUS_FAILED_RETRYABLE &&
|
|
@@ -24,6 +28,99 @@ module CommandTower
|
|
|
24
28
|
|
|
25
29
|
delivery.updated_at + GRACE_WINDOW
|
|
26
30
|
end
|
|
31
|
+
|
|
32
|
+
def push_channel?(delivery)
|
|
33
|
+
delivery.channel_key == "push"
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def max_execution_attempts_for(delivery)
|
|
37
|
+
push_channel?(delivery) ? PUSH_RETRY_LIMIT : MAX_ATTEMPTS
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def confirmed_not_submitted?(attempt)
|
|
41
|
+
attempt.provider_metadata.is_a?(Hash) &&
|
|
42
|
+
attempt.provider_metadata["submission"] == SUBMISSION_NOT_SUBMITTED
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def accepted_ticket_in_attempt?(attempt)
|
|
46
|
+
return true if attempt.status == Messaging::DeliveryAttempt::STATUS_SUCCEEDED
|
|
47
|
+
|
|
48
|
+
metadata = attempt.provider_metadata
|
|
49
|
+
return false unless metadata.is_a?(Hash)
|
|
50
|
+
|
|
51
|
+
endpoints = metadata.dig("expo", "endpoints")
|
|
52
|
+
return false unless endpoints.is_a?(Array)
|
|
53
|
+
|
|
54
|
+
endpoints.any? { |entry| entry["ticket_status"] == "ok" }
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def push_has_open_started_attempt?(delivery)
|
|
58
|
+
delivery.delivery_attempts
|
|
59
|
+
.where(status: Messaging::DeliveryAttempt::STATUS_STARTED, finished_at: nil)
|
|
60
|
+
.exists?
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def latest_finished_attempt(delivery)
|
|
64
|
+
delivery.delivery_attempts
|
|
65
|
+
.where.not(finished_at: nil)
|
|
66
|
+
.order(id: :desc)
|
|
67
|
+
.first
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Persisted proof only — not tied to channel delivery status.
|
|
71
|
+
def push_has_not_submitted_evidence?(delivery)
|
|
72
|
+
attempt = latest_finished_attempt(delivery)
|
|
73
|
+
attempt.present? && confirmed_not_submitted?(attempt)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def push_retry_budget_remaining?(delivery)
|
|
77
|
+
delivery.execution_attempt_count < PUSH_RETRY_LIMIT
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# recover_failed! — delivery is failed_retryable.
|
|
81
|
+
def push_redrive_eligible_from_failed_retryable?(delivery)
|
|
82
|
+
return false unless push_channel?(delivery)
|
|
83
|
+
return false unless delivery.status == ChannelDelivery::STATUS_FAILED_RETRYABLE
|
|
84
|
+
return false if push_has_open_started_attempt?(delivery)
|
|
85
|
+
return false unless push_has_not_submitted_evidence?(delivery)
|
|
86
|
+
|
|
87
|
+
push_retry_budget_remaining?(delivery)
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# recover_queued! / recover_stale_executing! — delivery may already be queued.
|
|
91
|
+
def push_redrive_eligible_while_queued?(delivery)
|
|
92
|
+
return true unless push_channel?(delivery)
|
|
93
|
+
return false if push_has_open_started_attempt?(delivery)
|
|
94
|
+
return true if delivery.delivery_attempts.none?
|
|
95
|
+
|
|
96
|
+
push_has_not_submitted_evidence?(delivery) && push_retry_budget_remaining?(delivery)
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# DeliverWorkflow — may this execution call Expo?
|
|
100
|
+
def push_may_submit_to_provider?(delivery, current_attempt_id:)
|
|
101
|
+
return true unless push_channel?(delivery)
|
|
102
|
+
|
|
103
|
+
prior_attempts = delivery.delivery_attempts.where.not(id: current_attempt_id)
|
|
104
|
+
return false if prior_attempts
|
|
105
|
+
.where(status: Messaging::DeliveryAttempt::STATUS_STARTED, finished_at: nil)
|
|
106
|
+
.exists?
|
|
107
|
+
|
|
108
|
+
finished = prior_attempts.where.not(finished_at: nil)
|
|
109
|
+
return true if finished.none?
|
|
110
|
+
|
|
111
|
+
return false if finished.any? { |attempt| accepted_ticket_in_attempt?(attempt) }
|
|
112
|
+
|
|
113
|
+
return false if finished.any? do |attempt|
|
|
114
|
+
attempt.error_code == SUBMISSION_UNCERTAIN ||
|
|
115
|
+
(
|
|
116
|
+
attempt.status == Messaging::DeliveryAttempt::STATUS_FAILED_TERMINAL &&
|
|
117
|
+
!confirmed_not_submitted?(attempt)
|
|
118
|
+
)
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
latest = finished.order(:id).last
|
|
122
|
+
confirmed_not_submitted?(latest) && delivery.execution_attempt_count <= PUSH_RETRY_LIMIT
|
|
123
|
+
end
|
|
27
124
|
end
|
|
28
125
|
end
|
|
29
126
|
end
|
|
@@ -75,13 +75,10 @@ module CommandTower
|
|
|
75
75
|
else
|
|
76
76
|
AdapterResult.build(outcome: :terminal_failure, error_code: "adapter_unconfigured")
|
|
77
77
|
end
|
|
78
|
+
rescue HttpClient::RequestNotSubmitted
|
|
79
|
+
not_submitted_result(attempts, history:)
|
|
78
80
|
rescue StandardError
|
|
79
|
-
|
|
80
|
-
AdapterResult.build(
|
|
81
|
-
outcome: :retryable_failure,
|
|
82
|
-
error_code: "expo_transient",
|
|
83
|
-
provider_metadata: provider_metadata_for(attempts),
|
|
84
|
-
)
|
|
81
|
+
uncertain_terminal_result(attempts, history:)
|
|
85
82
|
end
|
|
86
83
|
|
|
87
84
|
private
|
|
@@ -110,8 +107,8 @@ module CommandTower
|
|
|
110
107
|
sendable = attempts.select { |attempt| attempt.token.present? && attempt.outcome.nil? }
|
|
111
108
|
messages = sendable.map { |attempt| expo_message(token: attempt.token, rendered:, communication:) }
|
|
112
109
|
|
|
113
|
-
sendable.each { |attempt| attempt.submitted = true }
|
|
114
110
|
send_result = @http_client.send_messages(messages)
|
|
111
|
+
sendable.each { |attempt| attempt.submitted = true }
|
|
115
112
|
unless send_result[:ok]
|
|
116
113
|
return map_http_failure(send_result[:status_code], attempts:, history:)
|
|
117
114
|
end
|
|
@@ -125,7 +122,7 @@ module CommandTower
|
|
|
125
122
|
|
|
126
123
|
ok_ticket_ids = sendable.filter_map { |attempt| attempt.ticket&.dig(:id) if attempt.ticket&.dig(:status) == "ok" }
|
|
127
124
|
if ok_ticket_ids.any?
|
|
128
|
-
receipts_result =
|
|
125
|
+
receipts_result = fetch_receipts(ok_ticket_ids)
|
|
129
126
|
if receipts_result[:ok]
|
|
130
127
|
receipts = receipts_result[:receipts] || {}
|
|
131
128
|
sendable.each do |attempt|
|
|
@@ -148,6 +145,12 @@ module CommandTower
|
|
|
148
145
|
aggregate_result(attempts, history:)
|
|
149
146
|
end
|
|
150
147
|
|
|
148
|
+
def fetch_receipts(ticket_ids)
|
|
149
|
+
@http_client.get_receipts(ticket_ids)
|
|
150
|
+
rescue StandardError
|
|
151
|
+
{ ok: false, status_code: nil, receipts: {} }
|
|
152
|
+
end
|
|
153
|
+
|
|
151
154
|
# Every eligible endpoint already holds an accepted ticket or a recorded terminal
|
|
152
155
|
# ticket error from an earlier attempt, so there is nothing to send.
|
|
153
156
|
def settled_only_result(history)
|
|
@@ -194,8 +197,8 @@ module CommandTower
|
|
|
194
197
|
def apply_ticket_outcome!(attempt)
|
|
195
198
|
ticket = attempt.ticket
|
|
196
199
|
if ticket.nil?
|
|
197
|
-
attempt.outcome = :
|
|
198
|
-
attempt.error_code =
|
|
200
|
+
attempt.outcome = :terminal_failure
|
|
201
|
+
attempt.error_code = submission_uncertain_code
|
|
199
202
|
return
|
|
200
203
|
end
|
|
201
204
|
|
|
@@ -211,8 +214,8 @@ module CommandTower
|
|
|
211
214
|
attempt.error_code = "expo_rejected"
|
|
212
215
|
end
|
|
213
216
|
else
|
|
214
|
-
attempt.outcome = :
|
|
215
|
-
attempt.error_code =
|
|
217
|
+
attempt.outcome = :terminal_failure
|
|
218
|
+
attempt.error_code = submission_uncertain_code
|
|
216
219
|
end
|
|
217
220
|
end
|
|
218
221
|
|
|
@@ -246,29 +249,49 @@ module CommandTower
|
|
|
246
249
|
nil
|
|
247
250
|
end
|
|
248
251
|
|
|
249
|
-
#
|
|
250
|
-
#
|
|
251
|
-
# into the metadata so the accepted facts stay visible on this row.
|
|
252
|
+
# After the HTTP client returns, a non-success response is uncertain for duplicate policy
|
|
253
|
+
# (request may have been accepted). Terminal rejection (4xx except 429) maps to expo_rejected.
|
|
252
254
|
def map_http_failure(status_code, attempts:, history:)
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
255
|
+
code = status_code.to_i
|
|
256
|
+
error_code =
|
|
257
|
+
if code >= 500 || code == 429
|
|
258
|
+
submission_uncertain_code
|
|
259
|
+
else
|
|
260
|
+
"expo_rejected"
|
|
261
|
+
end
|
|
256
262
|
attempts.each { |attempt| attempt.error_code ||= error_code if attempt.submitted }
|
|
257
|
-
carried =
|
|
263
|
+
carried = error_code == submission_uncertain_code ? [] : carried_accepted_entries(history)
|
|
258
264
|
|
|
259
|
-
AdapterResult.build(
|
|
265
|
+
AdapterResult.build(
|
|
266
|
+
outcome: :terminal_failure,
|
|
267
|
+
error_code:,
|
|
268
|
+
provider_metadata: provider_metadata_for(attempts, carried:),
|
|
269
|
+
)
|
|
260
270
|
end
|
|
261
271
|
|
|
262
272
|
# Delivery-wide aggregation: this attempt's endpoints plus tickets recorded earlier.
|
|
263
273
|
def aggregate_result(attempts, history:)
|
|
274
|
+
submitted = attempts.select(&:submitted)
|
|
264
275
|
successes = attempts.select { |attempt| attempt.outcome == :success }
|
|
265
276
|
first_ticket_id = successes.filter_map { |attempt| attempt.ticket&.dig(:id) }.first
|
|
266
277
|
metadata = provider_metadata_for(attempts)
|
|
267
278
|
|
|
279
|
+
if submitted.any? { |attempt| attempt.outcome == :success } &&
|
|
280
|
+
submitted.any? do |attempt|
|
|
281
|
+
attempt.outcome == :terminal_failure && attempt.error_code == submission_uncertain_code
|
|
282
|
+
end
|
|
283
|
+
return AdapterResult.build(
|
|
284
|
+
outcome: :terminal_failure,
|
|
285
|
+
error_code: submission_uncertain_code,
|
|
286
|
+
provider_message_id: first_ticket_id,
|
|
287
|
+
provider_metadata: metadata,
|
|
288
|
+
)
|
|
289
|
+
end
|
|
290
|
+
|
|
268
291
|
if attempts.any? { |attempt| attempt.outcome == :retryable_failure }
|
|
269
292
|
return AdapterResult.build(
|
|
270
|
-
outcome: :
|
|
271
|
-
error_code:
|
|
293
|
+
outcome: :terminal_failure,
|
|
294
|
+
error_code: submission_uncertain_code,
|
|
272
295
|
provider_message_id: first_ticket_id,
|
|
273
296
|
provider_metadata: metadata,
|
|
274
297
|
)
|
|
@@ -290,13 +313,38 @@ module CommandTower
|
|
|
290
313
|
return AdapterResult.build(outcome: :terminal_failure, error_code:, provider_metadata: metadata)
|
|
291
314
|
end
|
|
292
315
|
|
|
316
|
+
AdapterResult.build(
|
|
317
|
+
outcome: :terminal_failure,
|
|
318
|
+
error_code: submission_uncertain_code,
|
|
319
|
+
provider_metadata: metadata,
|
|
320
|
+
)
|
|
321
|
+
end
|
|
322
|
+
|
|
323
|
+
def not_submitted_result(_attempts, history: nil)
|
|
293
324
|
AdapterResult.build(
|
|
294
325
|
outcome: :retryable_failure,
|
|
295
|
-
error_code: "
|
|
326
|
+
error_code: "expo_not_submitted",
|
|
327
|
+
provider_metadata: {
|
|
328
|
+
"submission" => Messaging::DeliveryRetryPolicy::SUBMISSION_NOT_SUBMITTED,
|
|
329
|
+
},
|
|
330
|
+
)
|
|
331
|
+
end
|
|
332
|
+
|
|
333
|
+
def uncertain_terminal_result(attempts, history:)
|
|
334
|
+
metadata = provider_metadata_for(attempts)
|
|
335
|
+
metadata = metadata.merge("submission" => submission_uncertain_code) if metadata.is_a?(Hash)
|
|
336
|
+
|
|
337
|
+
AdapterResult.build(
|
|
338
|
+
outcome: :terminal_failure,
|
|
339
|
+
error_code: submission_uncertain_code,
|
|
296
340
|
provider_metadata: metadata,
|
|
297
341
|
)
|
|
298
342
|
end
|
|
299
343
|
|
|
344
|
+
def submission_uncertain_code
|
|
345
|
+
Messaging::DeliveryRetryPolicy::SUBMISSION_UNCERTAIN
|
|
346
|
+
end
|
|
347
|
+
|
|
300
348
|
def carried_accepted_entries(history)
|
|
301
349
|
history.accepted_ticket_ids.map do |endpoint_id, ticket_id|
|
|
302
350
|
entry = { "endpoint_id" => endpoint_id, "ticket_status" => "ok" }
|
|
@@ -12,6 +12,8 @@ module CommandTower
|
|
|
12
12
|
# Narrow Expo Push HTTP client for Messaging notification push.
|
|
13
13
|
# Never logs tokens, Authorization headers, or raw bodies that may contain tokens.
|
|
14
14
|
class HttpClient
|
|
15
|
+
RequestNotSubmitted = Class.new(StandardError)
|
|
16
|
+
|
|
15
17
|
def initialize(configuration: nil)
|
|
16
18
|
@configuration = configuration || Configuration.new
|
|
17
19
|
end
|
|
@@ -65,6 +67,8 @@ module CommandTower
|
|
|
65
67
|
) do |http|
|
|
66
68
|
http.request(request)
|
|
67
69
|
end
|
|
70
|
+
rescue Net::OpenTimeout, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, SocketError => error
|
|
71
|
+
raise RequestNotSubmitted, error.message
|
|
68
72
|
end
|
|
69
73
|
|
|
70
74
|
def parse_send_response(response)
|
|
@@ -108,6 +108,31 @@ module CommandTower
|
|
|
108
108
|
now = Time.current
|
|
109
109
|
resolved_class = error_class || error&.class&.name || "UnknownError"
|
|
110
110
|
|
|
111
|
+
if Messaging::DeliveryRetryPolicy.push_channel?(delivery)
|
|
112
|
+
uncertain = Messaging::DeliveryRetryPolicy::SUBMISSION_UNCERTAIN
|
|
113
|
+
ActiveRecord::Base.transaction do
|
|
114
|
+
attempt.update!(
|
|
115
|
+
status: Messaging::DeliveryAttempt::STATUS_FAILED_TERMINAL,
|
|
116
|
+
finished_at: now,
|
|
117
|
+
error_code: uncertain,
|
|
118
|
+
error_class: resolved_class,
|
|
119
|
+
normalized_provider_status: nil,
|
|
120
|
+
provider_message_id: nil,
|
|
121
|
+
)
|
|
122
|
+
delivery.update!(status: Messaging::ChannelDelivery::STATUS_FAILED_TERMINAL)
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
delivery.reload
|
|
126
|
+
attempt.reload
|
|
127
|
+
OperationLogger.attempt_failed(channel_delivery: delivery, delivery_attempt: attempt)
|
|
128
|
+
OperationLogger.failed(
|
|
129
|
+
channel_delivery: delivery,
|
|
130
|
+
error_code: uncertain,
|
|
131
|
+
error_class: resolved_class,
|
|
132
|
+
)
|
|
133
|
+
return delivery
|
|
134
|
+
end
|
|
135
|
+
|
|
111
136
|
ActiveRecord::Base.transaction do
|
|
112
137
|
attempt.update!(
|
|
113
138
|
status: Messaging::DeliveryAttempt::STATUS_FAILED_RETRYABLE,
|
|
@@ -128,8 +153,17 @@ module CommandTower
|
|
|
128
153
|
error_code: "internal_adapter_error",
|
|
129
154
|
error_class: resolved_class,
|
|
130
155
|
)
|
|
131
|
-
rescue StandardError
|
|
132
|
-
|
|
156
|
+
rescue StandardError => error
|
|
157
|
+
OperationLogger.attempt_finalize_unpersisted(
|
|
158
|
+
channel_delivery: delivery,
|
|
159
|
+
delivery_attempt: attempt,
|
|
160
|
+
error_class: error.class.name,
|
|
161
|
+
)
|
|
162
|
+
if Messaging::DeliveryRetryPolicy.push_channel?(delivery)
|
|
163
|
+
mark_failed_terminal_best_effort!(delivery)
|
|
164
|
+
else
|
|
165
|
+
mark_failed_retryable_best_effort!(delivery)
|
|
166
|
+
end
|
|
133
167
|
end
|
|
134
168
|
|
|
135
169
|
private
|
|
@@ -143,6 +177,12 @@ module CommandTower
|
|
|
143
177
|
rescue StandardError
|
|
144
178
|
nil
|
|
145
179
|
end
|
|
180
|
+
|
|
181
|
+
def mark_failed_terminal_best_effort!(delivery)
|
|
182
|
+
delivery.update(status: Messaging::ChannelDelivery::STATUS_FAILED_TERMINAL)
|
|
183
|
+
rescue StandardError
|
|
184
|
+
nil
|
|
185
|
+
end
|
|
146
186
|
end
|
|
147
187
|
end
|
|
148
188
|
end
|
|
@@ -103,6 +103,16 @@ module CommandTower
|
|
|
103
103
|
)
|
|
104
104
|
end
|
|
105
105
|
|
|
106
|
+
def attempt_finalize_unpersisted(channel_delivery:, delivery_attempt:, error_class:)
|
|
107
|
+
emit(
|
|
108
|
+
:error,
|
|
109
|
+
"messaging.execution.attempt.finalize_unpersisted",
|
|
110
|
+
channel_delivery:,
|
|
111
|
+
delivery_attempt_id: delivery_attempt.id,
|
|
112
|
+
error_class:,
|
|
113
|
+
)
|
|
114
|
+
end
|
|
115
|
+
|
|
106
116
|
def render_started(channel_delivery:, delivery_attempt:)
|
|
107
117
|
emit_render(
|
|
108
118
|
:info,
|
|
@@ -43,30 +43,86 @@ module CommandTower
|
|
|
43
43
|
Messaging::ChannelDelivery
|
|
44
44
|
.queued_for_execution
|
|
45
45
|
.where(Messaging::ChannelDelivery.arel_table[:updated_at].lt(grace_before))
|
|
46
|
-
.find_each { |delivery|
|
|
46
|
+
.find_each { |delivery| redrive_queued_delivery!(delivery) }
|
|
47
47
|
end
|
|
48
48
|
|
|
49
49
|
def recover_failed!
|
|
50
50
|
Messaging::ChannelDelivery
|
|
51
51
|
.failed_retryable
|
|
52
52
|
.where(Messaging::ChannelDelivery.arel_table[:updated_at].lt(grace_before))
|
|
53
|
-
.
|
|
54
|
-
.find_each do |delivery|
|
|
55
|
-
reset_to_queued!(delivery)
|
|
56
|
-
redrive!(delivery.reload)
|
|
57
|
-
end
|
|
53
|
+
.find_each { |delivery| redrive_failed_delivery!(delivery) }
|
|
58
54
|
end
|
|
59
55
|
|
|
60
56
|
def recover_stale_executing!
|
|
61
57
|
Messaging::ChannelDelivery
|
|
62
58
|
.executing
|
|
63
59
|
.where(Messaging::ChannelDelivery.arel_table[:execution_claimed_at].lt(grace_before))
|
|
64
|
-
.find_each
|
|
65
|
-
|
|
60
|
+
.find_each { |delivery| redrive_stale_executing_delivery!(delivery) }
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def redrive_queued_delivery!(delivery)
|
|
64
|
+
if Messaging::DeliveryRetryPolicy.push_channel?(delivery)
|
|
65
|
+
if Messaging::DeliveryRetryPolicy.push_redrive_eligible_while_queued?(delivery)
|
|
66
|
+
redrive!(delivery)
|
|
67
|
+
else
|
|
68
|
+
terminalize_push_uncertain!(delivery)
|
|
69
|
+
end
|
|
70
|
+
return
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
redrive!(delivery)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def redrive_failed_delivery!(delivery)
|
|
77
|
+
if Messaging::DeliveryRetryPolicy.push_channel?(delivery)
|
|
78
|
+
if Messaging::DeliveryRetryPolicy.push_redrive_eligible_from_failed_retryable?(delivery)
|
|
79
|
+
reset_to_queued!(delivery)
|
|
80
|
+
redrive!(delivery.reload)
|
|
81
|
+
else
|
|
82
|
+
terminalize_push_uncertain!(delivery)
|
|
83
|
+
end
|
|
84
|
+
return
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
return if delivery.execution_attempt_count >= @max_attempts
|
|
88
|
+
|
|
89
|
+
reset_to_queued!(delivery)
|
|
90
|
+
redrive!(delivery.reload)
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def redrive_stale_executing_delivery!(delivery)
|
|
94
|
+
max_attempts = Messaging::DeliveryRetryPolicy.max_execution_attempts_for(delivery)
|
|
95
|
+
return if delivery.execution_attempt_count >= max_attempts
|
|
66
96
|
|
|
97
|
+
if Messaging::DeliveryRetryPolicy.push_channel?(delivery)
|
|
98
|
+
if Messaging::DeliveryRetryPolicy.push_redrive_eligible_while_queued?(delivery)
|
|
67
99
|
reset_to_queued!(delivery)
|
|
68
100
|
redrive!(delivery.reload)
|
|
101
|
+
else
|
|
102
|
+
terminalize_push_uncertain!(delivery)
|
|
69
103
|
end
|
|
104
|
+
return
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
reset_to_queued!(delivery)
|
|
108
|
+
redrive!(delivery.reload)
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def terminalize_push_uncertain!(delivery)
|
|
112
|
+
now = Time.current
|
|
113
|
+
uncertain = Messaging::DeliveryRetryPolicy::SUBMISSION_UNCERTAIN
|
|
114
|
+
|
|
115
|
+
Messaging::DeliveryAttempt
|
|
116
|
+
.where(channel_delivery_id: delivery.id, status: Messaging::DeliveryAttempt::STATUS_STARTED, finished_at: nil)
|
|
117
|
+
.find_each do |attempt|
|
|
118
|
+
attempt.update!(
|
|
119
|
+
status: Messaging::DeliveryAttempt::STATUS_FAILED_TERMINAL,
|
|
120
|
+
finished_at: now,
|
|
121
|
+
error_code: uncertain,
|
|
122
|
+
)
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
delivery.update!(status: Messaging::ChannelDelivery::STATUS_FAILED_TERMINAL)
|
|
70
126
|
end
|
|
71
127
|
|
|
72
128
|
def reset_to_queued!(delivery)
|
|
@@ -108,6 +108,25 @@ module CommandTower
|
|
|
108
108
|
)
|
|
109
109
|
end
|
|
110
110
|
|
|
111
|
+
unless CommandTower::Messaging::DeliveryRetryPolicy.push_may_submit_to_provider?(
|
|
112
|
+
delivery,
|
|
113
|
+
current_attempt_id: attempt.id,
|
|
114
|
+
)
|
|
115
|
+
ledger.finalize_terminal(
|
|
116
|
+
delivery:,
|
|
117
|
+
attempt:,
|
|
118
|
+
error_code: CommandTower::Messaging::DeliveryRetryPolicy::SUBMISSION_UNCERTAIN,
|
|
119
|
+
)
|
|
120
|
+
return success(
|
|
121
|
+
payload: {
|
|
122
|
+
outcome: :terminal,
|
|
123
|
+
channel_delivery_id: delivery.id,
|
|
124
|
+
error_code: CommandTower::Messaging::DeliveryRetryPolicy::SUBMISSION_UNCERTAIN,
|
|
125
|
+
},
|
|
126
|
+
http_status: :ok,
|
|
127
|
+
)
|
|
128
|
+
end
|
|
129
|
+
|
|
111
130
|
request = CommandTower::Messaging::Execution::AdapterRequest.build(
|
|
112
131
|
channel_delivery_id: delivery.id,
|
|
113
132
|
communication_id: delivery.communication_id,
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Upgrade: CommandTower 0.20.0
|
|
2
|
+
|
|
3
|
+
**From:** `0.19.0`
|
|
4
|
+
**To:** `0.20.0`
|
|
5
|
+
|
|
6
|
+
Minor release: push delivery certainty — persist Expo ticket metadata reliably on MariaDB and stop duplicate Expo sends when submission is uncertain. Email retry policy and Inbox behavior are unchanged.
|
|
7
|
+
|
|
8
|
+
## Host-visible changes
|
|
9
|
+
|
|
10
|
+
| Change | Host impact |
|
|
11
|
+
|--------|-------------|
|
|
12
|
+
| `DeliveryAttempt#provider_metadata` | JSON attribute caster (MariaDB LONGTEXT / `json_valid`). Accepted Expo ticket metadata persists on success and retryable non-submission paths. |
|
|
13
|
+
| Push execution recovery | Push-only duplicate-averse policy: at most one automatic retry when a finished attempt recorded `submission: not_submitted`. Uncertain outcomes (`submission_uncertain`), accepted tickets, and open `started` attempts without proof terminalize without another Expo POST. Push claim budget `PUSH_RETRY_LIMIT` = 2. Email keeps grace window and `MAX_ATTEMPTS` = 5. |
|
|
14
|
+
| `DeliverWorkflow` | Pre-send guard blocks Expo when prior push state does not allow another submission. |
|
|
15
|
+
| Expo adapter | Classifies pre-request failures as `not_submitted` (retryable once); post-dispatch uncertainty is terminal. |
|
|
16
|
+
| Observability | `messaging.execution.attempt.finalize_unpersisted` when attempt finalization cannot be persisted (diagnostic). |
|
|
17
|
+
|
|
18
|
+
## Host actions
|
|
19
|
+
|
|
20
|
+
1. Bump gem to `0.20.0` (or `>= 0.20.0`).
|
|
21
|
+
2. **No new migration.** Run `bundle exec rails db:migrate` only if other pending host migrations exist.
|
|
22
|
+
3. After deploy, legacy push rows with `started` attempts and null metadata are terminalized on the next recovery pass (no data repair script).
|
|
23
|
+
|
|
24
|
+
## Not in this release
|
|
25
|
+
|
|
26
|
+
- No Inbox document or rendering changes.
|
|
27
|
+
- No email adapter or email retry cap changes.
|
|
28
|
+
- No new HTTP routes or workflows.
|
|
29
|
+
- No automatic backfill of historical `provider_metadata`.
|
data/docs/upgrades/README.md
CHANGED
|
@@ -4,6 +4,7 @@ Host-facing upgrade / change summaries for CommandTower releases.
|
|
|
4
4
|
|
|
5
5
|
| Version | Summary |
|
|
6
6
|
|---------|---------|
|
|
7
|
+
| [0.20.0](0.20.0.md) | Push delivery certainty: JSON `provider_metadata` caster; duplicate-averse push recovery and Expo submission guards |
|
|
7
8
|
| [0.19.0](0.19.0.md) | Push presentation: badge, urgency, sound, deep links; Expo ticket and receipt metadata; preference urgency and sound |
|
|
8
9
|
| [0.18.0](0.18.0.md) | Inbox conversations: optional Produce identity, grouped list entries, `GET /me/inbox/conversation` window of at most 5, accept-time presentation snapshot |
|
|
9
10
|
| [0.17.0](0.17.0.md) | Client-version-compatibility gating (`observe`/`enforce`); Rich Messaging Inbox `content` documents; host-overridable rendering templates; shared email theming |
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: command_tower
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.20.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- matt-taylor
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-10-
|
|
11
|
+
date: 2026-10-15 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: phonelib
|
|
@@ -860,6 +860,7 @@ files:
|
|
|
860
860
|
- docs/upgrades/0.17.0.md
|
|
861
861
|
- docs/upgrades/0.18.0.md
|
|
862
862
|
- docs/upgrades/0.19.0.md
|
|
863
|
+
- docs/upgrades/0.20.0.md
|
|
863
864
|
- docs/upgrades/README.md
|
|
864
865
|
- lib/command_tower.rb
|
|
865
866
|
- lib/command_tower/admin_scope.rb
|