@goodandready/dsh-messenger-gateway 0.4.26 → 0.4.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/outbox.js ADDED
@@ -0,0 +1,262 @@
1
+ /**
2
+ * Outbound Transactional Queue (Outbox DLQ) with Offline Recovery & Backoff (#155).
3
+ *
4
+ * Guarantees zero message loss during temporary network hiccups, Telegram 429
5
+ * throttling, 502/504 gateway errors, and process restarts:
6
+ * - Atomic persistent disk queue (~/.dsh/storages/outbox-queue.json).
7
+ * - Exponential backoff with jitter and Telegram 429 `retry_after` awareness.
8
+ * - Dead Letter Queue (DLQ) for items failing after max retries or terminal errors.
9
+ * - Recovery of pending uncommitted outbound jobs on gateway startup.
10
+ */
11
+
12
+ import { existsSync, readFileSync } from 'node:fs'
13
+ import { join } from 'node:path'
14
+ import { homedir } from 'node:os'
15
+ import { randomUUID } from 'node:crypto'
16
+ import { writeJsonAtomic } from './storage-atomic.js'
17
+ import { isTopicGoneError } from './telegram-errors.js'
18
+
19
+ export function resolveOutboxStoragePath(baseDir) {
20
+ if (baseDir) return join(baseDir, 'outbox-queue.json')
21
+ const dshHome = (typeof process !== 'undefined' && process.env?.DSH_HOME) || ''
22
+ if (dshHome) return join(dshHome, 'storages', 'outbox-queue.json')
23
+ return join(homedir(), '.dsh', 'storages', 'outbox-queue.json')
24
+ }
25
+
26
+ export function parseRetryAfter(err) {
27
+ if (!err) return null
28
+ if (typeof err.parameters?.retry_after === 'number') {
29
+ return err.parameters.retry_after
30
+ }
31
+ if (typeof err.response?.parameters?.retry_after === 'number') {
32
+ return err.response.parameters.retry_after
33
+ }
34
+ const msg = String(err.message || '')
35
+ const match = msg.match(/retry after (\d+)/i)
36
+ if (match) {
37
+ const val = parseInt(match[1], 10)
38
+ if (!Number.isNaN(val) && val > 0) return val
39
+ }
40
+ return null
41
+ }
42
+
43
+ export function computeOutboxBackoffMs(attempts, retryAfterSeconds, maxDelayMs = 60_000) {
44
+ const jitter = Math.floor(Math.random() * 400)
45
+ if (typeof retryAfterSeconds === 'number' && retryAfterSeconds > 0) {
46
+ return Math.min(maxDelayMs, (retryAfterSeconds * 1000) + jitter)
47
+ }
48
+ const count = Math.max(1, Math.min(10, Number(attempts) || 1))
49
+ const baseDelay = 1000 * Math.pow(2, count - 1)
50
+ return Math.min(maxDelayMs, baseDelay + jitter)
51
+ }
52
+
53
+ export function isTerminalSendError(err) {
54
+ if (!err) return false
55
+ if (isTopicGoneError(err)) return true
56
+ const msg = String(err.message || '').toLowerCase()
57
+ return (
58
+ msg.includes('bot was blocked by the user') ||
59
+ msg.includes('user is deactivated') ||
60
+ msg.includes('chat not found') ||
61
+ msg.includes('message is too long') ||
62
+ msg.includes('message text is empty') ||
63
+ msg.includes("can't parse entities") ||
64
+ msg.includes('wrong file identifier') ||
65
+ msg.includes('have no rights to send a message')
66
+ )
67
+ }
68
+
69
+ export function createOutboxQueue(options = {}) {
70
+ const {
71
+ storagePath = resolveOutboxStoragePath(),
72
+ senderFn,
73
+ onDlq,
74
+ logger,
75
+ flushIntervalMs = 1000,
76
+ defaultMaxAttempts = 5,
77
+ } = options
78
+
79
+ let pending = []
80
+ let dlq = []
81
+ let timer = null
82
+ let isFlushing = false
83
+ let isStopped = false
84
+
85
+ function loadSync() {
86
+ try {
87
+ if (existsSync(storagePath)) {
88
+ const raw = readFileSync(storagePath, 'utf8')
89
+ const data = JSON.parse(raw)
90
+ if (Array.isArray(data.pending)) pending = data.pending
91
+ if (Array.isArray(data.dlq)) dlq = data.dlq
92
+ if (pending.length > 0 && logger) {
93
+ logger.info?.(`[outbox] recovered ${pending.length} pending outbound jobs from disk`)
94
+ }
95
+ }
96
+ } catch (err) {
97
+ if (logger) {
98
+ logger.warn?.(`[outbox] failed to load storage: ${err.message}`)
99
+ }
100
+ }
101
+ }
102
+
103
+ loadSync()
104
+
105
+ async function persist() {
106
+ try {
107
+ await writeJsonAtomic(storagePath, { pending, dlq })
108
+ } catch (err) {
109
+ if (logger) {
110
+ logger.warn?.(`[outbox] failed to persist queue to disk: ${err.message}`)
111
+ }
112
+ }
113
+ }
114
+
115
+ async function flush() {
116
+ if (isFlushing || isStopped) return
117
+ if (!pending.length) return
118
+ isFlushing = true
119
+ const now = Date.now()
120
+ let dirty = false
121
+
122
+ try {
123
+ const readyJobs = pending.filter((j) => (j.nextRetryAt || 0) <= now)
124
+ for (const job of readyJobs) {
125
+ if (isStopped) break
126
+ try {
127
+ if (typeof senderFn !== 'function') {
128
+ throw new Error('outbox senderFn not configured')
129
+ }
130
+ await senderFn(job)
131
+ pending = pending.filter((j) => j.id !== job.id)
132
+ dirty = true
133
+ } catch (err) {
134
+ job.attempts = (job.attempts || 0) + 1
135
+ job.lastError = err?.message || String(err)
136
+ const terminal = isTerminalSendError(err)
137
+ const max = job.maxAttempts || defaultMaxAttempts
138
+
139
+ if (terminal || job.attempts >= max) {
140
+ pending = pending.filter((j) => j.id !== job.id)
141
+ dlq.unshift({
142
+ id: job.id,
143
+ type: job.type,
144
+ target: job.target,
145
+ payload: job.payload,
146
+ attempts: job.attempts,
147
+ createdAt: job.createdAt,
148
+ failedAt: Date.now(),
149
+ terminal,
150
+ error: job.lastError,
151
+ })
152
+ if (dlq.length > 100) dlq.length = 100
153
+ dirty = true
154
+ if (typeof onDlq === 'function') {
155
+ try { onDlq(job, err) } catch (dlqErr) {
156
+ logger?.warn?.(`[outbox] onDlq error: ${dlqErr.message}`)
157
+ }
158
+ }
159
+ } else {
160
+ const retryAfter = parseRetryAfter(err)
161
+ const delayMs = computeOutboxBackoffMs(job.attempts, retryAfter)
162
+ job.nextRetryAt = Date.now() + delayMs
163
+ dirty = true
164
+ if (logger) {
165
+ logger.warn?.(`[outbox] job ${job.id} failed (attempt ${job.attempts}/${max}), retrying in ${Math.round(delayMs / 1000)}s: ${job.lastError}`)
166
+ }
167
+ }
168
+ }
169
+ }
170
+ } finally {
171
+ isFlushing = false
172
+ if (dirty) {
173
+ await persist()
174
+ }
175
+ }
176
+ }
177
+
178
+ async function enqueue(job) {
179
+ const item = {
180
+ id: job.id || randomUUID(),
181
+ type: job.type || 'message',
182
+ target: job.target,
183
+ payload: job.payload,
184
+ attempts: 0,
185
+ maxAttempts: job.maxAttempts || defaultMaxAttempts,
186
+ createdAt: Date.now(),
187
+ nextRetryAt: Date.now(),
188
+ lastError: null,
189
+ }
190
+ pending.push(item)
191
+ await persist()
192
+ void flush()
193
+ return item
194
+ }
195
+
196
+ function start() {
197
+ isStopped = false
198
+ void flush()
199
+ if (!timer && flushIntervalMs > 0) {
200
+ timer = setInterval(() => { void flush() }, flushIntervalMs)
201
+ timer.unref?.()
202
+ }
203
+ }
204
+
205
+ function stop() {
206
+ isStopped = true
207
+ if (timer) {
208
+ clearInterval(timer)
209
+ timer = null
210
+ }
211
+ }
212
+
213
+ function status() {
214
+ return {
215
+ pendingCount: pending.length,
216
+ dlqCount: dlq.length,
217
+ pending: [...pending],
218
+ dlq: [...dlq],
219
+ }
220
+ }
221
+
222
+ async function retryDlq() {
223
+ if (!dlq.length) return 0
224
+ const count = dlq.length
225
+ for (const item of dlq) {
226
+ pending.push({
227
+ id: item.id || randomUUID(),
228
+ type: item.type,
229
+ target: item.target,
230
+ payload: item.payload,
231
+ attempts: 0,
232
+ maxAttempts: defaultMaxAttempts,
233
+ createdAt: Date.now(),
234
+ nextRetryAt: Date.now(),
235
+ lastError: null,
236
+ })
237
+ }
238
+ dlq = []
239
+ await persist()
240
+ await flush()
241
+ return count
242
+ }
243
+
244
+ async function clearDlq() {
245
+ const count = dlq.length
246
+ dlq = []
247
+ await persist()
248
+ return count
249
+ }
250
+
251
+ return {
252
+ enqueue,
253
+ flush,
254
+ start,
255
+ stop,
256
+ status,
257
+ retryDlq,
258
+ clearDlq,
259
+ get pending() { return pending },
260
+ get dlq() { return dlq },
261
+ }
262
+ }