@goodandready/dsh-messenger-gateway 0.4.26 → 0.4.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/adapters/index.js +1 -0
- package/lib/adapters/telegram.js +16 -1
- package/lib/artifacts.js +16 -1
- package/lib/commands.js +13 -5
- package/lib/config.js +4 -0
- package/lib/cost-tracker.js +228 -0
- package/lib/gateway-commands.js +62 -0
- package/lib/gateway-turn.js +39 -0
- package/lib/gateway.js +37 -0
- package/lib/index.js +15 -0
- package/lib/outbox.js +262 -0
- package/lib/webapp.js +723 -0
- package/package.json +1 -1
package/lib/outbox.js
ADDED
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Outbound Transactional Queue (Outbox DLQ) with Offline Recovery & Backoff (#155).
|
|
3
|
+
*
|
|
4
|
+
* Guarantees zero message loss during temporary network hiccups, Telegram 429
|
|
5
|
+
* throttling, 502/504 gateway errors, and process restarts:
|
|
6
|
+
* - Atomic persistent disk queue (~/.dsh/storages/outbox-queue.json).
|
|
7
|
+
* - Exponential backoff with jitter and Telegram 429 `retry_after` awareness.
|
|
8
|
+
* - Dead Letter Queue (DLQ) for items failing after max retries or terminal errors.
|
|
9
|
+
* - Recovery of pending uncommitted outbound jobs on gateway startup.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { existsSync, readFileSync } from 'node:fs'
|
|
13
|
+
import { join } from 'node:path'
|
|
14
|
+
import { homedir } from 'node:os'
|
|
15
|
+
import { randomUUID } from 'node:crypto'
|
|
16
|
+
import { writeJsonAtomic } from './storage-atomic.js'
|
|
17
|
+
import { isTopicGoneError } from './telegram-errors.js'
|
|
18
|
+
|
|
19
|
+
export function resolveOutboxStoragePath(baseDir) {
|
|
20
|
+
if (baseDir) return join(baseDir, 'outbox-queue.json')
|
|
21
|
+
const dshHome = (typeof process !== 'undefined' && process.env?.DSH_HOME) || ''
|
|
22
|
+
if (dshHome) return join(dshHome, 'storages', 'outbox-queue.json')
|
|
23
|
+
return join(homedir(), '.dsh', 'storages', 'outbox-queue.json')
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export function parseRetryAfter(err) {
|
|
27
|
+
if (!err) return null
|
|
28
|
+
if (typeof err.parameters?.retry_after === 'number') {
|
|
29
|
+
return err.parameters.retry_after
|
|
30
|
+
}
|
|
31
|
+
if (typeof err.response?.parameters?.retry_after === 'number') {
|
|
32
|
+
return err.response.parameters.retry_after
|
|
33
|
+
}
|
|
34
|
+
const msg = String(err.message || '')
|
|
35
|
+
const match = msg.match(/retry after (\d+)/i)
|
|
36
|
+
if (match) {
|
|
37
|
+
const val = parseInt(match[1], 10)
|
|
38
|
+
if (!Number.isNaN(val) && val > 0) return val
|
|
39
|
+
}
|
|
40
|
+
return null
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export function computeOutboxBackoffMs(attempts, retryAfterSeconds, maxDelayMs = 60_000) {
|
|
44
|
+
const jitter = Math.floor(Math.random() * 400)
|
|
45
|
+
if (typeof retryAfterSeconds === 'number' && retryAfterSeconds > 0) {
|
|
46
|
+
return Math.min(maxDelayMs, (retryAfterSeconds * 1000) + jitter)
|
|
47
|
+
}
|
|
48
|
+
const count = Math.max(1, Math.min(10, Number(attempts) || 1))
|
|
49
|
+
const baseDelay = 1000 * Math.pow(2, count - 1)
|
|
50
|
+
return Math.min(maxDelayMs, baseDelay + jitter)
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function isTerminalSendError(err) {
|
|
54
|
+
if (!err) return false
|
|
55
|
+
if (isTopicGoneError(err)) return true
|
|
56
|
+
const msg = String(err.message || '').toLowerCase()
|
|
57
|
+
return (
|
|
58
|
+
msg.includes('bot was blocked by the user') ||
|
|
59
|
+
msg.includes('user is deactivated') ||
|
|
60
|
+
msg.includes('chat not found') ||
|
|
61
|
+
msg.includes('message is too long') ||
|
|
62
|
+
msg.includes('message text is empty') ||
|
|
63
|
+
msg.includes("can't parse entities") ||
|
|
64
|
+
msg.includes('wrong file identifier') ||
|
|
65
|
+
msg.includes('have no rights to send a message')
|
|
66
|
+
)
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export function createOutboxQueue(options = {}) {
|
|
70
|
+
const {
|
|
71
|
+
storagePath = resolveOutboxStoragePath(),
|
|
72
|
+
senderFn,
|
|
73
|
+
onDlq,
|
|
74
|
+
logger,
|
|
75
|
+
flushIntervalMs = 1000,
|
|
76
|
+
defaultMaxAttempts = 5,
|
|
77
|
+
} = options
|
|
78
|
+
|
|
79
|
+
let pending = []
|
|
80
|
+
let dlq = []
|
|
81
|
+
let timer = null
|
|
82
|
+
let isFlushing = false
|
|
83
|
+
let isStopped = false
|
|
84
|
+
|
|
85
|
+
function loadSync() {
|
|
86
|
+
try {
|
|
87
|
+
if (existsSync(storagePath)) {
|
|
88
|
+
const raw = readFileSync(storagePath, 'utf8')
|
|
89
|
+
const data = JSON.parse(raw)
|
|
90
|
+
if (Array.isArray(data.pending)) pending = data.pending
|
|
91
|
+
if (Array.isArray(data.dlq)) dlq = data.dlq
|
|
92
|
+
if (pending.length > 0 && logger) {
|
|
93
|
+
logger.info?.(`[outbox] recovered ${pending.length} pending outbound jobs from disk`)
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
} catch (err) {
|
|
97
|
+
if (logger) {
|
|
98
|
+
logger.warn?.(`[outbox] failed to load storage: ${err.message}`)
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
loadSync()
|
|
104
|
+
|
|
105
|
+
async function persist() {
|
|
106
|
+
try {
|
|
107
|
+
await writeJsonAtomic(storagePath, { pending, dlq })
|
|
108
|
+
} catch (err) {
|
|
109
|
+
if (logger) {
|
|
110
|
+
logger.warn?.(`[outbox] failed to persist queue to disk: ${err.message}`)
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
async function flush() {
|
|
116
|
+
if (isFlushing || isStopped) return
|
|
117
|
+
if (!pending.length) return
|
|
118
|
+
isFlushing = true
|
|
119
|
+
const now = Date.now()
|
|
120
|
+
let dirty = false
|
|
121
|
+
|
|
122
|
+
try {
|
|
123
|
+
const readyJobs = pending.filter((j) => (j.nextRetryAt || 0) <= now)
|
|
124
|
+
for (const job of readyJobs) {
|
|
125
|
+
if (isStopped) break
|
|
126
|
+
try {
|
|
127
|
+
if (typeof senderFn !== 'function') {
|
|
128
|
+
throw new Error('outbox senderFn not configured')
|
|
129
|
+
}
|
|
130
|
+
await senderFn(job)
|
|
131
|
+
pending = pending.filter((j) => j.id !== job.id)
|
|
132
|
+
dirty = true
|
|
133
|
+
} catch (err) {
|
|
134
|
+
job.attempts = (job.attempts || 0) + 1
|
|
135
|
+
job.lastError = err?.message || String(err)
|
|
136
|
+
const terminal = isTerminalSendError(err)
|
|
137
|
+
const max = job.maxAttempts || defaultMaxAttempts
|
|
138
|
+
|
|
139
|
+
if (terminal || job.attempts >= max) {
|
|
140
|
+
pending = pending.filter((j) => j.id !== job.id)
|
|
141
|
+
dlq.unshift({
|
|
142
|
+
id: job.id,
|
|
143
|
+
type: job.type,
|
|
144
|
+
target: job.target,
|
|
145
|
+
payload: job.payload,
|
|
146
|
+
attempts: job.attempts,
|
|
147
|
+
createdAt: job.createdAt,
|
|
148
|
+
failedAt: Date.now(),
|
|
149
|
+
terminal,
|
|
150
|
+
error: job.lastError,
|
|
151
|
+
})
|
|
152
|
+
if (dlq.length > 100) dlq.length = 100
|
|
153
|
+
dirty = true
|
|
154
|
+
if (typeof onDlq === 'function') {
|
|
155
|
+
try { onDlq(job, err) } catch (dlqErr) {
|
|
156
|
+
logger?.warn?.(`[outbox] onDlq error: ${dlqErr.message}`)
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
} else {
|
|
160
|
+
const retryAfter = parseRetryAfter(err)
|
|
161
|
+
const delayMs = computeOutboxBackoffMs(job.attempts, retryAfter)
|
|
162
|
+
job.nextRetryAt = Date.now() + delayMs
|
|
163
|
+
dirty = true
|
|
164
|
+
if (logger) {
|
|
165
|
+
logger.warn?.(`[outbox] job ${job.id} failed (attempt ${job.attempts}/${max}), retrying in ${Math.round(delayMs / 1000)}s: ${job.lastError}`)
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
} finally {
|
|
171
|
+
isFlushing = false
|
|
172
|
+
if (dirty) {
|
|
173
|
+
await persist()
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
async function enqueue(job) {
|
|
179
|
+
const item = {
|
|
180
|
+
id: job.id || randomUUID(),
|
|
181
|
+
type: job.type || 'message',
|
|
182
|
+
target: job.target,
|
|
183
|
+
payload: job.payload,
|
|
184
|
+
attempts: 0,
|
|
185
|
+
maxAttempts: job.maxAttempts || defaultMaxAttempts,
|
|
186
|
+
createdAt: Date.now(),
|
|
187
|
+
nextRetryAt: Date.now(),
|
|
188
|
+
lastError: null,
|
|
189
|
+
}
|
|
190
|
+
pending.push(item)
|
|
191
|
+
await persist()
|
|
192
|
+
void flush()
|
|
193
|
+
return item
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
function start() {
|
|
197
|
+
isStopped = false
|
|
198
|
+
void flush()
|
|
199
|
+
if (!timer && flushIntervalMs > 0) {
|
|
200
|
+
timer = setInterval(() => { void flush() }, flushIntervalMs)
|
|
201
|
+
timer.unref?.()
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
function stop() {
|
|
206
|
+
isStopped = true
|
|
207
|
+
if (timer) {
|
|
208
|
+
clearInterval(timer)
|
|
209
|
+
timer = null
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
function status() {
|
|
214
|
+
return {
|
|
215
|
+
pendingCount: pending.length,
|
|
216
|
+
dlqCount: dlq.length,
|
|
217
|
+
pending: [...pending],
|
|
218
|
+
dlq: [...dlq],
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
async function retryDlq() {
|
|
223
|
+
if (!dlq.length) return 0
|
|
224
|
+
const count = dlq.length
|
|
225
|
+
for (const item of dlq) {
|
|
226
|
+
pending.push({
|
|
227
|
+
id: item.id || randomUUID(),
|
|
228
|
+
type: item.type,
|
|
229
|
+
target: item.target,
|
|
230
|
+
payload: item.payload,
|
|
231
|
+
attempts: 0,
|
|
232
|
+
maxAttempts: defaultMaxAttempts,
|
|
233
|
+
createdAt: Date.now(),
|
|
234
|
+
nextRetryAt: Date.now(),
|
|
235
|
+
lastError: null,
|
|
236
|
+
})
|
|
237
|
+
}
|
|
238
|
+
dlq = []
|
|
239
|
+
await persist()
|
|
240
|
+
await flush()
|
|
241
|
+
return count
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
async function clearDlq() {
|
|
245
|
+
const count = dlq.length
|
|
246
|
+
dlq = []
|
|
247
|
+
await persist()
|
|
248
|
+
return count
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
return {
|
|
252
|
+
enqueue,
|
|
253
|
+
flush,
|
|
254
|
+
start,
|
|
255
|
+
stop,
|
|
256
|
+
status,
|
|
257
|
+
retryDlq,
|
|
258
|
+
clearDlq,
|
|
259
|
+
get pending() { return pending },
|
|
260
|
+
get dlq() { return dlq },
|
|
261
|
+
}
|
|
262
|
+
}
|