@ardennguyen/zalo-agent-cli 2.5.5 → 2.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +9 -0
- package/README.md +10 -6
- package/package.json +1 -1
- package/src/commands/fleet.js +439 -0
- package/src/commands/listen.js +87 -24
- package/src/commands/mcp-webhook.js +197 -0
- package/src/commands/mcp.js +162 -37
- package/src/commands/msg.js +13 -1
- package/src/core/accounts.js +182 -40
- package/src/core/credentials.js +5 -3
- package/src/core/daemon-channel.js +145 -19
- package/src/core/daemon-health.js +403 -0
- package/src/core/fleet-templates.js +177 -0
- package/src/core/fleet.js +255 -0
- package/src/core/log-sink.js +311 -0
- package/src/core/oa-client.js +10 -6
- package/src/core/startup-gate.js +206 -0
- package/src/core/webhook-outbox.js +575 -0
- package/src/index.js +13 -3
- package/src/mcp/mcp-config.js +42 -2
- package/src/mcp/mcp-http-transport.js +37 -18
- package/src/mcp/mcp-tools.js +82 -36
- package/src/mcp/message-buffer.js +94 -4
- package/src/mcp/wait-result.js +52 -0
- package/src/utils/atomic-file.js +209 -0
- package/src/utils/markdown.js +339 -0
|
@@ -0,0 +1,575 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Durable webhook delivery: an on-disk outbox and the dispatcher that drains it.
|
|
3
|
+
*
|
|
4
|
+
* `mcp start -w` and `listen -w` used to POST once, with a 5 second timeout, and forget: a receiver
|
|
5
|
+
* that was down, restarting or slow lost that message with nothing to show for it. Both now write
|
|
6
|
+
* the event here first, then deliver from this queue, so a receiver outage becomes a backlog that is
|
|
7
|
+
* visible (`mcp webhook status`, /health, /metrics) and drains by itself when the receiver returns.
|
|
8
|
+
*
|
|
9
|
+
* Guarantees, and what they are not:
|
|
10
|
+
* - **At least once.** A row is deleted only after a 2xx. A crash between the receiver's reply and
|
|
11
|
+
* the delete re-sends it, so a receiver de-duplicates on the event's message id.
|
|
12
|
+
* - **In order.** Rows go out oldest first, and the head's retry backoff holds the whole queue, so a
|
|
13
|
+
* later message never overtakes an earlier one. A payload the receiver rejects for being malformed
|
|
14
|
+
* (400, 413, 422) is dead-lettered instead, or one poison event would block everything behind it.
|
|
15
|
+
* - **Bounded.** A row older than `maxAgeMs` is dead-lettered (kept, never silently dropped) so a
|
|
16
|
+
* receiver that never returns cannot grow the file without end unnoticed.
|
|
17
|
+
* - Not exactly once, and not shared between processes: one daemon per account owns the dispatcher.
|
|
18
|
+
*
|
|
19
|
+
* The file is `accounts/<ownId>/webhook-outbox.db`, separate from `zalo.db`: it holds no message rows
|
|
20
|
+
* (AGENTS.md section 13 limits those to three writers) and a damaged outbox can be deleted without
|
|
21
|
+
* touching the cache.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import { createHmac } from "node:crypto";
|
|
25
|
+
import { mkdirSync } from "node:fs";
|
|
26
|
+
import { join } from "node:path";
|
|
27
|
+
import Database from "better-sqlite3";
|
|
28
|
+
|
|
29
|
+
export const OUTBOX_FILE = "webhook-outbox.db";
|
|
30
|
+
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
31
|
+
const BACKOFF_BASE_MS = 1000;
|
|
32
|
+
const BACKOFF_CAP_MS = 5 * 60 * 1000;
|
|
33
|
+
/** A receiver saying "this payload is malformed" will say it again; retrying only blocks the queue. */
|
|
34
|
+
const PAYLOAD_REJECTED = new Set([400, 413, 422]);
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* @param {string} accountDir - `accounts/<ownId>`
|
|
38
|
+
* @returns {string}
|
|
39
|
+
*/
|
|
40
|
+
export function outboxPath(accountDir) {
|
|
41
|
+
return join(accountDir, OUTBOX_FILE);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Sign a webhook body the way every webhook sender here does: `Authorization: Bearer <secret>` and
|
|
46
|
+
* `X-Zalo-Signature: sha256=<hex HMAC of the exact bytes sent>`.
|
|
47
|
+
*
|
|
48
|
+
* @param {string} body
|
|
49
|
+
* @param {string|null} secret
|
|
50
|
+
* @returns {Record<string, string>}
|
|
51
|
+
*/
|
|
52
|
+
export function webhookHeaders(body, secret) {
|
|
53
|
+
const headers = { "Content-Type": "application/json" };
|
|
54
|
+
if (typeof secret === "string" && secret.trim()) {
|
|
55
|
+
const s = secret.trim();
|
|
56
|
+
headers["Authorization"] = `Bearer ${s}`;
|
|
57
|
+
headers["X-Zalo-Signature"] = `sha256=${createHmac("sha256", s).update(body).digest("hex")}`;
|
|
58
|
+
}
|
|
59
|
+
return headers;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Open (creating if needed) the outbox of one account.
|
|
64
|
+
*
|
|
65
|
+
* @param {string} accountDir - `accounts/<ownId>`
|
|
66
|
+
* @param {{ now?: () => number, readonly?: boolean }} [options] - `readonly` for status commands: never creates the file
|
|
67
|
+
* @returns {object} Outbox handle
|
|
68
|
+
*/
|
|
69
|
+
export function openOutbox(accountDir, { now = Date.now, readonly = false } = {}) {
|
|
70
|
+
if (!readonly) mkdirSync(accountDir, { recursive: true });
|
|
71
|
+
const db = new Database(outboxPath(accountDir), readonly ? { readonly: true, fileMustExist: true } : {});
|
|
72
|
+
db.pragma("busy_timeout = 5000");
|
|
73
|
+
if (!readonly) {
|
|
74
|
+
db.pragma("journal_mode = WAL");
|
|
75
|
+
db.exec(`
|
|
76
|
+
CREATE TABLE IF NOT EXISTS outbox (
|
|
77
|
+
seq INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
78
|
+
event_key TEXT UNIQUE,
|
|
79
|
+
body TEXT NOT NULL,
|
|
80
|
+
created_at INTEGER NOT NULL,
|
|
81
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
82
|
+
status TEXT NOT NULL DEFAULT 'pending',
|
|
83
|
+
last_error TEXT,
|
|
84
|
+
last_attempt_at INTEGER,
|
|
85
|
+
dead_at INTEGER
|
|
86
|
+
);
|
|
87
|
+
CREATE INDEX IF NOT EXISTS outbox_status_seq ON outbox (status, seq);
|
|
88
|
+
CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT);
|
|
89
|
+
`);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
const stmt = (sql) => db.prepare(sql);
|
|
93
|
+
const insert = readonly
|
|
94
|
+
? null
|
|
95
|
+
: stmt("INSERT OR IGNORE INTO outbox (event_key, body, created_at) VALUES (?, ?, ?)");
|
|
96
|
+
const getMeta = stmt("SELECT value FROM meta WHERE key = ?");
|
|
97
|
+
const setMeta = readonly
|
|
98
|
+
? null
|
|
99
|
+
: stmt("INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value");
|
|
100
|
+
const meta = (key) => getMeta.get(key)?.value ?? null;
|
|
101
|
+
const inList = (seqs) => seqs.map(() => "?").join(",");
|
|
102
|
+
|
|
103
|
+
return {
|
|
104
|
+
path: outboxPath(accountDir),
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Persist one event before anything tries to send it.
|
|
108
|
+
* @param {string|null} eventKey - Dedupes while the event is still queued; null never dedupes
|
|
109
|
+
* @param {string} body - The exact JSON text a receiver will be sent
|
|
110
|
+
* @returns {{ queued: boolean, seq: number|null }} `queued: false` when the same key is already waiting
|
|
111
|
+
*/
|
|
112
|
+
enqueue(eventKey, body) {
|
|
113
|
+
const r = insert.run(eventKey ?? null, body, now());
|
|
114
|
+
return r.changes ? { queued: true, seq: Number(r.lastInsertRowid) } : { queued: false, seq: null };
|
|
115
|
+
},
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* @param {number} limit
|
|
119
|
+
* @returns {Array<{seq:number,body:string,created_at:number,attempts:number}>} Oldest pending rows
|
|
120
|
+
*/
|
|
121
|
+
head(limit) {
|
|
122
|
+
return stmt(
|
|
123
|
+
"SELECT seq, body, created_at, attempts FROM outbox WHERE status = 'pending' ORDER BY seq LIMIT ?",
|
|
124
|
+
).all(Math.max(1, limit));
|
|
125
|
+
},
|
|
126
|
+
|
|
127
|
+
/** @param {number[]} seqs */
|
|
128
|
+
markDelivered(seqs) {
|
|
129
|
+
if (!seqs.length) return;
|
|
130
|
+
db.transaction(() => {
|
|
131
|
+
db.prepare(`DELETE FROM outbox WHERE seq IN (${inList(seqs)})`).run(...seqs);
|
|
132
|
+
const total = Number(meta("delivered_total") || 0) + seqs.length;
|
|
133
|
+
setMeta.run("delivered_total", String(total));
|
|
134
|
+
setMeta.run("last_success_at", String(now()));
|
|
135
|
+
setMeta.run("last_error", "");
|
|
136
|
+
})();
|
|
137
|
+
},
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* @param {number[]} seqs
|
|
141
|
+
* @param {{ error: string, status?: number|null }} failure
|
|
142
|
+
*/
|
|
143
|
+
markFailed(seqs, { error, status = null }) {
|
|
144
|
+
if (!seqs.length) return;
|
|
145
|
+
db.transaction(() => {
|
|
146
|
+
db.prepare(
|
|
147
|
+
`UPDATE outbox SET attempts = attempts + 1, last_error = ?, last_attempt_at = ? WHERE status = 'pending' AND seq IN (${inList(seqs)})`,
|
|
148
|
+
).run(error, now(), ...seqs);
|
|
149
|
+
setMeta.run("last_error", error);
|
|
150
|
+
setMeta.run("last_error_at", String(now()));
|
|
151
|
+
setMeta.run("last_status", status === null ? "" : String(status));
|
|
152
|
+
})();
|
|
153
|
+
},
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Park rows that will not be retried on their own. They stay until `retryDead` or `purgeDead`.
|
|
157
|
+
* @param {number[]} seqs
|
|
158
|
+
* @param {string} reason
|
|
159
|
+
*/
|
|
160
|
+
markDead(seqs, reason) {
|
|
161
|
+
if (!seqs.length) return;
|
|
162
|
+
db.prepare(
|
|
163
|
+
`UPDATE outbox SET status = 'dead', last_error = ?, dead_at = ? WHERE seq IN (${inList(seqs)})`,
|
|
164
|
+
).run(reason, now(), ...seqs);
|
|
165
|
+
},
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Dead-letter every pending row created before `cutoff`.
|
|
169
|
+
* @param {number} cutoff - Epoch ms
|
|
170
|
+
* @returns {number} Rows moved
|
|
171
|
+
*/
|
|
172
|
+
expireOlderThan(cutoff) {
|
|
173
|
+
const r = db
|
|
174
|
+
.prepare(
|
|
175
|
+
"UPDATE outbox SET status = 'dead', last_error = 'expired: not delivered within the retention window', dead_at = ? WHERE status = 'pending' AND created_at < ?",
|
|
176
|
+
)
|
|
177
|
+
.run(now(), cutoff);
|
|
178
|
+
return r.changes;
|
|
179
|
+
},
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* Put dead rows back in the queue with a fresh clock, in their original order.
|
|
183
|
+
* @param {{ seq?: number }} [filter]
|
|
184
|
+
* @returns {number} Rows requeued
|
|
185
|
+
*/
|
|
186
|
+
retryDead({ seq } = {}) {
|
|
187
|
+
const r = seq
|
|
188
|
+
? db
|
|
189
|
+
.prepare(
|
|
190
|
+
"UPDATE outbox SET status = 'pending', attempts = 0, created_at = ?, dead_at = NULL WHERE status = 'dead' AND seq = ?",
|
|
191
|
+
)
|
|
192
|
+
.run(now(), seq)
|
|
193
|
+
: db
|
|
194
|
+
.prepare(
|
|
195
|
+
"UPDATE outbox SET status = 'pending', attempts = 0, created_at = ?, dead_at = NULL WHERE status = 'dead'",
|
|
196
|
+
)
|
|
197
|
+
.run(now());
|
|
198
|
+
return r.changes;
|
|
199
|
+
},
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* Delete dead rows for good.
|
|
203
|
+
* @param {{ seq?: number }} [filter]
|
|
204
|
+
* @returns {number} Rows removed
|
|
205
|
+
*/
|
|
206
|
+
purgeDead({ seq } = {}) {
|
|
207
|
+
const r = seq
|
|
208
|
+
? db.prepare("DELETE FROM outbox WHERE status = 'dead' AND seq = ?").run(seq)
|
|
209
|
+
: db.prepare("DELETE FROM outbox WHERE status = 'dead'").run();
|
|
210
|
+
return r.changes;
|
|
211
|
+
},
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* @param {{ status?: 'pending'|'dead', limit?: number }} [filter]
|
|
215
|
+
* @returns {Array<object>} Rows, oldest first; `body` is the JSON text
|
|
216
|
+
*/
|
|
217
|
+
list({ status, limit = 50 } = {}) {
|
|
218
|
+
const where = status ? "WHERE status = ?" : "";
|
|
219
|
+
const args = status ? [status, limit] : [limit];
|
|
220
|
+
return stmt(
|
|
221
|
+
`SELECT seq, status, attempts, created_at, last_attempt_at, last_error, dead_at, body FROM outbox ${where} ORDER BY seq LIMIT ?`,
|
|
222
|
+
).all(...args);
|
|
223
|
+
},
|
|
224
|
+
|
|
225
|
+
/**
|
|
226
|
+
* @returns {{ pending:number, dead:number, oldestPendingAt:number|null, deliveredTotal:number, lastSuccessAt:number|null, lastError:string|null, lastErrorAt:number|null, lastStatus:number|null }}
|
|
227
|
+
*/
|
|
228
|
+
stats() {
|
|
229
|
+
const c = stmt("SELECT status, COUNT(*) AS n, MIN(created_at) AS oldest FROM outbox GROUP BY status").all();
|
|
230
|
+
const by = Object.fromEntries(c.map((r) => [r.status, r]));
|
|
231
|
+
const num = (k) => (meta(k) ? Number(meta(k)) : null);
|
|
232
|
+
return {
|
|
233
|
+
pending: by.pending?.n ?? 0,
|
|
234
|
+
dead: by.dead?.n ?? 0,
|
|
235
|
+
oldestPendingAt: by.pending?.oldest ?? null,
|
|
236
|
+
deliveredTotal: Number(meta("delivered_total") || 0),
|
|
237
|
+
lastSuccessAt: num("last_success_at"),
|
|
238
|
+
lastError: meta("last_error") || null,
|
|
239
|
+
lastErrorAt: num("last_error_at"),
|
|
240
|
+
lastStatus: num("last_status"),
|
|
241
|
+
};
|
|
242
|
+
},
|
|
243
|
+
|
|
244
|
+
close() {
|
|
245
|
+
try {
|
|
246
|
+
db.close();
|
|
247
|
+
} catch {
|
|
248
|
+
/* already closed */
|
|
249
|
+
}
|
|
250
|
+
},
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/**
|
|
255
|
+
* Backoff for the Nth consecutive failed attempt: 1 s, 2 s, 4 s ... capped at 5 minutes, +/-20%.
|
|
256
|
+
*
|
|
257
|
+
* @param {number} attempts - Attempts so far (>= 1)
|
|
258
|
+
* @param {() => number} [random]
|
|
259
|
+
* @returns {number} Milliseconds
|
|
260
|
+
*/
|
|
261
|
+
export function backoffMs(attempts, random = Math.random) {
|
|
262
|
+
const base = Math.min(BACKOFF_CAP_MS, BACKOFF_BASE_MS * 2 ** Math.max(0, attempts - 1));
|
|
263
|
+
return Math.round(base * (0.8 + random() * 0.4));
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* Keep a URL's origin and path for status output; its query string can carry a token.
|
|
268
|
+
* @param {string} url
|
|
269
|
+
* @returns {string}
|
|
270
|
+
*/
|
|
271
|
+
export function describeWebhookUrl(url) {
|
|
272
|
+
try {
|
|
273
|
+
const u = new URL(url);
|
|
274
|
+
return `${u.origin}${u.pathname}`;
|
|
275
|
+
} catch {
|
|
276
|
+
return "(invalid URL)";
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* Drain an outbox to one receiver.
|
|
282
|
+
*
|
|
283
|
+
* @param {object} options
|
|
284
|
+
* @param {ReturnType<typeof openOutbox>} options.outbox
|
|
285
|
+
* @param {string} options.url
|
|
286
|
+
* @param {string|null} [options.secret]
|
|
287
|
+
* @param {string} [options.accountId]
|
|
288
|
+
* @param {number} [options.batchSize=1] - Events per POST. 1 sends each event exactly as before; more sends `{event:"batch", events:[...]}`
|
|
289
|
+
* @param {number} [options.maxAgeMs] - A queued event older than this is dead-lettered
|
|
290
|
+
* @param {number} [options.timeoutMs=10000]
|
|
291
|
+
* @param {number} [options.batchWindowMs=250] - With batching on, how long a new event waits for company
|
|
292
|
+
* @param {number} [options.pollMs=30000] - Look at the queue this often even when nothing was enqueued: rows another process put back (`mcp webhook retry`) have no one to wake the dispatcher
|
|
293
|
+
* @param {(line: string) => void} [options.log]
|
|
294
|
+
* @param {Function} [options.fetchFn]
|
|
295
|
+
* @param {() => number} [options.now]
|
|
296
|
+
* @param {() => number} [options.random]
|
|
297
|
+
* @param {Function} [options.setTimer]
|
|
298
|
+
* @param {Function} [options.clearTimer]
|
|
299
|
+
* @returns {{ start: () => void, notify: () => void, stop: (o?: {graceMs?: number}) => Promise<void>, status: () => object, drain: () => Promise<void> }}
|
|
300
|
+
*/
|
|
301
|
+
export function createWebhookDispatcher({
|
|
302
|
+
outbox,
|
|
303
|
+
url,
|
|
304
|
+
secret = null,
|
|
305
|
+
accountId = null,
|
|
306
|
+
batchSize = 1,
|
|
307
|
+
maxAgeMs = DAY_MS,
|
|
308
|
+
timeoutMs = 10_000,
|
|
309
|
+
batchWindowMs = 250,
|
|
310
|
+
pollMs = 30_000,
|
|
311
|
+
log = () => {},
|
|
312
|
+
fetchFn = fetch,
|
|
313
|
+
now = Date.now,
|
|
314
|
+
random = Math.random,
|
|
315
|
+
setTimer = setTimeout,
|
|
316
|
+
clearTimer = clearTimeout,
|
|
317
|
+
}) {
|
|
318
|
+
const size = Math.max(1, Math.floor(batchSize) || 1);
|
|
319
|
+
let timer = null;
|
|
320
|
+
let poller = null;
|
|
321
|
+
let timerAt = 0;
|
|
322
|
+
let running = null;
|
|
323
|
+
let stopped = false;
|
|
324
|
+
let aborter = null;
|
|
325
|
+
let backoffUntil = 0;
|
|
326
|
+
let failures = 0;
|
|
327
|
+
let lastFailureLogAt = 0;
|
|
328
|
+
let sentSinceRecovery = 0;
|
|
329
|
+
|
|
330
|
+
function schedule(delay) {
|
|
331
|
+
if (stopped) return;
|
|
332
|
+
const at = now() + delay;
|
|
333
|
+
if (timer && timerAt <= at) return;
|
|
334
|
+
if (timer) clearTimer(timer);
|
|
335
|
+
timerAt = at;
|
|
336
|
+
timer = setTimer(() => {
|
|
337
|
+
timer = null;
|
|
338
|
+
// A pump already draining picks up whatever was enqueued meanwhile on its next pass.
|
|
339
|
+
if (running) return;
|
|
340
|
+
running = pump().finally(() => {
|
|
341
|
+
running = null;
|
|
342
|
+
});
|
|
343
|
+
}, delay);
|
|
344
|
+
timer.unref?.();
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
/** One POST. Never throws. */
|
|
348
|
+
async function post(rows) {
|
|
349
|
+
const events = rows.map((r) => JSON.parse(r.body));
|
|
350
|
+
const body =
|
|
351
|
+
rows.length === 1
|
|
352
|
+
? rows[0].body
|
|
353
|
+
: JSON.stringify({ event: "batch", accountId, count: rows.length, events });
|
|
354
|
+
const headers = {
|
|
355
|
+
...webhookHeaders(body, secret),
|
|
356
|
+
"X-Zalo-Delivery": `${rows[0].seq}-${rows[rows.length - 1].seq}`,
|
|
357
|
+
"X-Zalo-Attempt": String(Math.max(...rows.map((r) => r.attempts)) + 1),
|
|
358
|
+
};
|
|
359
|
+
aborter = new AbortController();
|
|
360
|
+
const timeout = setTimer(() => aborter?.abort(new Error(`timed out after ${timeoutMs} ms`)), timeoutMs);
|
|
361
|
+
timeout.unref?.();
|
|
362
|
+
try {
|
|
363
|
+
const res = await fetchFn(url, { method: "POST", headers, body, signal: aborter.signal });
|
|
364
|
+
// The receiver's reply body is not read: draining it would let a slow body hold the queue.
|
|
365
|
+
res.body?.cancel?.().catch?.(() => {});
|
|
366
|
+
return res.status >= 200 && res.status < 300
|
|
367
|
+
? { ok: true, status: res.status }
|
|
368
|
+
: { ok: false, status: res.status, error: `HTTP ${res.status}` };
|
|
369
|
+
} catch (e) {
|
|
370
|
+
const cause = aborter?.signal?.reason;
|
|
371
|
+
return {
|
|
372
|
+
ok: false,
|
|
373
|
+
status: null,
|
|
374
|
+
error: (cause?.message || e?.cause?.code || e?.message || String(e)).slice(0, 300),
|
|
375
|
+
};
|
|
376
|
+
} finally {
|
|
377
|
+
clearTimer(timeout);
|
|
378
|
+
aborter = null;
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
function logFailure(error, queued, delay, attempt) {
|
|
383
|
+
const t = now();
|
|
384
|
+
if (failures > 1 && t - lastFailureLogAt < 60_000) return;
|
|
385
|
+
lastFailureLogAt = t;
|
|
386
|
+
log(
|
|
387
|
+
`Webhook delivery failed (${error}); ${queued} queued, retrying in ${Math.round(delay / 1000)}s (attempt ${attempt})`,
|
|
388
|
+
);
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
/** Send one event alone after a batch it was part of was refused as malformed. */
|
|
392
|
+
async function sendEach(rows) {
|
|
393
|
+
for (const row of rows) {
|
|
394
|
+
const r = await post([row]);
|
|
395
|
+
if (r.ok) {
|
|
396
|
+
outbox.markDelivered([row.seq]);
|
|
397
|
+
sentSinceRecovery++;
|
|
398
|
+
} else if (r.status !== null && PAYLOAD_REJECTED.has(r.status)) {
|
|
399
|
+
outbox.markDead([row.seq], `rejected by the receiver: ${r.error}`);
|
|
400
|
+
log(`Webhook event ${row.seq} dead-lettered (${r.error}); see: zalo-agent mcp webhook list --dead`);
|
|
401
|
+
} else {
|
|
402
|
+
return r;
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
return { ok: true };
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
async function pump() {
|
|
409
|
+
if (stopped) return;
|
|
410
|
+
try {
|
|
411
|
+
while (!stopped) {
|
|
412
|
+
const expired = outbox.expireOlderThan(now() - maxAgeMs);
|
|
413
|
+
if (expired)
|
|
414
|
+
log(
|
|
415
|
+
`${expired} undelivered webhook event(s) older than ${Math.round(maxAgeMs / 3600000)}h dead-lettered`,
|
|
416
|
+
);
|
|
417
|
+
const rows = outbox.head(size);
|
|
418
|
+
if (!rows.length) return;
|
|
419
|
+
if (now() < backoffUntil) {
|
|
420
|
+
schedule(backoffUntil - now());
|
|
421
|
+
return;
|
|
422
|
+
}
|
|
423
|
+
let r = await post(rows);
|
|
424
|
+
if (!r.ok && rows.length > 1 && r.status !== null && PAYLOAD_REJECTED.has(r.status))
|
|
425
|
+
r = await sendEach(rows);
|
|
426
|
+
else if (r.ok) {
|
|
427
|
+
outbox.markDelivered(rows.map((x) => x.seq));
|
|
428
|
+
sentSinceRecovery += rows.length;
|
|
429
|
+
} else if (r.status !== null && PAYLOAD_REJECTED.has(r.status)) {
|
|
430
|
+
outbox.markDead(
|
|
431
|
+
rows.map((x) => x.seq),
|
|
432
|
+
`rejected by the receiver: ${r.error}`,
|
|
433
|
+
);
|
|
434
|
+
log(
|
|
435
|
+
`Webhook event ${rows[0].seq} dead-lettered (${r.error}); see: zalo-agent mcp webhook list --dead`,
|
|
436
|
+
);
|
|
437
|
+
r = { ok: true };
|
|
438
|
+
}
|
|
439
|
+
if (stopped) return;
|
|
440
|
+
if (r.ok) {
|
|
441
|
+
if (failures > 0) {
|
|
442
|
+
const left = outbox.stats().pending;
|
|
443
|
+
log(`Webhook receiver is back: ${sentSinceRecovery} delivered, ${left} still queued`);
|
|
444
|
+
}
|
|
445
|
+
failures = 0;
|
|
446
|
+
backoffUntil = 0;
|
|
447
|
+
if (outbox.stats().pending === 0) sentSinceRecovery = 0;
|
|
448
|
+
continue;
|
|
449
|
+
}
|
|
450
|
+
failures++;
|
|
451
|
+
outbox.markFailed(
|
|
452
|
+
rows.map((x) => x.seq),
|
|
453
|
+
{ error: r.error, status: r.status },
|
|
454
|
+
);
|
|
455
|
+
const attempt = Math.max(failures, ...rows.map((x) => x.attempts + 1));
|
|
456
|
+
const delay = backoffMs(attempt, random);
|
|
457
|
+
backoffUntil = now() + delay;
|
|
458
|
+
logFailure(r.error, outbox.stats().pending, delay, attempt);
|
|
459
|
+
schedule(delay);
|
|
460
|
+
return;
|
|
461
|
+
}
|
|
462
|
+
} catch (e) {
|
|
463
|
+
// A broken outbox must not take the daemon with it; try again shortly.
|
|
464
|
+
log(`Webhook outbox error: ${e.message}`);
|
|
465
|
+
schedule(30_000);
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
return {
|
|
470
|
+
/** Resume whatever a previous run left queued. */
|
|
471
|
+
start() {
|
|
472
|
+
const s = outbox.stats();
|
|
473
|
+
if (s.pending || s.dead) {
|
|
474
|
+
log(`Webhook outbox: ${s.pending} queued, ${s.dead} dead-lettered from before this start`);
|
|
475
|
+
}
|
|
476
|
+
schedule(0);
|
|
477
|
+
if (!poller && pollMs > 0) {
|
|
478
|
+
poller = setInterval(() => schedule(0), pollMs);
|
|
479
|
+
poller.unref?.();
|
|
480
|
+
}
|
|
481
|
+
},
|
|
482
|
+
/** An event was just enqueued. */
|
|
483
|
+
notify() {
|
|
484
|
+
schedule(size > 1 ? batchWindowMs : 0);
|
|
485
|
+
},
|
|
486
|
+
/** Resolve once the queue is empty or delivery is waiting out a backoff (tests, shutdown). */
|
|
487
|
+
async drain() {
|
|
488
|
+
if (timer) {
|
|
489
|
+
clearTimer(timer);
|
|
490
|
+
timer = null;
|
|
491
|
+
}
|
|
492
|
+
if (running) await running;
|
|
493
|
+
running = pump().finally(() => {
|
|
494
|
+
running = null;
|
|
495
|
+
});
|
|
496
|
+
await running;
|
|
497
|
+
},
|
|
498
|
+
/**
|
|
499
|
+
* @param {{ graceMs?: number }} [o] - How long an in-flight POST may finish before it is cut
|
|
500
|
+
*/
|
|
501
|
+
async stop({ graceMs = 2000 } = {}) {
|
|
502
|
+
stopped = true;
|
|
503
|
+
if (poller) clearInterval(poller);
|
|
504
|
+
poller = null;
|
|
505
|
+
if (timer) clearTimer(timer);
|
|
506
|
+
timer = null;
|
|
507
|
+
if (!running) return;
|
|
508
|
+
const cut = setTimer(() => aborter?.abort(new Error("shutdown")), graceMs);
|
|
509
|
+
cut.unref?.();
|
|
510
|
+
await running;
|
|
511
|
+
clearTimer(cut);
|
|
512
|
+
},
|
|
513
|
+
status() {
|
|
514
|
+
const s = outbox.stats();
|
|
515
|
+
return {
|
|
516
|
+
url: describeWebhookUrl(url),
|
|
517
|
+
signed: Boolean(secret),
|
|
518
|
+
batchSize: size,
|
|
519
|
+
...s,
|
|
520
|
+
consecutiveFailures: failures,
|
|
521
|
+
nextAttemptAt: backoffUntil > now() ? backoffUntil : null,
|
|
522
|
+
oldestPendingAgeMs: s.oldestPendingAt ? Math.max(0, now() - s.oldestPendingAt) : 0,
|
|
523
|
+
};
|
|
524
|
+
},
|
|
525
|
+
};
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
/**
|
|
529
|
+
* What `mcp start` and `listen` call: open the account's outbox, resume whatever an earlier run left
|
|
530
|
+
* queued, and hand back the one function an event handler needs.
|
|
531
|
+
*
|
|
532
|
+
* @param {object} options
|
|
533
|
+
* @param {string} options.accountDir
|
|
534
|
+
* @param {string} options.accountId
|
|
535
|
+
* @param {string} options.url
|
|
536
|
+
* @param {string|null} [options.secret]
|
|
537
|
+
* @param {number} [options.batchSize]
|
|
538
|
+
* @param {number} [options.maxAgeMs]
|
|
539
|
+
* @param {number} [options.pollMs]
|
|
540
|
+
* @param {(line: string) => void} options.log
|
|
541
|
+
* @returns {{ enqueue: (eventKey: string|null, payload: object) => void, status: () => object, stop: () => void }}
|
|
542
|
+
*/
|
|
543
|
+
export function startWebhookDelivery({
|
|
544
|
+
accountDir,
|
|
545
|
+
accountId,
|
|
546
|
+
url,
|
|
547
|
+
secret = null,
|
|
548
|
+
batchSize = 1,
|
|
549
|
+
maxAgeMs,
|
|
550
|
+
pollMs,
|
|
551
|
+
log,
|
|
552
|
+
}) {
|
|
553
|
+
const outbox = openOutbox(accountDir);
|
|
554
|
+
const dispatcher = createWebhookDispatcher({ outbox, url, secret, accountId, batchSize, maxAgeMs, pollMs, log });
|
|
555
|
+
dispatcher.start();
|
|
556
|
+
return {
|
|
557
|
+
/**
|
|
558
|
+
* Persist an event and wake the dispatcher. Never throws into the listener loop.
|
|
559
|
+
* @param {string|null} eventKey - Dedupes while queued; null for events with no stable id
|
|
560
|
+
* @param {object} payload
|
|
561
|
+
*/
|
|
562
|
+
enqueue(eventKey, payload) {
|
|
563
|
+
try {
|
|
564
|
+
if (outbox.enqueue(eventKey, JSON.stringify(payload)).queued) dispatcher.notify();
|
|
565
|
+
} catch (e) {
|
|
566
|
+
log(`Webhook event could not be queued: ${e.message}`);
|
|
567
|
+
}
|
|
568
|
+
},
|
|
569
|
+
status: () => dispatcher.status(),
|
|
570
|
+
/** Stop sending and release the file. Queued events stay for the next start. */
|
|
571
|
+
stop() {
|
|
572
|
+
dispatcher.stop({ graceMs: 0 }).finally(() => outbox.close());
|
|
573
|
+
},
|
|
574
|
+
};
|
|
575
|
+
}
|
package/src/index.js
CHANGED
|
@@ -29,6 +29,7 @@ import { registerBankCommands } from "./commands/bank.js";
|
|
|
29
29
|
import { registerListenCommand } from "./commands/listen.js";
|
|
30
30
|
import { registerOACommands } from "./commands/oa.js";
|
|
31
31
|
import { registerMCPCommands } from "./commands/mcp.js";
|
|
32
|
+
import { registerFleetCommands } from "./commands/fleet.js";
|
|
32
33
|
import { registerSyncCommands } from "./commands/sync.js";
|
|
33
34
|
import { autoLogin } from "./core/zalo-client.js";
|
|
34
35
|
import { getActive, setPinnedAccount } from "./core/accounts.js";
|
|
@@ -96,7 +97,7 @@ program
|
|
|
96
97
|
const requestedAccount = actionCommand.opts()?.account ?? globalOpts.account ?? envAccount;
|
|
97
98
|
if (
|
|
98
99
|
requestedAccount !== undefined &&
|
|
99
|
-
!["mcp", "listen", "oa", "update", "help", "version", "login"].includes(cmdName)
|
|
100
|
+
!["mcp", "listen", "oa", "update", "help", "version", "login", "fleet"].includes(cmdName)
|
|
100
101
|
) {
|
|
101
102
|
try {
|
|
102
103
|
const pinned = selectDaemonAccount(requestedAccount);
|
|
@@ -110,7 +111,7 @@ program
|
|
|
110
111
|
// commands, and for a logout that will not call Zalo at all).
|
|
111
112
|
// Daemons authenticate their pinned identity after acquiring its lock.
|
|
112
113
|
const skipAutoLogin =
|
|
113
|
-
["login", "account", "help", "version", "update", "oa", "mcp", "listen"].includes(cmdName) ||
|
|
114
|
+
["login", "account", "help", "version", "update", "oa", "mcp", "listen", "fleet"].includes(cmdName) ||
|
|
114
115
|
(cmdName === "logout" && !logoutNeedsSession(actionCommand.opts(), getActive()));
|
|
115
116
|
if (!skipAutoLogin) {
|
|
116
117
|
await autoLogin(program.opts().json);
|
|
@@ -152,6 +153,15 @@ registerBankCommands(program);
|
|
|
152
153
|
registerListenCommand(program);
|
|
153
154
|
registerOACommands(program);
|
|
154
155
|
registerMCPCommands(program);
|
|
156
|
+
registerFleetCommands(program);
|
|
155
157
|
registerSyncCommands(program);
|
|
156
158
|
|
|
157
|
-
|
|
159
|
+
// parseAsync, so a registry that exists but cannot be read (src/core/accounts.js) reaches the user as a
|
|
160
|
+
// message, not as a stack trace from inside a command. Anything else is rethrown exactly as before.
|
|
161
|
+
program.parseAsync().catch((e) => {
|
|
162
|
+
if (e?.name === "RegistryError") {
|
|
163
|
+
error(e.message);
|
|
164
|
+
process.exit(1);
|
|
165
|
+
}
|
|
166
|
+
throw e;
|
|
167
|
+
});
|
package/src/mcp/mcp-config.js
CHANGED
|
@@ -2,9 +2,10 @@
|
|
|
2
2
|
* Load/save MCP-specific config from ~/.zalo-agent-cli/mcp-config.json.
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
|
-
import { readFileSync,
|
|
5
|
+
import { readFileSync, mkdirSync } from "fs";
|
|
6
6
|
import { join } from "path";
|
|
7
7
|
import { CONFIG_DIR } from "../core/credentials.js";
|
|
8
|
+
import { writeFileAtomic } from "../utils/atomic-file.js";
|
|
8
9
|
|
|
9
10
|
function mcpConfigFile() {
|
|
10
11
|
return join(CONFIG_DIR, "mcp-config.json");
|
|
@@ -44,6 +45,8 @@ export function getDefaultConfig() {
|
|
|
44
45
|
clients: [],
|
|
45
46
|
},
|
|
46
47
|
webhookSecret: null,
|
|
48
|
+
webhookBatchSize: 1, // events per webhook POST; 1 = each event alone, exactly as before the outbox
|
|
49
|
+
webhookMaxAge: "24h", // an undelivered event older than this is dead-lettered, never silently dropped
|
|
47
50
|
groupConfirm: true, // zalo_group asks for confirm: true before leave / remove_member
|
|
48
51
|
httpPort: null,
|
|
49
52
|
host: null,
|
|
@@ -205,7 +208,44 @@ export function resolveAccountMCPConfig(baseConfig, { accountId, accountDir, exp
|
|
|
205
208
|
*/
|
|
206
209
|
export function saveMCPConfig(config) {
|
|
207
210
|
mkdirSync(CONFIG_DIR, { recursive: true });
|
|
208
|
-
|
|
211
|
+
// Atomic, and 0600: the file can hold a webhook or OAuth secret.
|
|
212
|
+
writeFileAtomic(mcpConfigFile(), JSON.stringify(config, null, 2));
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/**
|
|
216
|
+
* The webhook delivery settings of one account: `ZALO_WEBHOOK_BATCH_SIZE[_<ownId>]` and
|
|
217
|
+
* `ZALO_WEBHOOK_MAX_AGE[_<ownId>]`, else `webhookBatchSize` / `webhookMaxAge` from the MCP config
|
|
218
|
+
* (`listen` has no config file and reads only the environment). A value that cannot be used falls
|
|
219
|
+
* back to the default and is reported, so a typo is not mistaken for a working setting.
|
|
220
|
+
*
|
|
221
|
+
* @param {string} accountId
|
|
222
|
+
* @param {object} [config] - Resolved MCP config, or none for `listen`
|
|
223
|
+
* @param {NodeJS.ProcessEnv} [env]
|
|
224
|
+
* @returns {{ batchSize: number, maxAgeMs: number, warnings: string[] }}
|
|
225
|
+
*/
|
|
226
|
+
export function resolveWebhookDelivery(accountId, config = {}, env = process.env) {
|
|
227
|
+
const warnings = [];
|
|
228
|
+
const defaults = getDefaultConfig();
|
|
229
|
+
|
|
230
|
+
const rawBatch = envForAccount("ZALO_WEBHOOK_BATCH_SIZE", accountId, env) ?? config?.webhookBatchSize;
|
|
231
|
+
let batchSize = defaults.webhookBatchSize;
|
|
232
|
+
if (rawBatch !== undefined && rawBatch !== null && rawBatch !== "") {
|
|
233
|
+
const n = Number(rawBatch);
|
|
234
|
+
if (Number.isInteger(n) && n >= 1 && n <= 100) batchSize = n;
|
|
235
|
+
else warnings.push(`webhook batch size "${rawBatch}" is not a whole number from 1 to 100; using ${batchSize}`);
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
const rawAge = envForAccount("ZALO_WEBHOOK_MAX_AGE", accountId, env) ?? config?.webhookMaxAge;
|
|
239
|
+
let maxAgeMs = parseDuration(defaults.webhookMaxAge);
|
|
240
|
+
if (rawAge !== undefined && rawAge !== null && rawAge !== "") {
|
|
241
|
+
const ms = parseDuration(rawAge);
|
|
242
|
+
if (ms >= 60_000) maxAgeMs = ms;
|
|
243
|
+
else
|
|
244
|
+
warnings.push(
|
|
245
|
+
`webhook max age "${rawAge}" is not a duration of at least 1m (e.g. 6h, 24h); using ${defaults.webhookMaxAge}`,
|
|
246
|
+
);
|
|
247
|
+
}
|
|
248
|
+
return { batchSize, maxAgeMs, warnings };
|
|
209
249
|
}
|
|
210
250
|
|
|
211
251
|
/**
|