@ardennguyen/zalo-agent-cli 2.5.8 → 3.0.0

package/src/core/voice-text.js

@@ -0,0 +1,171 @@
+/**
+ * Voice-message transcripts, kept as they arrive.
+ *
+ * `write run voice-to-text` asks Zalo to transcribe a voice message and gets back only a request id. The text comes
+ * later, as a `601` control on the socket: `act_type "voice2text"`, `act "convert_finish"`, its data `{req_id, err,
+ * text, url, has_more}` (seen live, 2026-10-27; the key names are Zalo's own). A command that is not the
+ * daemon has no socket to hear it on, so the daemon (`listen`, `mcp start`) writes each answer into a small file in the
+ * account folder, and `read run voice-transcript` reads it from there.
+ *
+ * A long message can arrive in pieces: `has_more` says another piece follows, and the pieces are joined in the order
+ * they come. The file holds the text of voice messages, like the message cache does, at 0600 and in the account
+ * folder, sealed under the account's data key (src/core/keystore/); it keeps the last {@link MAX_ITEMS} requests and
+ * nothing else (not the voice file's address).
+ */
+import { dirname, join } from "node:path";
+import { openFile, scopeOfDir, sealFile } from "./keystore/index.js";
+import { readControls } from "../zalo/queues/control-queue.js";
+
+/** The control push that carries it. */
+export const CMD_PUSH_CONTROLS = 601;
+/** The control's own names. */
+export const VOICE_ACT_TYPE = "voice2text";
+export const VOICE_ACT = "convert_finish";
+/** How many requests the file keeps: older ones are dropped first. */
+export const MAX_ITEMS = 100;
+
+/**
+ * Where an account's transcripts are kept.
+ *
+ * @param {string} accountDir
+ * @returns {string}
+ */
+export function voiceTextFile(accountDir) {
+    return join(accountDir, "voice-text.json");
+}
+
+const SPEC = { kind: "voice-text", use: "data" };
+
+/**
+ * The saved transcripts: `{reqId: {at, err, text, done}}`, empty when there is no usable file.
+ *
+ * @param {string} file
+ * @returns {Record<string, {at: number, err: number, text: string, done: boolean}>}
+ * @throws {Error} a KeystoreError when the file is there but its key is not (locked, no provider): "no transcript"
+ *   would be the wrong answer
+ */
+export function loadTranscripts(file) {
+    try {
+        const bytes = openFile(file, scopeOfDir(dirname(file)), SPEC);
+        const items = bytes === null ? null : JSON.parse(bytes.toString("utf8"))?.items;
+        return items && typeof items === "object" && !Array.isArray(items) ? { ...items } : {};
+    } catch (e) {
+        if (e?.name === "KeystoreError") throw e;
+        return {};
+    }
+}
+
+/**
+ * One transcript, or all of them (newest first) when no request id is given.
+ *
+ * @param {string} file
+ * @param {string} [reqId]
+ * @returns {{reqId: string, at: string, err: number, text: string, done: boolean}|null|Array<object>}
+ */
+export function readTranscript(file, reqId) {
+    const items = loadTranscripts(file);
+    const shape = (id, v) => ({ reqId: id, at: new Date(v.at).toISOString(), err: v.err, text: v.text, done: v.done });
+    if (reqId) return items[reqId] ? shape(reqId, items[reqId]) : null;
+    return Object.entries(items)
+        .sort((a, b) => b[1].at - a[1].at)
+        .map(([id, v]) => shape(id, v));
+}
+
+/**
+ * A recorder for one account.
+ *
+ * @param {object} args
+ * @param {string} args.file
+ * @param {() => number} [args.now]
+ * @param {(line: string) => void} [args.log] - told once per finished transcript (never the text)
+ * @returns {{observe: (cmd: number, data: object|null|undefined) => number}} `observe` returns how many answers of this
+ *   frame it recorded
+ */
+export function createVoiceText({ file, now = Date.now, log = () => {} }) {
+    let items = {};
+    // A file this process cannot open (its key is locked) is never overwritten: the recorder then keeps its answers in
+    // memory only.
+    let writable = true;
+    try {
+        items = loadTranscripts(file);
+    } catch (e) {
+        writable = false;
+        log(`voice transcripts are not kept on disk: ${e.message}`);
+    }
+
+    function save() {
+        if (!writable) return;
+        try {
+            sealFile(file, scopeOfDir(dirname(file)), SPEC, JSON.stringify({ items }));
+        } catch {
+            /* the file is a convenience: failing to write it must not disturb the socket */
+        }
+    }
+
+    function record(d) {
+        const reqId = String(d.req_id ?? "");
+        if (!reqId) return false;
+        const text = typeof d.text === "string" ? d.text : "";
+        const done = !(d.has_more === true || Number(d.has_more) === 1);
+        const err = Number(d.err) || 0;
+        const prev = items[reqId];
+        items[reqId] = {
+            at: now(),
+            err,
+            // The pieces of a long message are joined in the order they arrive.
+            text: prev && !prev.done ? prev.text + text : text,
+            done,
+        };
+        const keep = Object.entries(items)
+            .sort((a, b) => b[1].at - a[1].at)
+            .slice(0, MAX_ITEMS);
+        for (const id of Object.keys(items)) delete items[id];
+        for (const [id, v] of keep) items[id] = v;
+        if (done) {
+            try {
+                log(
+                    `voice transcript ${err ? `failed (error ${err})` : "ready"}: request ${reqId}; read it with \`read run voice-transcript\``,
+                );
+            } catch {
+                /* a broken logger must not break the socket */
+            }
+        }
+        return true;
+    }
+
+    return {
+        observe(cmd, data) {
+            let recorded = 0;
+            try {
+                if (cmd !== CMD_PUSH_CONTROLS) return 0;
+                for (const c of readControls(data)) {
+                    if (c.actType !== VOICE_ACT_TYPE || c.act !== VOICE_ACT || !c.data) continue;
+                    if (record(c.data)) recorded++;
+                }
+                if (recorded) save();
+            } catch {
+                /* a frame of a shape nobody expected is skipped */
+            }
+            return recorded;
+        },
+    };
+}
+
+/**
+ * Record transcripts from a socket tap, for one account. Built the same by `listen` and `mcp start`.
+ *
+ * @param {object} args
+ * @param {{on: (cmd: number, fn: (frame: object) => void) => () => void}} args.tap
+ * @param {string} args.file
+ * @param {(line: string) => void} [args.log]
+ * @returns {{stop: () => void}}
+ */
+export function attachVoiceText({ tap, file, log }) {
+    const recorder = createVoiceText({ file, log });
+    const off = tap.on(CMD_PUSH_CONTROLS, (frame) => recorder.observe(CMD_PUSH_CONTROLS, frame.data));
+    return {
+        stop() {
+            off();
+        },
+    };
+}

package/src/core/webhook-outbox.js

@@ -19,12 +19,20 @@
  * The file is `accounts/<ownId>/webhook-outbox.db`, separate from `zalo.db`: it holds no message rows
  * (AGENTS.md section 13 limits those to three writers) and a damaged outbox can be deleted without
  * touching the cache.
+ *
+ * At rest it is encrypted like `zalo.db` (SQLCipher v4, src/core/sqlite.js) under its own subkey of the account's data
+ * key (purpose `outbox`): the queued bodies are message content. A 2.x plaintext outbox is migrated by the first
+ * writer: one with nothing queued or dead-lettered is simply removed and recreated encrypted (it holds only counters),
+ * one with rows is copied, encrypted and verified like the database. A read-only opener (`mcp webhook status`) never
+ * migrates: it fails closed with the reason.
  */
 
 import { createHmac } from "node:crypto";
-import { mkdirSync } from "node:fs";
-import { join } from "node:path";
-import Database from "better-sqlite3";
+import { existsSync, mkdirSync, rmSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { KeystoreError } from "./keystore/container.js";
+import { checkLock } from "./lock.js";
+import { Database, dbFileState, migratePlaintextDatabase, openDatabase, shredFile } from "./sqlite.js";
 
 export const OUTBOX_FILE = "webhook-outbox.db";
 const DAY_MS = 24 * 60 * 60 * 1000;
@@ -59,19 +67,67 @@ export function webhookHeaders(body, secret) {
     return headers;
 }
 
+/** The data-key purpose the outbox is encrypted under. */
+export const OUTBOX_PURPOSE = "outbox";
+
+/**
+ * Encrypt a plaintext (2.x) outbox in place. Nothing queued and nothing dead-lettered: the file holds only counters,
+ * so it is shredded and removed (the next open recreates it encrypted). Otherwise it is migrated like the message
+ * database (copy, `PRAGMA rekey`, verify, swap, shred). Idempotent: an absent or encrypted outbox is left alone.
+ *
+ * @param {string} file - `accounts/<ownId>/webhook-outbox.db`
+ * @param {{log?: (line: string) => void, hooks?: object}} [opts]
+ * @returns {"absent"|"already-encrypted"|"recreated"|"encrypted"}
+ * @throws {import("./keystore/container.js").KeystoreError} `IN_USE`, `NO_SPACE`, and the key layer's codes
+ */
+export function migratePlaintextOutbox(file, { log = (l) => console.error(l), hooks } = {}) {
+    const state = dbFileState(file);
+    if (state === "absent" || state === "empty") return "absent";
+    if (state !== "plaintext") return "already-encrypted";
+    let rows = 0;
+    const plain = new Database(file, { readonly: true, fileMustExist: true, timeout: 5000 });
+    try {
+        const has = plain.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'outbox'").get();
+        if (has) rows = plain.prepare("SELECT count(*) AS n FROM outbox WHERE status IN ('pending', 'dead')").get().n;
+    } finally {
+        plain.close();
+    }
+    if (rows === 0) {
+        const holder = checkLock(dirname(file));
+        if (holder.locked && holder.pid !== process.pid) {
+            throw new KeystoreError(
+                `${file} is still plaintext and must be encrypted, but process ${holder.pid} holds this account ` +
+                    "(daemon.lock): stop it (an older version cannot read the encrypted outbox), then run again",
+                "IN_USE",
+            );
+        }
+        // Only counters: nothing worth a verified copy. A writer still in WAL has nothing queued either.
+        shredFile(file);
+        for (const s of ["-wal", "-shm", "-journal"]) rmSync(file + s, { force: true });
+        log(`[outbox] ${file} held nothing queued: removed, it is recreated encrypted`);
+        return "recreated";
+    }
+    const res = migratePlaintextDatabase(file, { purpose: OUTBOX_PURPOSE, log, ...(hooks ? { hooks } : {}) });
+    return res.migrated ? "encrypted" : "already-encrypted";
+}
+
 /**
  * Open (creating if needed) the outbox of one account.
  *
  * @param {string} accountDir - `accounts/<ownId>`
- * @param {{ now?: () => number, readonly?: boolean }} [options] - `readonly` for status commands: never creates the file
+ * @param {{ now?: () => number, readonly?: boolean, log?: (line: string) => void }} [options] - `readonly` for
+ *   status commands: never creates or migrates the file (a plaintext one is `KeystoreError` `PLAINTEXT`)
  * @returns {object} Outbox handle
+ * @throws {import("./keystore/container.js").KeystoreError} when the outbox's key cannot be had (fail closed)
  */
-export function openOutbox(accountDir, { now = Date.now, readonly = false } = {}) {
-    if (!readonly) mkdirSync(accountDir, { recursive: true });
-    const db = new Database(outboxPath(accountDir), readonly ? { readonly: true, fileMustExist: true } : {});
+export function openOutbox(accountDir, { now = Date.now, readonly = false, log } = {}) {
+    if (!readonly) {
+        mkdirSync(accountDir, { recursive: true });
+        migratePlaintextOutbox(outboxPath(accountDir), log ? { log } : {});
+    }
+    const db = openDatabase(outboxPath(accountDir), { purpose: OUTBOX_PURPOSE, readonly, ...(log ? { log } : {}) });
     db.pragma("busy_timeout = 5000");
     if (!readonly) {
-        db.pragma("journal_mode = WAL");
         db.exec(`
             CREATE TABLE IF NOT EXISTS outbox (
                 seq INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -211,12 +267,24 @@ export function openOutbox(accountDir, { now = Date.now, readonly = false } = {}
         },
 
         /**
-         * @param {{ status?: 'pending'|'dead', limit?: number }} [filter]
+         * Delete the dead rows that died before `cutoff` (`retention.outboxDead`, src/core/storage-quota.js).
+         * @param {number} cutoff - Epoch ms
+         * @returns {number} Rows removed
+         */
+        purgeDeadBefore(cutoff) {
+            return db
+                .prepare("DELETE FROM outbox WHERE status = 'dead' AND COALESCE(dead_at, created_at) < ?")
+                .run(Number(cutoff)).changes;
+        },
+
+        /**
+         * @param {{ status?: 'pending'|'dead', limit?: number|null }} [filter] - no `limit` = every row
          * @returns {Array<object>} Rows, oldest first; `body` is the JSON text
          */
-        list({ status, limit = 50 } = {}) {
+        list({ status, limit = null } = {}) {
             const where = status ? "WHERE status = ?" : "";
-            const args = status ? [status, limit] : [limit];
+            const n = Number(limit) > 0 ? Math.floor(Number(limit)) : -1; // SQLite: LIMIT -1 = no limit
+            const args = status ? [status, n] : [n];
             return stmt(
                 `SELECT seq, status, attempts, created_at, last_attempt_at, last_error, dead_at, body FROM outbox ${where} ORDER BY seq LIMIT ?`,
             ).all(...args);
@@ -315,7 +383,7 @@ export function createWebhookDispatcher({
     setTimer = setTimeout,
     clearTimer = clearTimeout,
 }) {
-    const size = Math.max(1, Math.floor(batchSize) || 1);
+    let size = Math.max(1, Math.floor(batchSize) || 1);
     let timer = null;
     let poller = null;
     let timerAt = 0;
@@ -479,6 +547,16 @@ export function createWebhookDispatcher({
                 poller.unref?.();
             }
         },
+        /**
+         * A live policy change (src/core/policy-store.js): a new receiver, batch size or max age, from the next POST on.
+         * @param {{ url?: string, batchSize?: number, maxAgeMs?: number }} [o]
+         */
+        reconfigure(o = {}) {
+            if (o.url) url = o.url;
+            if (o.batchSize) size = Math.max(1, Math.floor(o.batchSize) || 1);
+            if (o.maxAgeMs) maxAgeMs = o.maxAgeMs;
+            schedule(0);
+        },
         /** An event was just enqueued. */
         notify() {
             schedule(size > 1 ? batchWindowMs : 0);
@@ -567,9 +645,29 @@ export function startWebhookDelivery({
             }
         },
         status: () => dispatcher.status(),
+        /** A live policy change: the receiver, batch size or max age (createWebhookDispatcher reconfigure). */
+        reconfigure: (o) => dispatcher.reconfigure(o),
         /** Stop sending and release the file. Queued events stay for the next start. */
         stop() {
             dispatcher.stop({ graceMs: 0 }).finally(() => outbox.close());
         },
     };
 }
+
+/**
+ * Delete an account's dead webhook deliveries older than `cutoff` (the retention purge). An account with no outbox
+ * file has none, and none is created.
+ *
+ * @param {string} accountDir
+ * @param {number} cutoff - Epoch ms
+ * @returns {number} Rows removed
+ */
+export function purgeDeadDeliveries(accountDir, cutoff) {
+    if (!existsSync(outboxPath(accountDir))) return 0;
+    const outbox = openOutbox(accountDir);
+    try {
+        return outbox.purgeDeadBefore(cutoff);
+    } finally {
+        outbox.close();
+    }
+}