@ardennguyen/zalo-agent-cli 2.5.8 → 3.0.0

package/src/zalo/listener/reactions.js

@@ -0,0 +1,224 @@
+/**
+ * Retrieve existing reactions from Zalo's server over the socket.
+ *
+ * Reactions are the one thing a transfer sync cannot restore — the Sync2
+ * payload has no reaction field — which this project long read as "a reaction
+ * not captured live is gone for good". That conclusion was wrong: they can be
+ * fetched over the SAME socket with cmd 610 (1-1) and cmd 611 (group), asked
+ * right after every connect. Measured live on a real account, one request
+ * pair returned 28 DM and 50 group reactions with full detail — target msgId,
+ * icon, rType and the reacting user — while the local cache held one row.
+ *
+ * The listener already speaks the request (`listener.requestOldReactions`) and
+ * emits the objects (`old_reactions`). Two things it does not
+ * do, which is why this module exists:
+ *
+ *  1. It surfaces neither `lastActionId` nor `more`, so a caller using only the
+ *     event gets exactly one page and cannot tell whether more exist. The
+ *     server said `more: 1` for groups on the very first probe. We read those
+ *     fields off the raw socket, the same way SyncV2 taps cmd 601.
+ *  2. Its old-reactions handler omits the `JSON.parse(content)` its live
+ *     handler performs, so `content` arrives as a string.
+ *     The caller's `store` absorbs both forms.
+ *
+ * Storing is the caller's: this module requests, pages and hands every reaction to `opts.store`. The CLI's
+ * wrapper (src/core/sync-v2/reactions.js) passes its live-store writer.
+ */
+
+import { decodeFrame } from "./frames.js";
+
+/** Thread type values. */
+const THREAD_USER = 0;
+const THREAD_GROUP = 1;
+
+/** Socket commands carrying the reaction backlog. */
+const CMD_DM = 610;
+const CMD_GROUP = 611;
+
+/** Default per-request deadline. A queue that has nothing to say says nothing. */
+export const DEFAULT_TIMEOUT_MS = 15000;
+
+/** Pages per thread type: none. The server pages with `more`; the runaway stop is an anchor (`lastActionId`) that
+ * does not move, reported as `stalled`. A caller may still pass a bound. */
+export const DEFAULT_MAX_PAGES = Infinity;
+
+/**
+ * Drain the reaction backlog for both thread types and hand what comes back to `opts.store`.
+ *
+ * The caller owns the socket: it must hold `daemon.lock`, start the listener
+ * and stop it afterwards — same contract as {@link backfillOverSocket}.
+ *
+ * @param {object} opts
+ * @param {import("./listener.js").Listener} opts.listener - a started listener
+ * @param {(reaction: object) => {stored: boolean, changed?: number, reason?: string}} opts.store - writes one
+ *   reaction; `reason: "reaction target not resolvable"` puts it in `unresolved`
+ * @param {number} [opts.timeoutMs=15000] - per-page deadline
+ * @param {number} [opts.maxPages=Infinity] - opt-in bound per thread type
+ * @param {boolean} [opts.applyRemovals=true] - apply backlog entries with an empty
+ *   icon (un-reacts). ON by default, because the backlog is an ordered ACTION
+ *   LOG, not a snapshot of current state: it replays adds and removals in
+ *   sequence, so skipping the removals leaves reactions the user already took
+ *   off. Measured — one message came back with four reaction events where the
+ *   app showed one; replaying in order reproduced the one.
+ * @param {(p: object) => void} [opts.onProgress]
+ * @returns {Promise<{pages: number, received: number, stored: number, changed: number,
+ *   skippedRemovals: number, byType: object, truncated: boolean, stalled: boolean, unresolved: object[]}>}
+ *   `unresolved` holds the reactions whose target message was not in the cache
+ *   when they arrived. The backlog reaches further back than a windowed
+ *   restore, and when the drain runs beside a restore its target may simply
+ *   not be written yet -- so a caller that has since stored more messages can
+ *   pass these back through its store once more.
+ */
+export async function drainReactionBacklog(opts = {}) {
+    const {
+        listener,
+        store,
+        timeoutMs = DEFAULT_TIMEOUT_MS,
+        maxPages = DEFAULT_MAX_PAGES,
+        applyRemovals = true,
+        onProgress = () => {},
+    } = opts;
+    if (!listener) throw new Error("drainReactions needs a started listener");
+    if (typeof store !== "function") throw new Error("drainReactionBacklog needs a store function");
+
+    const stats = {
+        pages: 0,
+        received: 0,
+        stored: 0,
+        changed: 0,
+        skippedRemovals: 0,
+        byType: { dm: 0, group: 0 },
+        truncated: false,
+        stalled: false,
+        unresolved: [],
+    };
+
+    // The raw tap is the only place `more` and `lastActionId` exist. Without it
+    // a drain stops after one page and reports success.
+    const envelopes = [];
+    const onRaw = (buf) => {
+        if (!(buf instanceof Buffer) || buf.length < 4) return;
+        const cmd = buf.readUInt16LE(1);
+        if (cmd !== CMD_DM && cmd !== CMD_GROUP) return;
+        try {
+            const parsed = JSON.parse(buf.subarray(4).toString("utf8"));
+            if (typeof parsed.data !== "string") return;
+            // decodeFrame is the listener's own decoder (src/zalo/listener/frames.js).
+            const data = decodeFrame(parsed, listener.cipherKey)?.data;
+            envelopes.push({ cmd, lastActionId: data?.lastActionId, more: Number(data?.more) || 0 });
+        } catch {
+            /* a frame we cannot read is not a frame we need */
+        }
+    };
+
+    /** One request/response round for a thread type. */
+    const page = (threadType, anchor) =>
+        new Promise((resolve) => {
+            const cmd = threadType === THREAD_USER ? CMD_DM : CMD_GROUP;
+            let settled = false;
+            const before = envelopes.length;
+
+            const finish = (objs) => {
+                if (settled) return;
+                settled = true;
+                listener.removeListener("old_reactions", handler);
+                clearTimeout(timer);
+                // Match the envelope to this round by command.
+                const env = envelopes.slice(before).find((e) => e.cmd === cmd) || {};
+                resolve({ objs, more: env.more || 0, lastActionId: env.lastActionId });
+            };
+
+            const handler = (objs, isGroup) => {
+                const want = threadType === THREAD_GROUP;
+                if (isGroup !== want) return; // the other type's answer
+                // The envelope can land after the event; give it a tick.
+                setTimeout(() => finish(objs || []), 50);
+            };
+            const timer = setTimeout(() => finish(null), timeoutMs);
+
+            listener.on("old_reactions", handler);
+            try {
+                listener.requestOldReactions(threadType, anchor ?? null);
+            } catch (e) {
+                onProgress({ phase: "warn", detail: `request failed: ${e.message}` });
+                finish(null);
+            }
+        });
+
+    listener.ws?.on("message", onRaw);
+    try {
+        for (const threadType of [THREAD_USER, THREAD_GROUP]) {
+            const label = threadType === THREAD_USER ? "dm" : "group";
+            let anchor = null;
+            for (let p = 0; p < maxPages; p++) {
+                const { objs, more, lastActionId } = await page(threadType, anchor);
+                if (objs === null) {
+                    onProgress({ phase: "timeout", type: label, detail: `no answer within ${timeoutMs}ms` });
+                    break;
+                }
+                stats.pages++;
+                stats.received += objs.length;
+                for (const r of objs) {
+                    const icon = reactionIcon(r);
+                    if (!icon && !applyRemovals) {
+                        stats.skippedRemovals++;
+                        continue;
+                    }
+                    const res = store(r);
+                    if (res.stored) {
+                        stats.stored++;
+                        stats.changed += res.changed || 0;
+                        stats.byType[label]++;
+                    } else if (res.reason === "reaction target not resolvable") {
+                        stats.unresolved.push(r);
+                    }
+                }
+                onProgress({
+                    phase: "page",
+                    type: label,
+                    page: p + 1,
+                    received: objs.length,
+                    stored: stats.stored,
+                    more,
+                });
+                // `more` is the server's own "there is another page"; without a
+                // fresh anchor we would re-request the same one forever.
+                if (!more) break;
+                if (!lastActionId || String(lastActionId) === String(anchor)) {
+                    // More was promised but the cursor did not move: stop instead of asking for the same page forever.
+                    stats.stalled = true;
+                    onProgress({ phase: "warn", detail: `${label}: the server's cursor did not advance; stopped` });
+                    break;
+                }
+                anchor = String(lastActionId);
+                if (p === maxPages - 1) stats.truncated = true;
+            }
+        }
+    } finally {
+        try {
+            listener.ws?.off?.("message", onRaw);
+        } catch {
+            /* the socket may already be gone */
+        }
+    }
+
+    return stats;
+}
+
+/**
+ * The icon on a reaction object, whichever form its content arrived in.
+ *
+ * @param {object} reaction
+ * @returns {string}
+ */
+function reactionIcon(reaction) {
+    const c = reaction?.data?.content;
+    if (typeof c === "string") {
+        try {
+            return JSON.parse(c)?.rIcon ?? "";
+        } catch {
+            return "";
+        }
+    }
+    return c?.rIcon ?? "";
+}

package/src/zalo/listener/socket-tap.js

@@ -0,0 +1,228 @@
+/**
+ * A raw reader on the listener's WebSocket that keeps what its typed events leave out.
+ *
+ * The listener decodes each frame and emits typed events built from one or two
+ * of its arrays -- the 501 handler reads only `msgs`, the 521 handler only
+ * `groupMsgs`, the 510/511 `old_messages` handler only the message arrays --
+ * and keeps nothing of the handshake (cmd 1) but its cipher `key`. Everything
+ * else in the envelope is gone before our handlers run:
+ *
+ *   queueStatus {"<queue>": {ids, lastId, evict}}   the per-queue cursors
+ *   lastActionId, more                               offline-queue paging
+ *   resetLastActionId                                "start the cursor over"
+ *   clearUnreads, pageMsgs, delivereds, seens        read state, page messages
+ *   qCmds, ctrl_qCmds, bk_cmds (cmd 1)               which queues to drain
+ *
+ * A client is expected to read all of it. This does: it listens beside the listener on
+ * the same raw `ws`, decodes each frame with the listener's own decoder
+ * (src/zalo/listener/frames.js), and hands `{cmd, subCmd, body, data}` to whoever subscribed to that
+ * cmd. The reaction backlog drain has done this privately since 2026-09-28
+ * (./reactions.js); this is the same technique, shared, for everything
+ * that needs a field the typed events drop -- the offline-queue catch-up first
+ * (src/core/self-heal.js), read-state sync and pageMsgs after it.
+ *
+ * It reads and routes; it never writes and never sends. `listen` and `mcp start` both attach it (AGENTS.md §13).
+ */
+import { decodeFrame } from "./frames.js";
+
+/** The handshake: its answer carries the cipher key and the queue lists. */
+export const CMD_AUTHEN = 1;
+
+/**
+ * Read one raw socket frame: the 4-byte header, then its JSON body.
+ *
+ * The header is `[version, cmd as uint16 LE, subCmd]` (./frames.js `readHeader`).
+ * A body of `{encrypt, data: <string>}` is decoded with the session's cipher
+ * key -- the handshake frame carries its own `key`, which the listener adopts at the
+ * same moment -- and any other body is taken as it is.
+ *
+ * @param {Buffer} buf
+ * @param {string} [cipherKey]
+ * @returns {{version: number, cmd: number, subCmd: number, body: object, data: object|null}|null}
+ *   null for anything that is not a readable frame
+ */
+export function readFrame(buf, cipherKey) {
+    if (!Buffer.isBuffer(buf) || buf.length < 4) return null;
+    const version = buf[0];
+    const cmd = buf.readUInt16LE(1);
+    const subCmd = buf[3];
+    let parsed;
+    try {
+        parsed = JSON.parse(buf.subarray(4).toString("utf8"));
+    } catch {
+        return null;
+    }
+    if (!parsed || typeof parsed !== "object") return null;
+    let body = parsed;
+    if (typeof parsed.data === "string" && typeof parsed.encrypt === "number") {
+        try {
+            body = decodeFrame(parsed, parsed.key || cipherKey);
+        } catch {
+            return null;
+        }
+    }
+    const data = body && typeof body === "object" && body.data && typeof body.data === "object" ? body.data : null;
+    return { version, cmd, subCmd, body, data };
+}
+
+/**
+ * Why {@link readFrame} could not read a frame, in words without its content: for the log, so a dropped answer is
+ * never silent.
+ *
+ * @param {Buffer} buf
+ * @param {string} [cipherKey]
+ * @returns {string}
+ */
+export function unreadableReason(buf, cipherKey) {
+    if (!Buffer.isBuffer(buf) || buf.length < 4) return "shorter than a header";
+    let parsed;
+    try {
+        parsed = JSON.parse(buf.subarray(4).toString("utf8"));
+    } catch {
+        return "body is not JSON";
+    }
+    if (!parsed || typeof parsed !== "object") return "body is not an object";
+    if (typeof parsed.data === "string" && typeof parsed.encrypt === "number") {
+        try {
+            decodeFrame(parsed, parsed.key || cipherKey);
+        } catch (e) {
+            return `encrypt ${parsed.encrypt} body did not decode (${cipherKey || parsed.key ? "a key was set" : "no key yet"}): ${e?.message || e}`;
+        }
+    }
+    return "unknown";
+}
+
+/**
+ * Build the tap. One per daemon: it outlives each socket and each re-login.
+ *
+ * @param {object} [opts]
+ * @param {(line: string) => void} [opts.log] - where a failing subscriber is reported
+ * @returns {{
+ *   attach: (listener: object) => void,
+ *   on: (cmd: number, fn: (frame: object, listener: object) => void) => () => void, (a frame also carries `raw`, its bytes)
+ *   onAny: (fn: (view: object, listener: object) => void) => () => void, (every frame's header; `view.frame()` decodes)
+ *   auth: () => object|null,
+ * }} `attach` taps the listener's current socket and every one it opens after;
+ *   `on` subscribes to one cmd and returns the unsubscribe; `auth` is the last
+ *   handshake's `data` (qCmds, ctrl_qCmds, bk_cmds), or null before the first
+ */
+export function createSocketTap({ log = (line) => console.error(line) } = {}) {
+    /** @type {Map<number, Set<Function>>} */
+    const subscribers = new Map();
+    /** Subscribers to every frame's header, for what only needs to know that a command arrived. */
+    const headerSubscribers = new Set();
+    /** Sockets already tapped: a re-login attaches again, and a frame must arrive once. */
+    const tapped = new WeakSet();
+    /** Listeners already watched for new sockets. */
+    const watched = new WeakSet();
+    let lastAuth = null;
+
+    function deliver(frame, listener) {
+        if (frame.cmd === CMD_AUTHEN && frame.data) lastAuth = frame.data;
+        const fns = subscribers.get(frame.cmd);
+        if (!fns) return;
+        for (const fn of [...fns]) {
+            try {
+                fn(frame, listener);
+            } catch (e) {
+                try {
+                    log(`socket tap: a cmd ${frame.cmd} handler failed: ${e?.message || e}`);
+                } catch {
+                    /* a broken logger must not break the socket either */
+                }
+            }
+        }
+    }
+
+    function tapSocket(listener) {
+        const ws = listener?.ws;
+        if (!ws || typeof ws.on !== "function" || tapped.has(ws)) return;
+        tapped.add(ws);
+        ws.on("message", (buf) => {
+            // Only cmds someone asked for are decoded, so a quiet tap costs
+            // one header read per frame; the handshake is always kept.
+            if (!Buffer.isBuffer(buf) || buf.length < 4) return;
+            const cmd = buf.readUInt16LE(1);
+            if (headerSubscribers.size) {
+                // The header alone; `frame()` decodes the body only for a subscriber that asks, once.
+                let decoded;
+                const view = {
+                    cmd,
+                    subCmd: buf[3],
+                    size: buf.length,
+                    frame: () => (decoded === undefined ? (decoded = readFrame(buf, listener.cipherKey)) : decoded),
+                };
+                for (const fn of [...headerSubscribers]) {
+                    try {
+                        fn(view, listener);
+                    } catch (e) {
+                        try {
+                            log(`socket tap: a header handler failed: ${e?.message || e}`);
+                        } catch {
+                            /* a broken logger must not break the socket either */
+                        }
+                    }
+                }
+            }
+            if (cmd !== CMD_AUTHEN && !subscribers.has(cmd)) return;
+            const frame = readFrame(buf, listener.cipherKey);
+            if (frame) {
+                // The bytes as they came, for a subscriber that wants to hand a frame on (the control replay).
+                frame.raw = buf;
+                deliver(frame, listener);
+            } else if (subscribers.has(cmd)) {
+                // A frame someone waits for and could not read: the wait would otherwise just run out, unexplained.
+                try {
+                    log(
+                        `socket tap: a cmd ${cmd}/${buf[3]} frame (${buf.length} bytes) could not be read: ` +
+                            unreadableReason(buf, listener.cipherKey),
+                    );
+                } catch {
+                    /* a broken logger must not break the socket either */
+                }
+            }
+        });
+    }
+
+    return {
+        attach(listener) {
+            if (!listener) return;
+            tapSocket(listener);
+            // The listener opens a NEW ws on every retry and announces it with
+            // `connected` from its onopen, before any frame can arrive -- so
+            // re-arming there never misses the handshake.
+            if (!watched.has(listener) && typeof listener.on === "function") {
+                watched.add(listener);
+                listener.on("connected", () => tapSocket(listener));
+            }
+        },
+
+        on(cmd, fn) {
+            const key = Number(cmd);
+            if (!subscribers.has(key)) subscribers.set(key, new Set());
+            subscribers.get(key).add(fn);
+            return () => {
+                const set = subscribers.get(key);
+                if (!set) return;
+                set.delete(fn);
+                if (set.size === 0) subscribers.delete(key);
+            };
+        },
+
+        /**
+         * Subscribe to the header of EVERY frame: `{cmd, subCmd, size, frame()}`, where `frame()` decodes the body on
+         * demand. Costs one header read per frame while nobody decodes.
+         *
+         * @param {(view: {cmd: number, subCmd: number, size: number, frame: () => (object|null)}, listener: object) => void} fn
+         * @returns {() => void} unsubscribe
+         */
+        onAny(fn) {
+            headerSubscribers.add(fn);
+            return () => headerSubscribers.delete(fn);
+        },
+
+        auth() {
+            return lastAuth;
+        },
+    };
+}