@ardennguyen/zalo-agent-cli 2.5.8 → 3.0.0

package/src/core/sync-v2/board.js

@@ -21,64 +21,24 @@
  * only announcements. The objects themselves exist solely here.
  *
  * Cost shapes the design: a full account is ~1500 threads and two-plus calls
- * each, so this runs at low concurrency with a pause between calls, supports a
- * thread cap, and skips threads synced recently. It is deliberately not part
+ * each, so this runs at low concurrency (at most 3) with a pause between calls,
+ * and stops the whole pass on Zalo's 221. It is deliberately not part
  * of the phone-backed restore — no confirmation is needed and nothing here
  * touches the WebSocket.
  */
-import { createRequire } from "node:module";
-import { dirname, join } from "node:path";
-import { ThreadType } from "zca-js";
+import { THREAD_TYPE } from "../../zalo/enums.js";
+import { boardCursor, listOneOneBoard } from "../../zalo/api/boards.js";
 import { upsertBoardItem, upsertReminder } from "../db.js";
 
-const require = createRequire(import.meta.url);
-
-/** zca-js BoardType: what /api/board/list returns per item. */
-export const BOARD_TYPES = { 1: "note", 2: "pinned_message", 3: "poll" };
-
-let _utils = null;
-/** zca-js internals, located the same way ./gid.js does. */
-function zcaUtils() {
-    if (_utils) return _utils;
-    _utils = require(join(dirname(require.resolve("zca-js")), "utils.cjs"));
-    return _utils;
-}
-
 /**
- * List a 1-1 conversation's board (notes, pinned messages, polls).
- *
- * zca-js exposes /api/board/oneone/list only through getListReminder, hardwired
- * to board_type 1, so reminders were the only thing a DM ever returned. This
- * asks the same endpoint for every board type, which is what Zalo Web does.
- *
- * @returns {Promise<{items: Array<object>}>}
+ * List a 1-1 conversation's board (notes, pinned messages, reminders): `/api/board/oneone/list` with board type 0,
+ * every kind. It lives in the protocol layer (src/zalo/api/boards.js); re-exported here
+ * where it used to be.
  */
-async function listOneOneBoard(api, threadId, page, pageSize) {
-    const zu = zcaUtils();
-    const base = `${api.zpwServiceMap.group_board[0]}/api/board/oneone/list`;
-    const call = zu.apiFactory()((_api, ctx, utils) => async () => {
-        const params = {
-            objectData: JSON.stringify({
-                uid: String(threadId),
-                board_type: 0, // 0 = every kind, not just reminders
-                page,
-                count: pageSize,
-                last_id: 0,
-                last_type: 0,
-            }),
-            imei: ctx.imei,
-        };
-        const enc = utils.encodeAES(JSON.stringify(params));
-        if (!enc) throw new Error("failed to encrypt oneone board params");
-        const resp = await utils.request(utils.makeURL(base, { params: enc }), { method: "GET" });
-        return utils.resolve(resp);
-    })(api.getContext(), api);
+export { listOneOneBoard };
 
-    const raw = await call();
-    const parsed = typeof raw === "string" ? JSON.parse(raw) : raw;
-    const items = parsed?.items || parsed?.data?.items || (Array.isArray(parsed) ? parsed : []);
-    return { items };
-}
+/** What /api/board/list returns per item (`boardType`). */
+export const BOARD_TYPES = { 1: "note", 2: "pinned_message", 3: "poll" };
 
 /** Stringify an id Zalo may send as a number, or leave it null. */
 const str = (v) => (v === undefined || v === null ? null : String(v));
@@ -187,33 +147,11 @@ function asList(resp) {
     return [];
 }
 
-/**
- * Sync board items and reminders for the given threads.
- *
- * Every per-thread call is individually guarded: one thread that 403s (a group
- * you were removed from, a DM with a blocked user) must not abort a run that
- * covers hundreds of others.
- *
- * @param {object} opts
- * @param {object} opts.api - logged-in zca-js api
- * @param {Array<{threadId: string, type: "dm"|"group", name?: string}>} opts.threads
- * @param {boolean} [opts.boards=true] - fetch notes/pinned/polls (groups only)
- * @param {boolean} [opts.reminders=true] - fetch reminders
- * @param {number} [opts.concurrency=3] - parallel threads; Zalo rate-limits hard
- * @param {number} [opts.delayMs=120] - pause between calls within a worker
- * @param {number} [opts.pageSize=50]
- * @param {number} [opts.maxPages=10] - board pagination cap per thread
- * @param {(p: object) => void} [opts.onProgress] - one `{phase: "thread",
- *   threadId, ok}` event per finished thread, where `ok` is false if any of
- *   that thread's own calls failed. Callers that clear per-thread state must
- *   check it.
- * @returns {Promise<{threads:number, boardItems:number, reminders:number, failed:number, failures:Array<object>}>}
- */
 /**
  * A failure reason someone can act on.
  *
- * zca-js builds its error from the server's error_message, which for these
- * endpoints is often null -- so the message arrives as the literal string
+ * The error is built from the server's error_message, which for these
+ * endpoints is often null -- so the message can arrive as the literal string
  * "null" and the only signal is the numeric code. A live board pass reported
  * 165 failures, every one of them printed as "null".
  *
@@ -228,31 +166,97 @@ export function describeZaloError(e) {
     return text || String(e);
 }
 
+/** Board items per page (MAX_REF_LOAD_TOPIC). */
+export const BOARD_PAGE_SIZE = 20;
+/** Reminders asked per page. */
+export const REMINDER_PAGE_SIZE = 50;
+/** Threads fetched at once, at most: Zalo itself opens one board at a time, so this stays low. */
+export const MAX_BOARD_CONCURRENCY = 3;
+/** Zalo's "too many requests". */
+const TOO_MANY_REQUESTS = 221;
+
+/** The id a board item or reminder is known by, for the "did the page move" check. */
+const itemKey = (item) => {
+    const d = item?.data ?? item;
+    const id = d?.poll_id ?? d?.pollId ?? d?.id ?? d?.reminderId ?? item?.reminderId;
+    return id === undefined || id === null ? null : `${item?.boardType ?? ""}:${id}`;
+};
+
+/**
+ * Sync board items and reminders for the given threads.
+ *
+ * Every per-thread call is individually guarded: one thread that 403s (a group
+ * you were removed from, a DM with a blocked user) must not abort a run that
+ * covers hundreds of others. A 221 ("too many requests") is different: it is
+ * about the account, not the thread, so the whole pass stops there and says how
+ * many threads were left (`stoppedBy: "rate-limit"`, `left`, `nextThreadId`).
+ *
+ * A board is paged 20 items a page, until a page
+ * comes back empty. There is no page cap; the runaway guard is a page that
+ * brings nothing new (the cursor did not move), which ends that board and is
+ * counted in `stalled`.
+ *
+ * @param {object} opts
+ * @param {object} opts.api - the logged-in api
+ * @param {Array<{threadId: string, type: "dm"|"group", name?: string}>} opts.threads
+ * @param {boolean} [opts.boards=true] - fetch notes/pinned/polls
+ * @param {boolean} [opts.reminders=true] - fetch reminders
+ * @param {number} [opts.concurrency=3] - parallel threads, at most 3
+ * @param {number} [opts.delayMs=120] - pause between calls within a worker
+ * @param {number} [opts.pageSize=20] - board items per page
+ * @param {number} [opts.maxPages=Infinity] - opt-in board page bound per thread
+ * @param {(p: object) => void} [opts.onProgress] - one `{phase: "thread",
+ *   threadId, ok}` event per finished thread, where `ok` is false if any of
+ *   that thread's own calls failed. Callers that clear per-thread state must
+ *   check it.
+ * @returns {Promise<{threads:number, boardItems:number, reminders:number, failed:number, failures:Array<object>,
+ *   stalled:number, stoppedBy:null|"rate-limit", left:number, nextThreadId:string|null}>}
+ */
 export async function syncBoards(opts = {}) {
     const {
         api,
         threads = [],
         boards = true,
         reminders = true,
-        concurrency = 3,
+        concurrency = MAX_BOARD_CONCURRENCY,
         delayMs = 120,
-        pageSize = 50,
-        maxPages = 10,
+        pageSize = BOARD_PAGE_SIZE,
+        maxPages = Infinity,
         onProgress = () => {},
     } = opts;
 
-    const stats = { threads: 0, boardItems: 0, reminders: 0, failed: 0, failures: [] };
+    const stats = {
+        threads: 0,
+        boardItems: 0,
+        reminders: 0,
+        failed: 0,
+        failures: [],
+        stalled: 0,
+        stoppedBy: null,
+        left: 0,
+        nextThreadId: null,
+    };
     if (!api || !threads.length) return stats;
 
     const note = (threadId, what, e) => {
         stats.failed++;
         if (stats.failures.length < 20) stats.failures.push({ threadId, what, reason: describeZaloError(e) });
     };
+    /** First thread index a rate limit left undone; the pass ends there. */
+    let stopAt = Infinity;
+    const limited = (e, index) => {
+        if (Number(e?.code) !== TOO_MANY_REQUESTS) return false;
+        stats.stoppedBy = "rate-limit";
+        stopAt = Math.min(stopAt, index);
+        return true;
+    };
 
     let cursor = 0;
     const worker = async () => {
         for (;;) {
-            const t = threads[cursor++];
+            if (stats.stoppedBy) return;
+            const index = cursor++;
+            const t = threads[index];
             if (!t) return;
             const isGroup = t.type === "group";
 
@@ -268,19 +272,35 @@ export async function syncBoards(opts = {}) {
 
             // Notes / pinned messages / polls, for BOTH thread kinds. A group
             // uses /api/board/list; a 1-1 uses /api/board/oneone/list, which
-            // takes the same board_type filter.
+            // takes the same board_type filter. A group's page 2 and later carry
+            // the cursor of the page before (the id and type of its last item),
+            // so an item added mid-scan does not shift one
+            // past the page boundary.
             if (boards) {
+                let pageCursor = { lastId: 0, lastType: 0 };
+                const seen = new Set();
                 for (let page = 1; page <= maxPages; page++) {
                     let resp;
                     try {
                         resp = isGroup
-                            ? await api.getListBoard({ page, count: pageSize }, t.threadId)
+                            ? await api.getListBoard({ page, count: pageSize, ...pageCursor }, t.threadId)
                             : await listOneOneBoard(api, t.threadId, page, pageSize);
                     } catch (e) {
+                        if (limited(e, index)) return;
                         noteHere("board", e);
                         break;
                     }
                     const items = resp?.items || resp?.data?.items || [];
+                    // Only an empty page ends the board: with the cursor, Zalo can answer
+                    // a short page while more items follow.
+                    if (items.length === 0) break;
+                    const keys = items.map(itemKey);
+                    if (keys.every((k) => k !== null && seen.has(k))) {
+                        // The cursor did not move: the same page again. Stop rather than loop.
+                        stats.stalled++;
+                        break;
+                    }
+                    for (const k of keys) if (k !== null) seen.add(k);
                     for (const item of items) {
                         const row = normalizeBoardItem(item, t.threadId, t.type);
                         if (!row) continue;
@@ -291,19 +311,32 @@ export async function syncBoards(opts = {}) {
                             noteHere("board-write", e);
                         }
                     }
-                    if (items.length < pageSize) break;
+                    pageCursor = boardCursor(items);
                     if (delayMs) await sleep(delayMs);
                 }
             }
 
             if (reminders) {
-                try {
-                    const resp = await api.getListReminder(
-                        { page: 1, count: pageSize },
-                        t.threadId,
-                        isGroup ? ThreadType.Group : ThreadType.User,
-                    );
-                    for (const r of asList(resp)) {
+                const seen = new Set();
+                for (let page = 1; ; page++) {
+                    let list;
+                    try {
+                        list = asList(
+                            await api.getListReminder(
+                                { page, count: REMINDER_PAGE_SIZE },
+                                t.threadId,
+                                isGroup ? THREAD_TYPE.GROUP : THREAD_TYPE.USER,
+                            ),
+                        );
+                    } catch (e) {
+                        if (limited(e, index)) return;
+                        noteHere("reminder", e);
+                        break;
+                    }
+                    const keys = list.map(itemKey);
+                    if (!list.length || (page > 1 && keys.every((k) => k !== null && seen.has(k)))) break;
+                    for (const k of keys) if (k !== null) seen.add(k);
+                    for (const r of list) {
                         const row = normalizeReminder(r, t.threadId, t.type);
                         if (!row) continue;
                         try {
@@ -313,8 +346,9 @@ export async function syncBoards(opts = {}) {
                             noteHere("reminder-write", e);
                         }
                     }
-                } catch (e) {
-                    noteHere("reminder", e);
+                    // A short page is the last one.
+                    if (list.length < REMINDER_PAGE_SIZE) break;
+                    if (delayMs) await sleep(delayMs);
                 }
             }
 
@@ -333,6 +367,10 @@ export async function syncBoards(opts = {}) {
         }
     };
 
-    await Promise.all(Array.from({ length: Math.max(1, Math.min(8, concurrency)) }, worker));
+    await Promise.all(Array.from({ length: Math.max(1, Math.min(MAX_BOARD_CONCURRENCY, concurrency)) }, worker));
+    if (stats.stoppedBy) {
+        stats.left = threads.length - stats.threads;
+        stats.nextThreadId = threads[stopAt]?.threadId ?? null;
+    }
     return stats;
 }

package/src/core/sync-v2/conv-state.js

@@ -13,7 +13,7 @@
  *
  * 2. Unread-mark ids arrive as JSON NUMBERS. A thread id is 19 digits, far
  *    past Number.MAX_SAFE_INTEGER, so `4123456789012345678` comes back as
- *    `4123456789012346000` before zca-js even returns it. Written with
+ *    `4123456789012346000` before the call even returns it. Written with
  *    String(id), the mark lands on a thread that does not exist. The exact id
  *    cannot be recovered from the number, but it can be MATCHED: a cached
  *    thread whose id rounds to the same double is the one -- provided exactly
@@ -35,10 +35,27 @@ export function parsePinnedId(raw) {
     return { threadId: s, type: null };
 }
 
+/**
+ * The pinned-list entry that is not a conversation: `m1` is skipped when the list is read (spec-actions
+ * 7.3), so it is neither a pin nor an unresolved id here.
+ */
+export const PINNED_NOT_A_CONVERSATION = "m1";
+
+/**
+ * The entries of a pinned-conversation list that are conversations: the list without {@link
+ * PINNED_NOT_A_CONVERSATION}. `conv pinned` and the pin sync both read the list through this.
+ *
+ * @param {unknown} list - `getPinConversations().conversations`
+ * @returns {Array<string|number>} the entries kept, in order ([] for anything not an array)
+ */
+export function pinnedConversations(list) {
+    return (Array.isArray(list) ? list : []).filter((raw) => String(raw ?? "") !== PINNED_NOT_A_CONVERSATION);
+}
+
 /**
  * Resolve a precision-lost numeric id to the one cached thread it came from.
  *
- * @param {number|string} lossy - the id as zca-js returned it
+ * @param {number|string} lossy - the id as the call returned it
  * @param {Array<{threadId: string, type: string}>} threads - cached threads to match against
  * @param {"group"|"dm"} [type] - restrict the match to one kind
  * @returns {{threadId: string|null, why?: "unknown"|"ambiguous"}}
@@ -61,7 +78,7 @@ export function resolveLossyThreadId(lossy, threads, type) {
  * whose state was cleared on another device.
  *
  * @param {object} args
- * @param {object} args.api - logged-in zca-js api (REST only; no socket)
+ * @param {object} args.api - logged-in session (REST only; no socket)
  * @param {number} [args.now] - clock, for tests
  * @param {Array<{threadId: string, type: string}>} [args.threads] - cached threads; defaults to the cache
  * @returns {Promise<{pinned: number, unpinned: number, unread: number, unmarked: number,
@@ -82,7 +99,7 @@ export async function syncConvState({ api, now = Date.now(), threads } = {}) {
     try {
         const res = await api.getPinConversations();
         pinnedIds = new Set();
-        for (const raw of res?.conversations || []) {
+        for (const raw of pinnedConversations(res?.conversations)) {
             const { threadId } = parsePinnedId(raw);
             // Only conversations the cache knows: a pin for a thread we have
             // never seen has nothing to attach to, and inventing a row would

package/src/core/sync-v2/gap-advice.js

@@ -3,9 +3,8 @@
  *
  * A gap is a window the daemon was not connected for, so whatever arrived in it
  * is missing from zalo.db. The daemon used to "attempt a mobile-sync backfill"
- * itself, over `pullMobileMsg`/`getCrossDB` — endpoints Zalo retired (MEASURED
- * 2026-09-20: still in Zalo Web's bundle, ZERO call sites across its 4,642
- * modules). That call cannot return history, so the daemon printed a hopeful
+ * itself, over `pullMobileMsg`/`getCrossDB` — endpoints Zalo retired. That
+ * call cannot return history, so the daemon printed a hopeful
  * line and then went quiet, leaving the gap `pending` forever with nothing said.
  *
  * The working restore is `transfer-sync-v2` (socket cmd 590/591), which the