@sleep2agi/commhub-server 0.9.0-preview.13 → 0.9.0-preview.130

package/src/task-queue-ahead-http.test.ts

@@ -0,0 +1,309 @@
+// #500 step 2 —— 派活时让发送方看见目标的队列。HTTP 集成测试(真实 Bun.serve,私有端口,临时库)。
+// test2123 在真实 PostgreSQL 上原样再跑一遍(COMMHUB_TEST_PG_URL)。
+//
+// 任务一律经真入口派出:POST /api/task(用户 / 节点令牌)和 /mcp send_task(节点令牌);消息经 /mcp send_message。
+// 只有「历史耗时」和「已开工」这类节点侧时间戳用 SQL 拨。
+//
+// 钉住:
+//   1. queue_ahead = 派这一条之前目标上开着的任务(created/delivered/acked/running,24 小时内);
+//      消息不算、终态不算、超过 24 小时的遗弃行不算、别的节点 / 别的网络不算。REST 与 MCP 同一个数。
+//   2. target_busy:有已开工(acked / running / consumed_at)的开着任务 → true;会话 working(report_status 真入口)→ true。
+//   3. est_wait_minutes:前面没人 → 0;有人但没历史(< 3 条 replied)→ null;有历史 → 中位数 × queue_ahead。
+//   4. warning:queue_ahead 2 → 没有;3 → 有,且仍然 200 ok(从不拒绝);est_wait > 30 → 有。
+//   5. /api/status 全量每行 queue_depth;light / 旧别名解析器投影没有这个字段(字节不变);
+//      只写 tasks 的语句也让全量缓存失效(ETag 变、queue_depth 跟着变);MCP get_all_status 每行 queue_depth。
+//
+// 跑法:cd server && COMMHUB_DB=/tmp/x.db bun test src/task-queue-ahead-http.test.ts
+
+import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+const DIR = mkdtempSync(join(tmpdir(), "anet-queue-ahead-"));
+process.env.COMMHUB_DB ||= join(DIR, "hub.db");
+process.env.COMMHUB_UPLOADS_DIR = join(DIR, "uploads");
+process.env.HOST = "127.0.0.1";
+const PW = "QueueAhead123!x";
+const stamp = Date.now();
+
+let db: any;
+let hub: any = null;
+let BASE = "";
+let NET = "", NET2 = "";
+let bossToken = "", boss2Token = "";
+type Node = { alias: string; nodeId: string; token: string };
+const nodes: Record<"sender" | "target" | "idle", Node> = {} as any;
+let other: Node;
+
+async function send(token: string, method: string, path: string, payload?: unknown, headers: Record<string, string> = {}) {
+  const res = await fetch(`${BASE}${path}`, {
+    method,
+    headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json", ...headers },
+    body: payload === undefined ? undefined : JSON.stringify(payload),
+  });
+  return { status: res.status, headers: res.headers, body: await res.json().catch(() => null) as any };
+}
+async function tool(token: string, name: string, args: Record<string, unknown>) {
+  const res = await fetch(`${BASE}/mcp`, {
+    method: "POST",
+    headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json", Accept: "application/json, text/event-stream", "MCP-Protocol-Version": "2025-03-26" },
+    body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name, arguments: args } }),
+  });
+  const raw = await res.text();
+  const data = raw.split("\n").filter((x) => x.startsWith("data:"));
+  const payload = data.length ? JSON.parse(data.at(-1)!.slice(5).trim()) : JSON.parse(raw);
+  return JSON.parse(payload.result.content[0].text);
+}
+
+let seq = 0;
+const text = () => `queue case ${stamp} #${++seq}`;
+/** 经 POST /api/task 派一条,返回整个响应体。 */
+async function dispatch(token: string, to: string, extra: Record<string, unknown> = {}) {
+  const r = await send(token, "POST", "/api/task", { alias: to, task: text(), network_id: NET, ...extra });
+  expect(r.status).toBe(200);
+  expect(r.body.ok).toBe(true);
+  return r.body;
+}
+const setStatus = (taskId: string, status: string) => db.run("UPDATE tasks SET status = ?1 WHERE task_id = ?2", [status, taskId]);
+const reportStatus = (n: Node, status: string, task?: string) =>
+  tool(n.token, "report_status", { resume_id: `sdk-${n.nodeId}`, alias: n.alias, status, node_id: n.nodeId, network_id: NET, ...(task ? { task } : {}) });
+/** 目标上 3 条 replied 历史:耗时 10 / 20 / 30 分钟(started_at 到 completed_at),中位数 20。 */
+async function seedHistory(to: Node, minutes = [10, 20, 30]) {
+  for (const m of minutes) {
+    const b = await dispatch(bossToken, to.alias);
+    db.run(
+      `UPDATE tasks SET status = 'replied', created_at = datetime('now', '-${m + 5} minutes'), started_at = datetime('now', '-${m + 1} minutes'),
+         completed_at = datetime('now', '-1 minutes') WHERE task_id = ?1`,
+      [b.task_id],
+    );
+  }
+}
+
+beforeAll(async () => {
+  ({ db } = await import("./db.js"));
+  const { createNetworkTokenForNode, register } = await import("./auth.js");
+  const boss = register(`qa_boss_${stamp}`, PW);
+  expect(boss.ok).toBe(true);
+  bossToken = boss.token!; NET = boss.network_id!;
+  const bossId = boss.user!.user_id;
+  for (const key of ["sender", "target", "idle"] as const) {
+    const alias = `qa-${key}-${stamp}`;
+    const nodeId = `n_qa_${key}_${stamp}`;
+    const minted = createNetworkTokenForNode(bossId, NET, alias, nodeId);
+    expect(minted.ok).toBe(true);
+    nodes[key] = { alias, nodeId, token: minted.token! };
+  }
+  // 另一个网络里一个**同名**节点:它的任务不许算进 NET 的队列。
+  const boss2 = register(`qa_boss2_${stamp}`, PW);
+  boss2Token = boss2.token!; NET2 = boss2.network_id!;
+  const m2 = createNetworkTokenForNode(boss2.user!.user_id, NET2, nodes.target.alias, `n_qa_other_${stamp}`);
+  other = { alias: nodes.target.alias, nodeId: `n_qa_other_${stamp}`, token: m2.token! };
+  const mod: any = await import("./server.js");
+  hub = mod.bootServer({ port: 0, hostname: "127.0.0.1" });
+  BASE = `http://127.0.0.1:${hub.port}`;
+  for (const n of Object.values(nodes)) expect((await reportStatus(n, "idle")).ok).toBe(true);
+  expect((await tool(other.token, "report_status", { resume_id: `sdk-${other.nodeId}`, alias: other.alias, status: "idle", node_id: other.nodeId, network_id: NET2 })).ok).toBe(true);
+}, 30_000);
+
+beforeEach(async () => {
+  // 每条用例从「目标上没有开着的任务、没有历史、会话 idle」开始。
+  db.run("UPDATE tasks SET status = 'cancelled', completed_at = datetime('now') WHERE network_id IN (?1, ?2) AND status IN ('created', 'delivered', 'acked', 'running')", [NET, NET2]);
+  db.run("DELETE FROM tasks WHERE network_id = ?1 AND status = 'replied'", [NET]);
+  for (const n of Object.values(nodes)) await reportStatus(n, "idle");
+});
+
+afterAll(() => {
+  try { hub?.stop?.(true); } catch {}
+  try { rmSync(DIR, { recursive: true, force: true }); } catch {}
+});
+
+describe("#500 queue_ahead on dispatch", () => {
+  test("empty target → queue_ahead 0, not busy, est_wait 0, no warning (REST and MCP)", async () => {
+    const r = await dispatch(nodes.sender.token, nodes.target.alias);
+    expect(r).toMatchObject({ queue_ahead: 0, target_busy: false, est_wait_minutes: 0 });
+    expect(r.warning).toBeUndefined();
+    setStatus(r.task_id, "cancelled");
+    const m = await tool(nodes.sender.token, "send_task", { alias: nodes.target.alias, task: text(), network_id: NET });
+    expect(m.ok).toBe(true);
+    expect(m).toMatchObject({ queue_ahead: 0, target_busy: false, est_wait_minutes: 0 });
+    expect(m.warning).toBeUndefined();
+  });
+
+  test("counts open tasks only: messages, terminal, >24 h stale, other nodes and other networks are excluded", async () => {
+    const a = await dispatch(bossToken, nodes.target.alias);        // delivered → counts
+    const b = await dispatch(nodes.idle.token, nodes.target.alias); // delivered → counts (another sender)
+    // 消息:只写 inbox,不算。
+    expect((await tool(nodes.sender.token, "send_message", { alias: nodes.target.alias, message: `msg ${stamp}`, network_id: NET })).ok).toBe(true);
+    // 终态:不算。
+    for (const st of ["replied", "failed", "expired", "cancelled"]) setStatus((await dispatch(bossToken, nodes.target.alias)).task_id, st);
+    // 24 小时之前派出、从没终态的遗弃行:不算。
+    const stale = await dispatch(bossToken, nodes.target.alias);
+    db.run("UPDATE tasks SET status = 'running', created_at = datetime('now', '-25 hours') WHERE task_id = ?1", [stale.task_id]);
+    // 别的节点 / 别的网络里同名节点:不算。
+    await dispatch(bossToken, nodes.idle.alias);
+    const o = await send(boss2Token, "POST", "/api/task", { alias: other.alias, task: text(), network_id: NET2 });
+    expect(o.status).toBe(200);
+    expect(o.body.queue_ahead).toBe(0);
+
+    const r = await dispatch(nodes.sender.token, nodes.target.alias);
+    expect(r.queue_ahead).toBe(2);
+    expect(r.target_busy).toBe(false);
+    // MCP 同一个数(上一条也开着了 → 3)。
+    const m = await tool(nodes.sender.token, "send_task", { alias: nodes.target.alias, task: text(), network_id: NET });
+    expect(m.queue_ahead).toBe(3);
+    // acked / running 也是开着的。
+    setStatus(a.task_id, "acked"); setStatus(b.task_id, "running");
+    const r2 = await dispatch(nodes.sender.token, nodes.target.alias);
+    expect(r2.queue_ahead).toBe(4);
+  });
+
+  test("target_busy: a started open task, or the session reporting working", async () => {
+    const a = await dispatch(bossToken, nodes.target.alias);
+    expect((await dispatch(nodes.sender.token, nodes.target.alias)).target_busy).toBe(false);
+    // 运行时取走(consumed_at)但状态还是 delivered → 已开工。
+    db.run("UPDATE tasks SET consumed_at = datetime('now') WHERE task_id = ?1", [a.task_id]);
+    expect((await dispatch(nodes.sender.token, nodes.target.alias)).target_busy).toBe(true);
+    db.run("UPDATE tasks SET status = 'cancelled' WHERE network_id = ?1 AND status = 'delivered'", [NET]);
+    expect((await dispatch(nodes.sender.token, nodes.target.alias)).target_busy).toBe(false);
+    // 真入口:节点 report_status(working, task=<原文>)→ 任务 running + 会话 working。
+    const c = await dispatch(bossToken, nodes.target.alias);
+    const content = db.get("SELECT content FROM tasks WHERE task_id = ?1", c.task_id).content;
+    expect((await reportStatus(nodes.target, "working", content)).ok).toBe(true);
+    expect(db.get("SELECT status FROM tasks WHERE task_id = ?1", c.task_id).status).toBe("running");
+    const r = await dispatch(nodes.sender.token, nodes.target.alias);
+    expect(r.target_busy).toBe(true);
+    expect(r.queue_ahead).toBe(2);
+    // 会话 working 但没有开着的任务 → 仍然 busy(节点自己说在忙)。
+    db.run("UPDATE tasks SET status = 'cancelled' WHERE network_id = ?1 AND status IN ('delivered','running')", [NET]);
+    const w = await dispatch(nodes.sender.token, nodes.target.alias);
+    expect(w).toMatchObject({ queue_ahead: 0, target_busy: true, est_wait_minutes: 0 });
+  });
+
+  test("est_wait_minutes: null without history, median × queue_ahead with it", async () => {
+    await dispatch(bossToken, nodes.target.alias);
+    const noHist = await dispatch(nodes.sender.token, nodes.target.alias);
+    expect(noHist.queue_ahead).toBe(1);
+    expect(noHist.est_wait_minutes).toBeNull();
+    // 两条历史还不够(< 3)。
+    await seedHistory(nodes.target, [10, 20]);
+    expect((await dispatch(nodes.sender.token, nodes.target.alias)).est_wait_minutes).toBeNull();
+    db.run("DELETE FROM tasks WHERE network_id = ?1 AND status = 'replied'", [NET]);
+    await seedHistory(nodes.target, [10, 20, 60]);
+    const r = await dispatch(nodes.sender.token, nodes.target.alias);
+    expect(r.queue_ahead).toBe(3);
+    expect(r.est_wait_minutes).toBe(60); // 中位数 20 × 3(平均数会是 30 × 3 = 90)
+    // 别的节点的历史不借用。
+    const idle = await dispatch(nodes.sender.token, nodes.idle.alias);
+    expect(idle.est_wait_minutes).toBe(0);
+    const idle2 = await dispatch(nodes.sender.token, nodes.idle.alias);
+    expect(idle2.est_wait_minutes).toBeNull();
+  });
+
+  test("warning at queue_ahead ≥ 3 or est_wait > 30 — the send still succeeds", async () => {
+    await dispatch(bossToken, nodes.target.alias);
+    await dispatch(bossToken, nodes.target.alias);
+    const two = await dispatch(nodes.sender.token, nodes.target.alias); // 前面 2 条
+    expect(two.queue_ahead).toBe(2);
+    expect(two.warning).toBeUndefined();
+    const three = await dispatch(nodes.sender.token, nodes.target.alias, { ttl_seconds: 1800 }); // 前面 3 条
+    expect(three.ok).toBe(true);
+    expect(three.queue_ahead).toBe(3);
+    expect(typeof three.warning).toBe("string");
+    for (const frag of [nodes.target.alias, "3 open task(s) ahead", "queued, not refused", "idle node", "ttl_seconds", "30 min", "Do not resend"]) {
+      expect(three.warning).toContain(frag);
+    }
+    // 任务确实写进去了(没被拒)。
+    expect(db.get("SELECT status FROM tasks WHERE task_id = ?1", three.task_id).status).toBe("delivered");
+    // MCP 同样带 warning。
+    const m = await tool(nodes.sender.token, "send_task", { alias: nodes.target.alias, task: text(), network_id: NET });
+    expect(m.ok).toBe(true);
+    expect(m.queue_ahead).toBe(4);
+    expect(m.warning).toContain("4 open task(s) ahead");
+
+    // 只排 1 个,但估计要等 > 30 分钟 → 也给。
+    db.run("UPDATE tasks SET status = 'cancelled' WHERE network_id = ?1 AND status = 'delivered'", [NET]);
+    await seedHistory(nodes.target, [40, 45, 50]);
+    await dispatch(bossToken, nodes.target.alias);
+    const slow = await dispatch(nodes.sender.token, nodes.target.alias, { ttl_seconds: 1800 });
+    expect(slow).toMatchObject({ queue_ahead: 1, est_wait_minutes: 45 });
+    expect(slow.warning).toContain("rough wait ~45 min");
+    expect(slow.warning).toContain("may expire before it starts");
+  });
+});
+
+describe("#500 queue_depth on status reads", () => {
+  const rowOf = (body: any, alias: string) => body.sessions.find((s: any) => s.alias === alias);
+
+  test("/api/status full rows carry queue_depth; light and the old alias-resolver projection do not", async () => {
+    await dispatch(bossToken, nodes.target.alias);
+    await dispatch(bossToken, nodes.target.alias);
+    const full = await send(bossToken, "GET", `/api/status?network_id=${NET}`);
+    expect(full.status).toBe(200);
+    expect(rowOf(full.body, nodes.target.alias).queue_depth).toBe(2);
+    expect(rowOf(full.body, nodes.idle.alias).queue_depth).toBe(0);
+    const light = await send(bossToken, "GET", `/api/status?network_id=${NET}&light=1`);
+    expect("queue_depth" in rowOf(light.body, nodes.target.alias)).toBe(false);
+    const resolver = await send(nodes.idle.token, "GET", `/api/status?network_id=${NET}`, undefined, { Accept: "application/json" });
+    expect(resolver.headers.get("x-status-projection")).toBe("alias-resolver");
+    expect("queue_depth" in rowOf(resolver.body, nodes.target.alias)).toBe(false);
+    // 另一个网络里的同名节点:看不到 NET 的队列。
+    const net2 = await send(boss2Token, "GET", `/api/status?network_id=${NET2}`);
+    expect(rowOf(net2.body, other.alias).queue_depth).toBe(0);
+    // App 的单行读(?alias=)只查这一个别名,数一样。
+    const one = await send(bossToken, "GET", `/api/status?network_id=${NET}&alias=${encodeURIComponent(nodes.target.alias)}`);
+    expect(one.body.sessions.length).toBe(1);
+    expect(one.body.sessions[0].queue_depth).toBe(2);
+  });
+
+  test("queueDepthByNode: the alias-list query and the whole-table scan (> 50 aliases) give the same counts", async () => {
+    const { queueDepthByNode, queueDepthKey, QUEUE_DEPTH_ALIAS_LIST_MAX } = await import("./task-queue-ahead.js");
+    await dispatch(bossToken, nodes.target.alias);
+    await dispatch(bossToken, nodes.target.alias);
+    await dispatch(bossToken, nodes.idle.alias);
+    await send(boss2Token, "POST", "/api/task", { alias: other.alias, task: text(), network_id: NET2 });
+    const few = queueDepthByNode([nodes.target.alias, nodes.idle.alias]);
+    const fillers = Array.from({ length: QUEUE_DEPTH_ALIAS_LIST_MAX }, (_, i) => `qa-filler-${i}`);
+    const many = queueDepthByNode([nodes.target.alias, nodes.idle.alias, ...fillers]);
+    for (const m of [few, many]) {
+      expect(m.get(queueDepthKey(NET, nodes.target.alias))).toBe(2);
+      expect(m.get(queueDepthKey(NET, nodes.idle.alias))).toBe(1);
+      expect(m.get(queueDepthKey(NET2, other.alias))).toBe(1); // 同名、别的网络:分开计
+    }
+    expect(queueDepthByNode([]).size).toBe(0);
+  });
+
+  test("a write that touches only tasks invalidates the cached full body (new ETag, new queue_depth); light stays cached", async () => {
+    const a = await dispatch(bossToken, nodes.target.alias);
+    await dispatch(bossToken, nodes.target.alias);
+    const first = await send(bossToken, "GET", `/api/status?network_id=${NET}`);
+    const etag = first.headers.get("etag")!;
+    expect(rowOf(first.body, nodes.target.alias).queue_depth).toBe(2);
+    const lightBefore = await send(bossToken, "GET", `/api/status?network_id=${NET}&light=1`);
+    // 只改 tasks(不碰 sessions):全量必须重算。
+    setStatus(a.task_id, "replied");
+    const again = await send(bossToken, "GET", `/api/status?network_id=${NET}`, undefined, { "If-None-Match": etag });
+    expect(again.status).toBe(200);
+    expect(again.headers.get("etag")).not.toBe(etag);
+    expect(rowOf(again.body, nodes.target.alias).queue_depth).toBe(1);
+    // light 不读 tasks:同一个 ETag → 304。
+    const lightAfter = await fetch(`${BASE}/api/status?network_id=${NET}&light=1`, { headers: { Authorization: `Bearer ${bossToken}`, "If-None-Match": lightBefore.headers.get("etag")! } });
+    expect(lightAfter.status).toBe(304);
+    // 没有写 → 全量命中缓存:同一个 ETag → 304。
+    const cached = await fetch(`${BASE}/api/status?network_id=${NET}`, { headers: { Authorization: `Bearer ${bossToken}`, "If-None-Match": again.headers.get("etag")! } });
+    expect(cached.status).toBe(304);
+  });
+
+  test("MCP get_all_status rows carry queue_depth", async () => {
+    await dispatch(bossToken, nodes.target.alias);
+    const r = await tool(bossToken, "get_all_status", { network_id: NET, filter_alias: `${nodes.target.alias},${nodes.idle.alias}` });
+    expect(r.ok).toBe(true);
+    expect(rowOf(r, nodes.target.alias).queue_depth).toBe(1);
+    expect(rowOf(r, nodes.idle.alias).queue_depth).toBe(0);
+    // filter_status + filter_alias on top of the network scope: every placeholder numbered (PG rejected the mix).
+    const f = await tool(bossToken, "get_all_status", { network_id: NET, filter_status: "idle", filter_alias: nodes.target.alias });
+    expect(f.ok).toBe(true);
+    expect(f.sessions.length).toBe(1);
+    expect(f.sessions[0].queue_depth).toBe(1);
+  });
+});

package/src/task-queue-ahead-scale.test.ts

@@ -0,0 +1,236 @@
+// #500 follow-up to #2325 —— queue_depth at production scale. SQLite only (the plan assertions are SQLite's
+// EXPLAIN QUERY PLAN; the result-equality side runs on PostgreSQL in task-queue-ahead-http.test.ts via test2123).
+//
+// The .100 release drill on a read-only copy of a production-shaped DB (306 sessions, 57k tasks, ~31k stuck in
+// `acked` for weeks, ~900 `running`, only ~1.9k created in the last 24 h) measured full /api/status at p50 66–77 ms
+// against 8–9 ms on .99: the queue_depth GROUP BY went through idx_tasks_status and walked every open-status row
+// ever written to keep a 24 h window. This file seeds that shape and pins:
+//   1. the plans: the whole-table queue_depth query and the per-send open-task count never use idx_tasks_status;
+//      the whole-table one is driven by the covering idx_tasks_created_queue (no task-row lookups), the alias list
+//      and the per-send count by idx_tasks_to_created;
+//   2. the cost, relative only (no absolute ms, so a slow runner can't fail it): the queue_depth statement is at most
+//      half the #2325 statement measured on the same DB in the same loop; and, on the low percentile of alternated
+//      samples (#517), the statement costs at most half a full /api/status read (cache bypassed) without it, and that
+//      read with queue_depth is at most 50 % slower than with it switched off (= the pre-#2325 read);
+//   3. the counts are still right at that scale (stale `acked` rows are outside the 24 h window).
+//
+// 跑法:cd server && COMMHUB_DB=/tmp/x.db bun test src/task-queue-ahead-scale.test.ts
+
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+const PG = !!process.env.COMMHUB_TEST_PG_URL;
+const DIR = mkdtempSync(join(tmpdir(), "anet-queue-scale-"));
+if (!PG) process.env.COMMHUB_DB ||= join(DIR, "hub.db");
+process.env.COMMHUB_UPLOADS_DIR = join(DIR, "uploads");
+process.env.HOST = "127.0.0.1";
+
+const NODES = 306;
+const STUCK_ACKED = 31_000;   // older than 24 h, never terminal
+const STUCK_RUNNING = 900;    // older than 24 h
+const OLD_DONE = 23_400;      // replied / failed / expired, older than 24 h
+const RECENT = 1_900;         // created in the last 24 h
+const OPEN_EVERY = 30;        // 3 in every OPEN_EVERY recent tasks are still open (~190 of 1.9k; see the cost test)
+const HOT = "scale-node-7";   // most stuck rows land on this one (the per-send plan must not care)
+const N = 25;
+/** #517: A/B pairs per pass, the percentile compared, and how many extra passes a failing pass gets. */
+const AB_PAIRS = 60, LOW_P = 0.2, RETRIES = 2;
+/** The #2325 statement (bare columns): the in-test baseline the fixed query is measured against. */
+const SQL_2325 = `SELECT network_id, to_name, COUNT(*) AS n FROM tasks
+  WHERE status IN ('created', 'delivered', 'acked', 'running') AND created_at >= datetime('now', '-86400 seconds')
+  GROUP BY network_id, to_name`;
+
+let db: any;
+let hub: any = null;
+let BASE = "", NET = "", token = "";
+let mod: any;
+let expectedOpen = new Map<string, number>();
+let seedMs = 0, regMs = 0;
+
+const pct = (xs: number[], p: number) => { const s = [...xs].sort((a, b) => a - b); return s[Math.min(s.length - 1, Math.floor(p * s.length))]; };
+const plan = (sql: string, params: any[]) => (db.all("EXPLAIN QUERY PLAN " + sql, ...params) as Array<{ detail: string }>).map((r) => r.detail).join(" | ");
+
+beforeAll(async () => {
+  if (PG) return;
+  ({ db } = await import("./db.js"));
+  mod = await import("./task-queue-ahead.js");
+  const { register } = await import("./auth.js");
+  const seedStart = performance.now();
+  const u = register(`scale_${Date.now()}`, "QueueScale123!x");
+  token = u.token!; NET = u.network_id!;
+  // Seed with a handful of set-based INSERT … SELECT over a recursive CTE (no per-row JS round trips): the whole
+  // 57k-row seed is ~0.2 s, so the file stays far inside test638's per-file budget even under strace and with two
+  // aggregates running at once. Rows are deterministic (functions of the row number), so the expected counts below
+  // are computed from the same formulas, independently of the code under test.
+  const SEQ = `WITH RECURSIVE seq(i) AS (SELECT 0 UNION ALL SELECT i + 1 FROM seq WHERE i + 1 < ?1)`;
+  const OLD_AGE = `(90000 + (i * 7919) % 2592000)`; // 25 h … 25 h + 30 d, in seconds
+  const insertTasks = (prefix: string, count: number, toExpr: string, statusExpr: string, ageExpr: string, doneExpr: string) => db.run(
+    `${SEQ}
+     INSERT INTO tasks (task_id, from_name, to_name, priority, status, content, created_at, delivered_at, completed_at, expires_at, network_id)
+     SELECT '${prefix}-' || i, 'scale-sender', ${toExpr}, 'normal', ${statusExpr}, 'c',
+            datetime('now', '-' || ${ageExpr} || ' seconds'), datetime('now', '-' || ${ageExpr} || ' seconds'),
+            CASE WHEN ${doneExpr} THEN datetime('now', '-' || (${ageExpr} - 120) || ' seconds') END,
+            datetime('now', '-' || (${ageExpr} - 3600) || ' seconds'), ?2
+       FROM seq`,
+    [count, NET],
+  );
+  const NODE_OF = (expr: string) => `'scale-node-' || ((${expr}) % ${NODES})`;
+  // Throwaway DB only: no fsync, and a page cache big enough that the seed transaction never spills to the WAL
+  // mid-way (test638 runs this file under `strace -f`, where every write syscall is expensive).
+  regMs = performance.now() - seedStart;
+  db.exec("PRAGMA synchronous = OFF; PRAGMA cache_size = -131072;");
+  db.transaction(() => {
+    db.run(`${SEQ}
+      INSERT INTO sessions (resume_id, alias, status, agent, network_id, node_id, updated_at, last_seen_at, task)
+      SELECT 'sdk-scale-' || i, 'scale-node-' || i, 'idle', 'agent-node:codex', ?2, 'n_scale_' || i, datetime('now'), datetime('now'), 'x' FROM seq`,
+      [NODES, NET]);
+    // Two thirds of the stuck rows on one node, the rest spread out.
+    insertTasks("acked", STUCK_ACKED, `CASE WHEN i % 3 = 0 THEN ${NODE_OF("i")} ELSE '${HOT}' END`, `'acked'`, OLD_AGE, "0");
+    insertTasks("running", STUCK_RUNNING, NODE_OF("i"), `'running'`, OLD_AGE, "0");
+    insertTasks("done", OLD_DONE, NODE_OF("i"), `CASE i % 5 WHEN 3 THEN 'failed' WHEN 4 THEN 'expired' ELSE 'replied' END`, OLD_AGE, "1");
+    // Last 24 h: i % OPEN_EVERY = 3 / 4 / 5 → delivered / running / acked (open), else failed (i % 7 = 6) or replied;
+    // age 0 … 23 h.
+    insertTasks("recent", RECENT, NODE_OF("i * 13"),
+      `CASE i % ${OPEN_EVERY} WHEN 3 THEN 'delivered' WHEN 4 THEN 'running' WHEN 5 THEN 'acked' ELSE CASE i % 7 WHEN 6 THEN 'failed' ELSE 'replied' END END`,
+      `((i * 37) % 82800)`, `i % ${OPEN_EVERY} NOT IN (3, 4, 5)`);
+  });
+  for (let i = 0; i < RECENT; i++) {
+    if (![3, 4, 5].includes(i % OPEN_EVERY)) continue;
+    const to = `scale-node-${(i * 13) % NODES}`;
+    expectedOpen.set(to, (expectedOpen.get(to) ?? 0) + 1);
+  }
+  seedMs = performance.now() - seedStart;
+  const srv: any = await import("./server.js");
+  hub = srv.bootServer({ port: 0, hostname: "127.0.0.1" });
+  BASE = `http://127.0.0.1:${hub.port}`;
+}, 120_000);
+
+afterAll(() => {
+  try { hub?.stop?.(true); } catch {}
+  try { rmSync(DIR, { recursive: true, force: true }); } catch {}
+});
+
+describe.skipIf(PG)("#500 queue_depth at production scale (57k tasks, 31k stuck acked)", () => {
+  test("plans: never idx_tasks_status; whole table via covering idx_tasks_created_queue, alias list and per-send via idx_tasks_to_created", () => {
+    const total = Number(db.get("SELECT COUNT(*) AS n FROM tasks WHERE network_id = ?1", NET).n);
+    expect(total).toBe(STUCK_ACKED + STUCK_RUNNING + OLD_DONE + RECENT);
+    const stale = Number(db.get("SELECT COUNT(*) AS n FROM tasks WHERE network_id = ?1 AND status IN ('acked','running') AND created_at < datetime('now', '-86400 seconds')", NET).n);
+    expect(stale).toBe(STUCK_ACKED + STUCK_RUNNING);
+    console.log(`[seed] ${total} tasks in ${seedMs.toFixed(0)} ms (register ${regMs.toFixed(0)} ms)`);
+    // Positive control: the shape is one where the #2325 statement really does go through idx_tasks_status.
+    expect(plan(SQL_2325, [])).toContain("idx_tasks_status");
+
+    const whole = mod.queueDepthQuery();
+    const wholePlan = plan(whole.sql, whole.params);
+    console.log(`[plan] queue_depth whole table: ${wholePlan}`);
+    expect(wholePlan).not.toContain("idx_tasks_status");
+    expect(wholePlan).toContain("COVERING INDEX idx_tasks_created_queue");
+
+    // > QUEUE_DEPTH_ALIAS_LIST_MAX aliases → the whole-table form too.
+    const many = mod.queueDepthQuery(Array.from({ length: NODES }, (_, i) => `scale-node-${i}`));
+    expect(many.sql).toBe(whole.sql);
+
+    const few = mod.queueDepthQuery([HOT, "scale-node-8"]);
+    const fewPlan = plan(few.sql, few.params);
+    console.log(`[plan] queue_depth alias list: ${fewPlan}`);
+    expect(fewPlan).not.toContain("idx_tasks_status");
+    expect(fewPlan).toContain("idx_tasks_to_created");
+
+    for (const net of [NET, null]) {
+      const send = mod.openTaskCountsQuery(HOT, net);
+      const sendPlan = plan(send.sql, send.params);
+      console.log(`[plan] per-send open count (net=${net ? "set" : "none"}): ${sendPlan}`);
+      expect(sendPlan).not.toContain("idx_tasks_status");
+      expect(sendPlan).toContain("idx_tasks_to_created");
+    }
+  });
+
+  test("counts stay right at scale: stale acked rows are outside the 24 h window", async () => {
+    const depth: Map<string, number> = mod.queueDepthByNode();
+    for (const [alias, n] of expectedOpen) expect(depth.get(mod.queueDepthKey(NET, alias))).toBe(n);
+    expect(depth.get(mod.queueDepthKey(NET, HOT)) ?? 0).toBe(expectedOpen.get(HOT) ?? 0);
+    const r = await fetch(`${BASE}/api/status?network_id=${NET}&alias=${HOT}`, { headers: { Authorization: `Bearer ${token}` } });
+    const body = await r.json() as any;
+    expect(body.sessions[0].queue_depth).toBe(expectedOpen.get(HOT) ?? 0);
+    expect(mod.openTaskCounts(HOT, NET).open).toBe(expectedOpen.get(HOT) ?? 0);
+    // And a node that does have open tasks in the window (HOT's open rows are all > 24 h old).
+    expect(expectedOpen.size).toBeGreaterThan(100);
+    const [busy, busyN] = [...expectedOpen].sort((a, b) => b[1] - a[1])[0];
+    expect(busyN).toBeGreaterThan(1);
+    const rb = await fetch(`${BASE}/api/status?network_id=${NET}&alias=${busy}`, { headers: { Authorization: `Bearer ${token}` } });
+    expect(((await rb.json()) as any).sessions[0].queue_depth).toBe(busyN);
+    expect(mod.openTaskCounts(busy, NET).open).toBe(busyN);
+  });
+
+  test("cost (relative, robust on slow runners): ≤ half the #2325 statement on the same DB, and ≤ +50% on a full /api/status read vs the same read without queue_depth", async () => {
+    // Interleave the two statements so a slow / noisy runner slows both alike.
+    const fixed: number[] = [], old: number[] = [];
+    for (let i = 0; i < N + 3; i++) {
+      let s0 = performance.now(); mod.queueDepthByNode(); const a = performance.now() - s0;
+      s0 = performance.now(); db.all(SQL_2325); const b = performance.now() - s0;
+      if (i >= 3) { fixed.push(a); old.push(b); }
+    }
+    // Measured locally (this fixture, covering index): ≈ 0.15× (1.0 vs 6.5 ms); on the production copy 0.3 vs 55 ms.
+    const qP50 = pct(fixed, 0.5), oldP50 = pct(old, 0.5);
+    console.log(`[perf] queue_depth p50=${qP50.toFixed(2)}ms vs #2325 statement p50=${oldP50.toFixed(2)}ms`);
+    expect(qP50).toBeLessThanOrEqual(0.5 * oldP50);
+
+    // Second budget: queue_depth adds at most +50 % to a full /api/status read (cache bypassed) — the same read
+    // with it switched off is the pre-#2325 read. #517: the first version compared the p50 of `with` and `without`
+    // on a ~4 ms read and asserted the *difference*; under test638's concurrent aggregates a difference of two
+    // noisy p50s is mostly noise (CI #2355: statement 0.64 ms, yet with−without = 2.4 ms > 2.03 ms budget). Now:
+    //   - each pass alternates A/B request by request (order flipped every pair) and also times the bare
+    //     queue_depth statement between the reads, so all three series share one time window;
+    //   - it compares a low percentile (LOW_P) rather than the p50: scheduler noise only ever adds time, so the
+    //     lower tail is the closest a busy runner gets to the real cost;
+    //   - it asserts both what the comment claims — the statement's own cost ≤ 50 % of the read without it — and
+    //     the end-to-end with/without delta on the same low percentile;
+    //   - a failing pass is re-measured up to RETRIES more times; the real regression (the #2325 statement through
+    //     idx_tasks_status, 5–8× the fixed cost) fails every pass, noise does not.
+    const cache = await import("./status-read-cache.js");
+    cache.__setStatusCacheBypassForTest(true);
+    const readOnce = async (depthOn: boolean) => {
+      mod.__setQueueDepthOffForTest(!depthOn);
+      const s0 = performance.now();
+      await (await fetch(`${BASE}/api/status?network_id=${NET}`, { headers: { Authorization: `Bearer ${token}` } })).text();
+      return performance.now() - s0;
+    };
+    const abPass = async () => {
+      const on: number[] = [], off: number[] = [], q: number[] = [];
+      for (let i = 0; i < AB_PAIRS + 3; i++) {
+        const first = i % 2 === 0;
+        const a = await readOnce(first), b = await readOnce(!first);
+        mod.__setQueueDepthOffForTest(false);
+        const s0 = performance.now(); mod.queueDepthByNode(); const c = performance.now() - s0;
+        if (i >= 3) { (first ? on : off).push(a); (first ? off : on).push(b); q.push(c); }
+      }
+      const onLow = pct(on, LOW_P), offLow = pct(off, LOW_P), qLow = pct(q, LOW_P);
+      return { onLow, offLow, qLow, onP50: pct(on, 0.5), offP50: pct(off, 0.5),
+        okDirect: qLow <= 0.5 * offLow, okRead: onLow - offLow <= 0.5 * offLow };
+    };
+    let r: Awaited<ReturnType<typeof abPass>> | null = null;
+    try {
+      for (let attempt = 0; attempt <= RETRIES; attempt++) {
+        r = await abPass();
+        console.log(`[perf] pass ${attempt + 1}: p${LOW_P * 100} queue_depth=${r.qLow.toFixed(2)}ms; full /api/status (bypass) p${LOW_P * 100} with=${r.onLow.toFixed(2)}ms without=${r.offLow.toFixed(2)}ms (+${(100 * (r.onLow - r.offLow) / r.offLow).toFixed(0)}%); p50 with=${r.onP50.toFixed(2)}ms without=${r.offP50.toFixed(2)}ms; direct ${r.okDirect ? "ok" : "OVER"} read ${r.okRead ? "ok" : "OVER"}`);
+        if (r.okDirect && r.okRead) break;
+      }
+    } finally {
+      mod.__setQueueDepthOffForTest(false);
+      cache.__setStatusCacheBypassForTest(false);
+    }
+    // What the read budget was measured at (2026-10-04, p50s, before #517):
+    //   production copy (VACUUM INTO of the live DB: 306 sessions, 57k tasks, 64 open in the last 24 h on 4 nodes),
+    //   full /api/status p50, cache bypassed, 3 runs each —
+    //     pre-#2325 10.5–11.3 ms · #2325 68–71 ms · first fix (idx_tasks_created + row lookups) 16.1–17.1 ms (+50 %,
+    //     statement 3.1 ms) · covering idx_tasks_created_queue 11.5–12.2 ms; this A/B on that copy: +7 / +8 / +9 %.
+    //   this fixture: +17–31 %. It is harsher than production: ~190 open tasks in the window spread over ~150 nodes
+    //   (3× production's open count, ~38× its groups) and one-word session rows that make the rest of the read cheap.
+    //   With the original 1-in-7 open share (813 open) it measured +55–62 % even with the covering index — the
+    //   GROUP BY over 300 groups, not the index.
+    expect(r!.qLow).toBeLessThanOrEqual(0.5 * r!.offLow);
+    expect(r!.onLow - r!.offLow).toBeLessThanOrEqual(0.5 * r!.offLow);
+  }, 120_000);
+});