agent-coord-mcp 0.26.34 → 0.26.36

package/src/tools/admin.ts

@@ -16,6 +16,7 @@ import {
   listCursorFiles,
   listInboxFiles,
   listReceiptFiles,
+  moveCursorOffset,
   normalizeRoom,
   pruneHistory,
   readJson,
@@ -23,6 +24,7 @@ import {
   rewriteJsonl,
   roomFile,
   updateJson,
+  type ArchiveResult,
   type RoomRegistry,
 } from "../store.js";
 import {
@@ -60,48 +62,59 @@ function keepEntry(
   return isDecision(e) ? e.ts > decisionCutoff : e.ts > cutoff;
 }
 
-// Shift every agent's cursor offsets down by the number of entries removed
-// ahead of them (append-only + oldest-first removal ⇒ plain subtraction,
-// clamped at 0). Shared by prune and live compaction — without this, an
-// offset past the new file length makes read_messages return [] forever.
-export async function adjustCursors(removals: {
-  roomRemovedByChan?: Record<string, number>;
-  statusRemoved?: number;
-  perAgentInboxRemoved?: Record<string, number>;
+/**
+ * Carry every agent's cursors across a rewrite of the streams they index. Shared by prune and
+ * live compaction — without this, an offset past the new file length makes read_messages return
+ * [] forever.
+ *
+ * ⛔ EACH CURSOR IS MOVED IN ITS OWN UNIT (⟨q-f44f17a7⟩). A READ cursor (`<agent>.json`) counts
+ * entries: it is shifted down by the entries removed (clamped at 0; at-least-once if kept
+ * decisions sat behind it). A PUSH cursor (`<agent>.push.json`) is a BYTE offset: it is mapped
+ * through the archive pass's `rebaseBytes`, exactly. This took three visits to one function:
+ * ⟨q-40449919⟩ fixed the push FILENAME and left the unit wrong, and ⟨q-c36a078f⟩ fixed the same
+ * confusion in push-cursor seeding. So the caller cannot pass a bare count any more — it passes the
+ * whole `ArchiveResult`, which carries both units, and the choice between them is made HERE by the
+ * cursor's own kind rather than trusted to every call site.
+ */
+export type StreamRewrite = Pick<ArchiveResult, "removed" | "rebaseBytes">;
+export async function adjustCursors(rewrites: {
+  roomByChan?: Record<string, StreamRewrite>;
+  status?: StreamRewrite;
+  inboxByAgent?: Record<string, StreamRewrite>;
 }): Promise<string[]> {
-  const roomRemovedByChan = removals.roomRemovedByChan ?? {};
-  const statusRemoved = removals.statusRemoved ?? 0;
-  const perAgentInboxRemoved = removals.perAgentInboxRemoved ?? {};
+  const roomByChan = rewrites.roomByChan ?? {};
+  const inboxByAgent = rewrites.inboxByAgent ?? {};
   const cursorsAdjusted: string[] = [];
   for (const cname of await listCursorFiles()) {
     // q-40449919: this was `cname.replace(/\.json$/, "")`, which strips only
     // the outer extension. A push cursor's `<agent>.push.json` then keyed
-    // `perAgentInboxRemoved` lookups below on `<agent>.push` — a key that
+    // `inboxByAgent` lookups below on `<agent>.push` — a key that
     // never matches — so a push cursor's inboxOffset was silently never
-    // decremented after a prune shrank the inbox, drifting it toward a
-    // false past-EOF the next time doctor ran.
-    const { id } = agentIdFromCursorFilename(cname);
+    // adjusted after a prune shrank the inbox.
+    const { id, kind } = agentIdFromCursorFilename(cname);
+    const move = (offset: number, rw: StreamRewrite) => moveCursorOffset(kind, offset, rw);
     const cursorPath = path.join(CURSOR_DIR, cname);
     let touched = false;
     await updateJson<Cursor>(cursorPath, {}, (current) => {
-      for (const [chan, removed] of Object.entries(roomRemovedByChan)) {
+      for (const [chan, rw] of Object.entries(roomByChan)) {
+        if (rw.removed <= 0) continue;
         if (chan === DEFAULT_ROOM) {
           if (current.roomOffset !== undefined) {
-            current.roomOffset = Math.max(0, current.roomOffset - removed);
+            current.roomOffset = move(current.roomOffset, rw);
             touched = true;
           }
         } else if (current.roomOffsets?.[chan] !== undefined) {
-          current.roomOffsets[chan] = Math.max(0, current.roomOffsets[chan] - removed);
+          current.roomOffsets[chan] = move(current.roomOffsets[chan], rw);
           touched = true;
         }
       }
-      if (current.statusOffset !== undefined && statusRemoved > 0) {
-        current.statusOffset = Math.max(0, current.statusOffset - statusRemoved);
+      if (current.statusOffset !== undefined && rewrites.status && rewrites.status.removed > 0) {
+        current.statusOffset = move(current.statusOffset, rewrites.status);
         touched = true;
       }
-      const myInboxRemoved = perAgentInboxRemoved[id] ?? 0;
-      if (current.inboxOffset !== undefined && myInboxRemoved > 0) {
-        current.inboxOffset = Math.max(0, current.inboxOffset - myInboxRemoved);
+      const myInbox = inboxByAgent[id];
+      if (current.inboxOffset !== undefined && myInbox && myInbox.removed > 0) {
+        current.inboxOffset = move(current.inboxOffset, myInbox);
         touched = true;
       }
       return current;
@@ -232,26 +245,26 @@ export async function pruneTool(args: {
   // Per-channel removed counts so we can shift the matching offset in each
   // cursor (default channel → roomOffset, others → roomOffsets[chan]).
   // Aged-out entries are archived to archive/rooms/<chan>.jsonl, never deleted.
-  const roomRemovedByChan: Record<string, number> = {};
+  const roomByChan: Record<string, StreamRewrite> = {};
   let roomRemovedTotal = 0;
   if (targets.has("rooms")) {
     for (const chan of channels) {
       const r = await archiveJsonl<Message>(roomFile(chan), archiveRoomFile(chan), keep);
       if (r.removed > 0) {
-        roomRemovedByChan[chan] = r.removed;
+        roomByChan[chan] = r;
         roomRemovedTotal += r.removed;
       }
     }
   }
   const statusResult = targets.has("status")
     ? await archiveJsonl<StatusEntry>(STATUS_FILE, ARCHIVE_STATUS_FILE, (e) => e.ts > cutoff)
-    : { kept: 0, removed: 0, archived: 0 };
+    : { kept: 0, removed: 0, archived: 0, rebaseBytes: () => 0 };
 
   let inboxRemoved = 0;
   const deletedOrphans: string[] = [];
-  // perAgentInboxRemoved: how many entries were stripped from each *kept* agent's
-  // inbox, so we can shift only that agent's inboxOffset cursor below.
-  const perAgentInboxRemoved: Record<string, number> = {};
+  // inboxByAgent: what was stripped from each *kept* agent's inbox (in both
+  // units), so we can shift only that agent's inboxOffset cursors below.
+  const inboxByAgent: Record<string, StreamRewrite> = {};
   if (targets.has("inbox")) {
     for (const fname of await listInboxFiles()) {
       const id = fname.replace(/\.jsonl$/, "");
@@ -266,14 +279,14 @@ export async function pruneTool(args: {
       }
       const r = await archiveJsonl<Message>(filePath, archiveInboxFile(id), (e) => e.ts > cutoff);
       inboxRemoved += r.removed;
-      perAgentInboxRemoved[id] = r.removed;
+      inboxByAgent[id] = r;
     }
   }
 
   const cursorsAdjusted = await adjustCursors({
-    roomRemovedByChan,
-    statusRemoved: statusResult.removed,
-    perAgentInboxRemoved,
+    roomByChan,
+    status: statusResult,
+    inboxByAgent,
   });
 
   // Membership sweep: drop orphans (not in registry) and stale members (no

package/src/tools/herdr-tail.ts

@@ -41,7 +41,7 @@ import { readFileSync, statSync } from "node:fs";
 import { inboxFile, roomFile, getRooms, transportFile } from "../store.js";
 import { activeTransport, HERDR, targetOf, type Transport, type TransportMarker } from "../transports/index.js";
 import { renderForPane, advancePushCursor, readPushCursorFor, type Message } from "./herdr-delivery.js";
-import { newMessagesIn } from "./jsonl-offsets.js";
+import { newMessagesIn, idEndingAt, endOfId } from "./jsonl-offsets.js";
 
 export const DEFAULT_POLL_MS = 1000;
 /** Marker and room membership are re-read this often. A room joined mid-session is picked up within this many ticks. */
@@ -61,6 +61,12 @@ export type TailContext = {
   rooms: string[];
   /** In-process high-water mark per source: never re-type below this, even if the persistent cursor could not be written. */
   hw: Map<string, number>;
+  /**
+   * The id of the message each `hw` mark was set just past (null: set with no message, e.g. an EOF
+   * seed of a file whose last line has no id). A byte mark means something only while THAT message
+   * still ends there — see the rewrite check in tailOnce.
+   */
+  hwId: Map<string, string | null>;
   /** Last seen size per source: a source that has not grown costs one stat. */
   sizes: Map<string, number>;
   /**
@@ -73,10 +79,17 @@ export type TailContext = {
   seedEof: boolean;
   /** Sources already seeded — a room discovered on a later refresh is seeded on ITS first sight, not skipped. */
   seeded: Set<string>;
+  /** Sources whose persisted cursor is past EOF right now — logged once on entry, not on every tick. */
+  pastEof: Set<string>;
 };
-export const newContext = (seedEof = false): TailContext => ({ tick: 0, marker: null, rooms: [], hw: new Map(), sizes: new Map(), seedEof, seeded: new Set() });
+export const newContext = (seedEof = false): TailContext => ({ tick: 0, marker: null, rooms: [], hw: new Map(), hwId: new Map(), sizes: new Map(), seedEof, seeded: new Set(), pastEof: new Set() });
 
 const keyOf = (s: { kind: string; chan?: string }) => (s.kind === "dm" ? "dm" : `room:${s.chan}`);
+/** Set the in-process mark just past `msg`, remembering which message it follows. */
+const mark = (ctx: TailContext, key: string, end: number, msg: Record<string, unknown>) => {
+  ctx.hw.set(key, end);
+  ctx.hwId.set(key, msg.id === undefined || msg.id === null ? null : String(msg.id));
+};
 const sizeOf = (file: string): number => {
   try { return statSync(file).size; } catch { return 0; }
 };
@@ -169,10 +182,34 @@ export async function tailOnce(
     if (ctx.seedEof && !ctx.seeded.has(key)) {
       ctx.seeded.add(key);
       ctx.hw.set(key, size);
+      ctx.hwId.set(key, idEndingAt(src.file, size));
       log(`[herdr-tail] seeded ${key} at EOF (${size} bytes) for ${agentId} — first sight this process, nothing predating the tail will be typed`);
       continue;
     }
+    // ⛔ A BYTE MARK IS ONLY VALID WHILE THE SAME MESSAGE STILL ENDS THERE (⟨q-f44f17a7⟩). Any sender's
+    // server compacts a room on send, and every byte after the cut then means something else. A mark
+    // left PAST the new EOF made the seat deaf until the room regrew past it (the 40-minute blind
+    // stretch of 2026-09-24); a mark left INSIDE the new file resumes mid-stream and skips whatever
+    // now sits below it. So the mark is checked against the message it was set after: moved to where
+    // that message now ends, or — if it was archived — dropped, and the PERSISTED cursor, which the
+    // compactor rebased exactly (adjustCursors → rebaseBytes), is the position again.
+    const held = ctx.hw.get(key);
+    if (held !== undefined && idEndingAt(src.file, held) !== (ctx.hwId.get(key) ?? null)) {
+      const id = ctx.hwId.get(key);
+      const moved = id ? endOfId(src.file, id) : null;
+      log(`[herdr-tail] ${key} for ${agentId} was rewritten beneath the in-process mark (byte ${held}, after ${id ?? "no id"}) — ${moved !== null ? `moved to byte ${moved}, where that message now ends` : "that message is gone; resuming from the rebased push cursor"}`);
+      if (moved !== null) ctx.hw.set(key, moved);
+      else { ctx.hw.delete(key); ctx.hwId.delete(key); }
+    }
     const from = await offsetOf(agentId, src, ctx);
+    // …and if the PERSISTED cursor is past EOF too, the compactor has not rebased it yet (it rewrites
+    // the file first, then the cursors) or a server without this fix compacted. `size <= from` below
+    // skips it either way; this only makes the state VISIBLE, once per entry — on 2026-09-24 the only
+    // measurements of this defect came from a seat reading the room file by hand.
+    if (from > size) {
+      if (!ctx.pastEof.has(key)) log(`[herdr-tail] ${key} push cursor for ${agentId} is past EOF (${from} > ${size} bytes) — nothing can be typed from there until it is rebased or the file regrows past it (q-f44f17a7)`);
+      ctx.pastEof.add(key);
+    } else ctx.pastEof.delete(key);
     // ⭐ NEGATIVE CONTROL FOR THE CLASS THIS ROW IS ABOUT: nothing is logged here when
     // `size === from` — a source with nothing owed must stay silent, or every idle room on
     // every tick becomes a line and the log that is supposed to make a real stall visible
@@ -180,7 +217,7 @@ export async function tailOnce(
     if (size <= from) continue;
 
     for (const { msg, end } of newMessagesIn(src.file, from)) {
-      if (msg.from === agentId) { ctx.hw.set(key, end); continue; }
+      if (msg.from === agentId) { mark(ctx, key, end, msg); continue; }
       const where = src.kind === "dm" ? ({ kind: "dm" } as const) : ({ kind: "room", chan: src.chan! } as const);
       const rendered = await renderForPane(msg as unknown as Message, agentId, where);
       let r: { delivered: boolean; error?: string; verified?: boolean; safeToRetry?: boolean; outcome?: string };
@@ -207,13 +244,13 @@ export async function tailOnce(
       // re-typing a message that may already be sitting in the box is the paste loop in another coat.
       // A throw lands here too: nothing says whether a key went out before it.
       if (!r.delivered || r.verified !== true) {
-        ctx.hw.set(key, end);
+        mark(ctx, key, end, msg);
         out.held.push({ kind: src.kind, chan: src.chan, id: String(msg.id ?? ""), why: `${r.outcome ?? (r.delivered ? "unverified" : "not delivered")} — ${r.error ?? "not verified"}; the persistent cursor was not moved (q-15d763dc)` });
         break;
       }
       // It landed and was verified. Record that in-process FIRST, so a cursor write that fails
       // cannot turn one delivery into a paste loop, then persist exactly past this line.
-      ctx.hw.set(key, end);
+      mark(ctx, key, end, msg);
       const persisted = await advancePushCursor(agentId, where, end);
       out.delivered.push({ kind: src.kind, chan: src.chan, id: String(msg.id ?? "") });
       if (!persisted) {

package/src/tools/jsonl-offsets.ts

@@ -27,3 +27,26 @@ export function newMessagesIn(file: string, from: number): OffsetMessage[] {
   }
   return out;
 }
+
+/**
+ * The id of the message whose line ends EXACTLY at byte `end`, or null (no such line, past EOF, not
+ * JSON, no id). What makes a byte offset checkable: a position is only still valid while the SAME
+ * message ends there — a compaction rewrites the file and every byte after the cut then means
+ * something else (⟨q-f44f17a7⟩).
+ */
+export function idEndingAt(file: string, end: number): string | null {
+  let buf: Buffer;
+  try { buf = readFileSync(file); } catch { return null; }
+  if (end <= 0 || end > buf.length || buf[end - 1] !== 0x0a) return null;
+  const start = end >= 2 ? buf.lastIndexOf(0x0a, end - 2) + 1 : 0;
+  try {
+    const id = (JSON.parse(buf.subarray(start, end - 1).toString("utf8")) as { id?: unknown }).id;
+    return id === undefined || id === null ? null : String(id);
+  } catch { return null; }
+}
+
+/** The byte just past the line of the message with this id, or null if the file no longer holds it. */
+export function endOfId(file: string, id: string): number | null {
+  for (const { msg, end } of newMessagesIn(file, 0)) if (String(msg.id ?? "") === id) return end;
+  return null;
+}

package/src/tools/messaging.ts

@@ -549,7 +549,7 @@ async function maybeCompactRoom(chan: string): Promise<void> {
     archiveRoomFile(chan),
     (e) => e.ts >= boundaryTs || (isDecision(e) && e.ts > decisionCutoff)
   );
-  if (r.removed > 0) await adjustCursors({ roomRemovedByChan: { [chan]: r.removed } });
+  if (r.removed > 0) await adjustCursors({ roomByChan: { [chan]: r } });
 }
 
 async function maybeCompactStatus(): Promise<void> {
@@ -562,7 +562,7 @@ async function maybeCompactStatus(): Promise<void> {
     ARCHIVE_STATUS_FILE,
     (e) => e.ts >= boundaryTs
   );
-  if (r.removed > 0) await adjustCursors({ statusRemoved: r.removed });
+  if (r.removed > 0) await adjustCursors({ status: r });
 }
 
 // ---------- read_messages ----------