@ccmsg/cli 0.14.1 → 0.15.0

package/src/transcript/files.ts

@@ -1,4 +1,4 @@
-import { readdirSync, type Stats, statSync } from "node:fs";
+import type { Stats } from "node:fs";
 import { readdir, readFile, stat } from "node:fs/promises";
 import { basename, dirname, join } from "node:path";
 import type { Sid, TranscriptSubject } from "@ccmsg/protocol";
@@ -95,16 +95,16 @@ export class TranscriptFiles {
    * greeted or is no longer running. Both stay inside this harness's own
    * transcript tree — the announced path because it was taken only if it was
    * inside it, the walk because that tree is what it walks (M6). */
-  path(sid: Sid): string | undefined {
+  async path(sid: Sid): Promise<string | undefined> {
     const announced = this.deps.announced(sid);
-    if (announced !== undefined && isFile(announced)) return announced;
-    return this.find(sid);
+    if (announced !== undefined && (await stated(announced)) !== undefined) return announced;
+    return await this.find(sid);
   }
 
   /** The session's own transcript, for an op that has nothing to answer
    * without one. */
-  session(sid: Sid): string {
-    const found = this.path(sid);
+  async session(sid: Sid): Promise<string> {
+    const found = await this.path(sid);
     if (found === undefined) throw new OpError("not_found", `no transcript is held for ${sid}`);
     return found;
   }
@@ -116,7 +116,7 @@ export class TranscriptFiles {
    * cannot be combined — a request carrying both names two files and is a
    * caller's mistake rather than a choice this makes for them. */
   async locate(sid: Sid, names: AgentNames = {}): Promise<string> {
-    const file = this.session(sid);
+    const file = await this.session(sid);
     if (names.agent_id !== undefined && names.teammate !== undefined) {
       throw new OpError("invalid_args", "agent_id and teammate name two different transcripts");
     }
@@ -238,21 +238,23 @@ export class TranscriptFiles {
    * which is one `stat` per directory instead of a listing; one whose name
    * carries more than the sid is walked, because the rest of the name is
    * exactly what this does not know. */
-  private find(sid: Sid): string | undefined {
+  private async find(sid: Sid): Promise<string | undefined> {
     const layout = LAYOUTS[this.deps.harness];
-    const dirs = directories(this.#root(), layout.depth);
+    const dirs = await walked(this.#root(), layout.depth);
     const nameOf = layout.nameOf;
     if (nameOf !== undefined) {
       if (!SID.test(sid)) return undefined;
       for (const dir of dirs) {
         const file = join(dir, nameOf(sid));
-        if (isFile(file)) return file;
+        if ((await stated(file)) !== undefined) return file;
       }
       return undefined;
     }
     for (const dir of dirs) {
-      for (const entry of names(dir)) {
-        if (layout.sidOf(entry) === sid && isFile(join(dir, entry))) return join(dir, entry);
+      for (const entry of await listed(dir)) {
+        if (layout.sidOf(entry) !== sid) continue;
+        const file = join(dir, entry);
+        if ((await stated(file)) !== undefined) return file;
       }
     }
     return undefined;
@@ -265,13 +267,9 @@ export class TranscriptFiles {
  * wrote, and a tree with a directory nobody expected is one whose files are
  * still found.
  *
- * There are two of this walk, and of the two readings below it, because the
- * two callers are not alike: `all()` runs from an op and reads the tree
- * without holding the instance, while `path()` also runs from a subscription
- * being opened, which states its value in the turn it is opened in and so
- * cannot wait (DESIGN §6.2). The one that cannot wait is the one CT-Q8 and the
- * issue `fold-from-head-with-versioned-cache` settle, and the two become one
- * reading again when they do. */
+ * One walk for every caller: enumerating the tree and finding one session's
+ * file in it are the same reading, and a subscription that opens on it waits
+ * for it like any other caller (CT-Q8). */
 async function walked(root: string, depth: number): Promise<string[]> {
   let level = [root];
   for (let step = 0; step < depth; step += 1) {
@@ -284,14 +282,6 @@ async function walked(root: string, depth: number): Promise<string[]> {
   return level;
 }
 
-function directories(root: string, depth: number): string[] {
-  let level = [root];
-  for (let step = 0; step < depth; step += 1) {
-    level = level.flatMap((dir) => names(dir).map((entry) => join(dir, entry)));
-  }
-  return level;
-}
-
 export interface AgentNames {
   readonly agent_id?: string;
   readonly run_id?: string;
@@ -343,14 +333,6 @@ async function listed(dir: string): Promise<string[]> {
   }
 }
 
-function names(dir: string): string[] {
-  try {
-    return readdirSync(dir);
-  } catch {
-    return [];
-  }
-}
-
 async function stated(file: string): Promise<Stats | undefined> {
   try {
     const known = await stat(file);
@@ -359,16 +341,3 @@ async function stated(file: string): Promise<Stats | undefined> {
     return undefined;
   }
 }
-
-function statOf(file: string) {
-  try {
-    const known = statSync(file);
-    return known.isFile() ? known : undefined;
-  } catch {
-    return undefined;
-  }
-}
-
-function isFile(file: string): boolean {
-  return statOf(file) !== undefined;
-}

package/src/transcript/fold.ts

@@ -61,6 +61,29 @@ export const NO_FACTS: TranscriptFacts = {
   agent_tree: { teammates: [], agents: [], workflows: [] },
 };
 
+/** Everything the fold carries from one record to the next, as data.
+ *
+ * The facts are what a consumer reads; this is what reading further needs. The
+ * two are not the same — a todo is held under the key a later record updates it
+ * by, a call is held until its result arrives — so a fold resumed from its
+ * facts alone would answer the next record differently from one that had read
+ * every record before it. What is kept here is the whole of that difference,
+ * which is what makes resuming from it the same reading rather than a similar
+ * one. */
+export interface FoldState {
+  readonly api_error?: SessionApiError;
+  readonly last_user_input_at?: Timestamp;
+  readonly model?: string;
+  readonly effort?: string;
+  readonly files: readonly (readonly [string, ExternalFile])[];
+  readonly todos: readonly (readonly [string, SessionTodo])[];
+  readonly teammates: readonly (readonly [string, Teammate])[];
+  readonly background: readonly (readonly [string, SessionBackgroundStatus])[];
+  readonly workflows: readonly (readonly [string, SessionWorkflowStatus])[];
+  readonly agents: readonly (readonly [string, AgentTreeNode])[];
+  readonly calls: readonly (readonly [string, PendingCall])[];
+}
+
 /** The one place a transcript line is interpreted.
  *
  * Nothing outside this module parses a transcript record. A line arrives, the
@@ -104,6 +127,43 @@ export class TranscriptFold {
     };
   }
 
+  /** Everything the reading carries forward, so that it can be taken up again
+   * where it stopped. */
+  get held(): FoldState {
+    return {
+      ...(this.#apiError === undefined ? {} : { api_error: this.#apiError }),
+      ...(this.#lastUserInputAt === undefined ? {} : { last_user_input_at: this.#lastUserInputAt }),
+      ...(this.#model === undefined ? {} : { model: this.#model }),
+      ...(this.#effort === undefined ? {} : { effort: this.#effort }),
+      files: [...this.#files],
+      todos: [...this.#todos],
+      teammates: [...this.#teammates],
+      background: [...this.#background],
+      workflows: [...this.#workflows],
+      agents: [...this.#agents],
+      calls: [...this.#calls],
+    };
+  }
+
+  /** Take up a reading somebody else left off, replacing whatever this fold
+   * held. The state is trusted as the record of records already read — what
+   * makes it trustworthy is decided where it is kept (`FOLD_CACHE_VERSION`),
+   * not here. */
+  restore(state: FoldState): void {
+    this.reset();
+    this.#apiError = state.api_error;
+    this.#lastUserInputAt = state.last_user_input_at;
+    this.#model = state.model;
+    this.#effort = state.effort;
+    for (const [key, value] of state.files) this.#files.set(key, value);
+    for (const [key, value] of state.todos) this.#todos.set(key, value);
+    for (const [key, value] of state.teammates) this.#teammates.set(key, value);
+    for (const [key, value] of state.background) this.#background.set(key, { ...value });
+    for (const [key, value] of state.workflows) this.#workflows.set(key, { ...value });
+    for (const [key, value] of state.agents) this.#agents.set(key, { ...value });
+    for (const [key, value] of state.calls) this.#calls.set(key, value);
+  }
+
   reset(): void {
     this.#apiError = undefined;
     this.#lastUserInputAt = undefined;
@@ -648,13 +708,13 @@ type Mutable<T> = { -readonly [K in keyof T]: T[K] };
 
 /** A teammate as the fold holds it: what the contract states about it, and the
  * two facts only the tree needs. */
-interface Teammate {
+export interface Teammate {
   readonly status: Mutable<SessionTeammate>;
   agent_id?: string;
   team_name?: string;
 }
 
-interface PendingCall {
+export interface PendingCall {
   readonly name: string;
   readonly input: Record<string, unknown>;
   readonly at?: Timestamp;

package/src/transcript/index.ts

@@ -4,7 +4,9 @@ export {
   TranscriptFiles,
   type TranscriptFilesDeps,
 } from "./files.ts";
+export { FOLD_CACHE_VERSION, FoldCache, type FoldCacheEntry } from "./cache.ts";
 export {
+  type FoldState,
   NO_FACTS,
   readRecord,
   type TranscriptFacts,
@@ -12,5 +14,5 @@ export {
   type TranscriptRecord,
 } from "./fold.ts";
 export { READ_LIMIT, readSlice } from "./read.ts";
-export { type Appended, FOLD_TAIL_BYTES, TranscriptTail } from "./tail.ts";
+export { type Appended, READ_CHUNK_BYTES, TranscriptTail } from "./tail.ts";
 export { ITEMS_SNAPSHOT, Transcripts, type TranscriptsDeps } from "./transcripts.ts";

package/src/transcript/items/classify.ts

@@ -97,6 +97,27 @@ type Draft = Record<string, unknown> & {
   at: number;
 };
 
+/** Everything a reading carries from one record to the next, as data.
+ *
+ * The items already handed over are not here — a reader holds those — but what
+ * decides how the next record reads is: which turn the file is in, which slash
+ * command output belongs to, whose file it is, and the calls still waiting for
+ * an answer. A reading taken up from this answers the next record exactly as
+ * the reading that produced it would have.
+ *
+ * The outstanding calls name their items by id rather than carrying them: the
+ * items are what a reader already holds, and naming them is what lets a result
+ * point back at the call in the very list the reader has. */
+export interface ClassificationState {
+  readonly turn: number;
+  readonly slash?: string;
+  readonly subject: TranscriptSubject;
+  readonly calls: readonly (readonly [
+    string,
+    { readonly tool: string; readonly message?: string; readonly name: string },
+  ])[];
+}
+
 /** A whole transcript read as items, in the order the file holds them.
  *
  * The file is read through once and the links are filled in as the answers
@@ -144,6 +165,62 @@ export class Classification {
     this.#subject = subject;
   }
 
+  /** What this reading would carry into the next record. */
+  get held(): ClassificationState {
+    const calls: ClassificationState["calls"] = [...this.#calls].map(([id, call]) => [
+      id,
+      {
+        tool: call.tool.id,
+        name: call.name,
+        ...optional("message", call.message?.id),
+      },
+    ]);
+    return {
+      turn: this.#turn,
+      subject: this.#subject,
+      calls,
+      ...optional("slash", this.#slash),
+    };
+  }
+
+  /** Take up a reading somebody else left off.
+   *
+   * `known` is the items that reading handed over and this one still holds, so
+   * an outstanding call is matched back to the item a reader has rather than to
+   * a copy of it: what the answer writes into the call then lands where the
+   * reader will look for it. A call whose item the reader no longer holds is
+   * kept by name and id alone — the result still says which tool answered and
+   * which item it belongs to, and there is no item left to write into. */
+  restore(state: ClassificationState, known: Iterable<Item>): void {
+    this.#items = [];
+    this.#calls.clear();
+    this.#turn = state.turn;
+    this.#subject = state.subject;
+    this.#slash = state.slash;
+    const held = new Map<string, Draft>();
+    for (const item of known) {
+      const draft = item as unknown as Draft;
+      if (typeof draft.id === "string") held.set(draft.id, draft);
+    }
+    const detached = (id: string): Draft =>
+      held.get(id) ??
+      ({
+        id,
+        uuid: "",
+        subject: this.#subject,
+        source: { offset: 0, bytes: 0 },
+        type: "",
+        at: 0,
+      } as Draft);
+    for (const [id, call] of state.calls) {
+      this.#calls.set(id, {
+        tool: detached(call.tool),
+        name: call.name,
+        ...optional("message", call.message === undefined ? undefined : detached(call.message)),
+      });
+    }
+  }
+
   /** The records of one chunk as the items they were read as, oldest first.
    *
    * What comes back is this chunk's items alone. A call the chunk before it

package/src/transcript/items/index.ts

@@ -1,4 +1,4 @@
-export { classify, Classification } from "./classify.ts";
+export { classify, Classification, type ClassificationState } from "./classify.ts";
 export { document, type DumpView } from "./document.ts";
 export { fields, type Item } from "./item.ts";
 export { ledger } from "./ids.ts";

package/src/transcript/tail.ts

@@ -1,22 +1,19 @@
-import { closeSync, type FSWatcher, openSync, readSync, statSync, watch } from "node:fs";
+import { type FSWatcher, watch } from "node:fs";
 import { open, stat } from "node:fs/promises";
 import { CONFIRM_POLL_MS } from "../sessions/harness.ts";
+import { breathe } from "./scan.ts";
 
-/** How much of an existing transcript is read when a tail starts.
+/** How much of a transcript one read takes at a time.
  *
- * The fold's two values both describe the present — the error the latest turn
- * ended on, and the last time a person spoke — so what a tail needs on opening
- * is the recent end of the file, not its history. A megabyte is a few hundred
- * records at the sizes the harness writes, which reaches back past the current
- * turn by a wide margin while costing one read of fixed size however large the
- * file has grown (DESIGN §2.3: a transcript of any size is read from its end).
- *
- * A person who has not spoken within it is reported as having no known input
- * rather than as having spoken long ago, which is what the contract's absent
- * `last_user_input_at` already means. */
-export const FOLD_TAIL_BYTES = 1024 * 1024;
-
-/** What the tail found appended, with the offsets that place it.
+ * The reading of a whole file is cut into reads of this size and the loop is
+ * handed back between them, so following a transcript of any size costs the
+ * instance a read of fixed size rather than a pause proportional to the file.
+ * A megabyte is a few hundred records at the sizes the harness writes, which
+ * is large enough that the turns themselves cost nothing measurable against
+ * the parsing they carry. */
+export const READ_CHUNK_BYTES = 1024 * 1024;
+
+/** What the tail read, with the offsets that place it.
  *
  * The offsets are the ones a transcript read pages by, so what arrives live
  * and what was read stitch together without reading anything twice. `end` is
@@ -32,10 +29,10 @@ export interface Appended {
 export interface TailDeps {
   /** Complete lines only; a record still being written waits for its end. */
   readonly onAppended: (appended: Appended) => void;
-  /** The end of the file as it stood when the tail opened, oldest first, with
-   * the offsets that place it. The seed of what is folded and of what is
+  /** What was already in the file when the tail opened, oldest first, in the
+   * reads it arrived in. The input of what is folded and of what is
    * classified, not something a subscriber is sent. */
-  readonly onSeed: (seeded: Appended) => void;
+  readonly onExisting: (read: Appended) => void;
   /** The file is not the one the tail was reading: it shrank, so what was
    * folded out of the old contents no longer describes it. */
   readonly onTruncated: () => void;
@@ -63,17 +60,7 @@ export class TranscriptTail {
   constructor(
     private readonly path: string,
     private readonly deps: TailDeps,
-  ) {
-    // Where the file ends, read before anything can ask. A subscription's
-    // snapshot states this and is answered in the same turn the tail is
-    // created, so a size that only the awaited seed had filled in would be
-    // reported as zero and every byte already written would look appended.
-    // Reading it here also fixes what the seed reads: the seed takes this size
-    // rather than stating a newer one, so nothing lands between the size the
-    // subscriber was given and the first frame it is sent.
-    this.#size = sizeNow(path);
-    this.#offset = this.#size;
-  }
+  ) {}
 
   /** The transcript's size as last observed, which is what a subscription's
    * snapshot states and where the frames after it begin. */
@@ -81,21 +68,29 @@ export class TranscriptTail {
     return this.#size;
   }
 
+  /** Just past the last record read, which is what a reading resumed later
+   * starts from. */
+  get offset(): number {
+    return this.#offset;
+  }
+
   get running(): boolean {
     return this.#watcher !== undefined || this.#timer !== undefined;
   }
 
-  /** Begin following, seeding the fold from the end of what is already there.
+  /** Read what is already there, then begin following.
+   *
+   * `from` is where a reading of this same file left off — nothing for a file
+   * being read for the first time, which is read from its beginning. Everything
+   * before the tail's own first append is therefore accounted for, whether it
+   * was read now or read once before and remembered.
    *
-   * The seed is read before this returns rather than awaited, for the reason
-   * the size is read in the constructor: a subscription's snapshot is answered
-   * in the same turn the tail is started, and what the seed settles — the
-   * fold's values, and the items a subscriber opens on — would otherwise be
-   * stated as empty and the whole existing end of the file would arrive later
-   * as though it had just been appended. */
-  async start(): Promise<void> {
+   * This is awaited by whoever states a value derived from the file, so a
+   * subscriber is told what the whole transcript says rather than what its end
+   * says (CT-Q8). */
+  async start(from = 0): Promise<void> {
     if (this.running) return;
-    this.#seed();
+    await this.#catchUp(from);
     try {
       this.#watcher = watch(this.path, () => void this.refresh());
     } catch {
@@ -121,20 +116,17 @@ export class TranscriptTail {
     return this.#reading;
   }
 
-  #seed(): void {
-    const size = this.#size;
-    if (size === 0) return;
-    const from = Math.max(0, size - FOLD_TAIL_BYTES);
-    const complete = whole(this.#sliceSync(from, size));
-    this.#offset = from + complete.byteLength;
-    // The first line is half a record whenever the read began mid-file, so it
-    // is dropped: what is read are whole records or nothing.
-    const at = from === 0 ? 0 : complete.indexOf(NEWLINE) + 1;
-    this.deps.onSeed({ lines: split(complete, at), start: from + at, end: this.#offset, size });
+  /** The file as it already stands, read to the end it had when this began.
+   * What is written while it runs is left to the first append, so the size a
+   * subscription opens on and the first frame after it meet exactly. */
+  async #catchUp(from: number): Promise<void> {
+    this.#offset = from;
+    this.#size = await this.#stat(from);
+    await this.#consume(this.#size, this.deps.onExisting);
   }
 
   async #read(): Promise<void> {
-    const size = await this.#stat();
+    const size = await this.#stat(this.#offset);
     if (size < this.#offset) {
       // Shorter than what was already consumed: the file was replaced, so what
       // was folded out of it describes nothing, and reading resumes from its
@@ -143,21 +135,36 @@ export class TranscriptTail {
       this.deps.onTruncated();
     }
     this.#size = size;
-    if (size === this.#offset) return;
-    const complete = whole(await this.#slice(this.#offset, size));
-    if (complete.byteLength === 0) return;
-    const start = this.#offset;
-    const end = start + complete.byteLength;
-    this.#offset = end;
-    this.deps.onAppended({ lines: split(complete, 0), start, end, size });
+    await this.#consume(size, this.deps.onAppended);
   }
 
-  async #stat(): Promise<number> {
+  /** Everything up to `size`, in reads of a fixed size with the loop handed
+   * back between them. A record still being written ends the pass: it is left
+   * for the read that finds its end. */
+  async #consume(size: number, state: (read: Appended) => void): Promise<void> {
+    while (this.#offset < size) {
+      // A record longer than one read is taken in one piece rather than in
+      // halves, so the window grows until it holds a whole one.
+      let to = Math.min(this.#offset + READ_CHUNK_BYTES, size);
+      let complete = whole(await this.#slice(this.#offset, to));
+      while (complete.byteLength === 0 && to < size) {
+        to = Math.min(to + READ_CHUNK_BYTES, size);
+        complete = whole(await this.#slice(this.#offset, to));
+      }
+      if (complete.byteLength === 0) return;
+      const start = this.#offset;
+      this.#offset = start + complete.byteLength;
+      state({ lines: split(complete), start, end: this.#offset, size });
+      if (this.#offset < size) await breathe();
+    }
+  }
+
+  async #stat(fallback: number): Promise<number> {
     try {
       return (await stat(this.path)).size;
     } catch {
       // Not there. Nothing was appended, and the poll keeps looking.
-      return this.#offset;
+      return fallback;
     }
   }
 
@@ -169,26 +176,6 @@ export class TranscriptTail {
    * arithmetic puts it, inside a character as readily as before one, and
    * decoding first would turn those bytes into a replacement character of a
    * different length and move every offset derived from it. */
-  /** The same range, read without yielding, which is what the seed is read
-   * through: the turn that starts a tail is the turn that answers a
-   * subscription, and it has to hold the end of the file by then. Bounded by
-   * `FOLD_TAIL_BYTES` however large the transcript is. */
-  #sliceSync(from: number, to: number): Buffer {
-    if (to <= from) return Buffer.alloc(0);
-    let handle: number;
-    try {
-      handle = openSync(this.path, "r");
-    } catch {
-      return Buffer.alloc(0);
-    }
-    try {
-      const buffer = Buffer.alloc(to - from);
-      return buffer.subarray(0, readSync(handle, buffer, 0, buffer.length, from));
-    } finally {
-      closeSync(handle);
-    }
-  }
-
   async #slice(from: number, to: number): Promise<Buffer> {
     if (to <= from) return Buffer.alloc(0);
     const handle = await open(this.path, "r").catch(() => undefined);
@@ -213,19 +200,13 @@ function whole(bytes: Buffer): Buffer {
   return last < 0 ? Buffer.alloc(0) : bytes.subarray(0, last + 1);
 }
 
-/** The records in what was read, from a byte that begins one. */
-function split(complete: Buffer, at: number): string[] {
+/** The records in what was read, which begins at a record. */
+function split(complete: Buffer): string[] {
   const lines: string[] = [];
-  for (let from = at; from < complete.byteLength;) {
+  for (let from = 0; from < complete.byteLength;) {
     const newline = complete.indexOf(NEWLINE, from);
     lines.push(complete.toString("utf8", from, newline));
     from = newline + 1;
   }
   return lines;
 }
-
-/** How large the file is right now, or zero for one that is not there yet.
- * Synchronous because the value is wanted before the first await. */
-function sizeNow(path: string): number {
-  return statSync(path, { throwIfNoEntry: false })?.size ?? 0;
-}