@ccmsg/cli 0.14.0 → 0.15.0

package/src/transcript/cache.ts

@@ -0,0 +1,208 @@
+import { createHash } from "node:crypto";
+import { mkdir, rename, stat, unlink, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import type { ClassificationState, Item } from "./items/index.ts";
+import type { FoldState } from "./fold.ts";
+
+/** Which shape of folded state this build can read back.
+ *
+ * A cache entry is the answer a past run derived, taken on trust: nothing
+ * re-reads the bytes it was derived from. So it may only be read back by a
+ * build that would have derived the same answer from them, and the version is
+ * how that is asserted — raise it whenever what the fold keeps or how it reads
+ * a record changes, and every entry written before says nothing to this build.
+ *
+ * `test/transcript.test.ts` holds the digest of the sources this number stands
+ * for and fails when they move without it, so the assertion is checked rather
+ * than remembered. */
+export const FOLD_CACHE_VERSION = 2;
+
+/** What one session's fold had reached, as it is written down.
+ *
+ * The offset is what makes the rest of it usable: it says which bytes the
+ * state accounts for, so the next run reads from there instead of from the
+ * beginning. The file's identity is beside it because the offset counts bytes
+ * of one file — a transcript replaced by another of the same name has bytes
+ * this state describes none of. */
+export interface FoldCacheEntry {
+  readonly version: number;
+  readonly path: string;
+  readonly dev: number;
+  readonly ino: number;
+  /** Just past the last record the state accounts for. */
+  readonly offset: number;
+  readonly fold: FoldState;
+  /** What the reading that produced the items would carry into the next record:
+   * the turn it had reached, whose file it decided this is, and the calls still
+   * waiting for an answer. Without it a resumed run would answer a record
+   * differently from the run that read everything before it — counting turns
+   * from zero again, and calling a result whose call is known here the reserved
+   * name for a call nobody saw. */
+  readonly reading: ClassificationState;
+  /** The end of the reading, which is what a subscription opens on. Kept with
+   * the fold because both are derived from the same pass, and a resumed run
+   * that held only the fold would open the items topic on an empty list while
+   * claiming to know the session. */
+  readonly items: readonly Item[];
+}
+
+/** Where folded transcripts are kept between runs.
+ *
+ * Everything here can be derived again from the transcript it came from, so a
+ * miss, a discarded entry and an emptied directory are all the same thing: the
+ * file is read from its beginning. Nothing asks whether a write succeeded for
+ * that reason — a cache that could not be written costs the next run one read.
+ *
+ * Writes are chained rather than overlapped, so two saves of the same session
+ * cannot race over the temporary file they rename from. */
+export class FoldCache {
+  #writing: Promise<void> = Promise.resolve();
+
+  constructor(private readonly dir: string) {}
+
+  /** What was folded out of this file, or nothing when what is on disk does not
+   * describe the file as it stands. */
+  async read(path: string): Promise<FoldCacheEntry | undefined> {
+    let entry: FoldCacheEntry;
+    try {
+      entry = JSON.parse(await Bun.file(this.#fileFor(path)).text()) as FoldCacheEntry;
+    } catch {
+      return undefined;
+    }
+    // The shape is checked and not assumed. An entry whose version matches but
+    // whose fields are not what this build reads back would otherwise be taken
+    // apart by whoever reads it, and a reading that throws leaves the session
+    // unopenable until the instance restarts — where a file that says nothing
+    // this build can use costs one reading (DR-0015 §2.5: what an await brings
+    // back is an input, not a promise kept).
+    if (!describes(entry, path)) return undefined;
+    let known: Awaited<ReturnType<typeof stat>>;
+    try {
+      known = await stat(path);
+    } catch {
+      return undefined;
+    }
+    // A different file under the same name, or the same file grown shorter than
+    // the bytes the state accounts for: either way the state describes bytes
+    // that are not there, and the file is read from its beginning.
+    if (known.dev !== entry.dev || known.ino !== entry.ino) return undefined;
+    if (known.size < entry.offset) return undefined;
+    return entry;
+  }
+
+  /** Keep what has been folded so far. The file's identity is read here rather
+   * than taken from the caller, so what is written describes the file the
+   * offset was actually counted in. */
+  save(
+    path: string,
+    offset: number,
+    fold: FoldState,
+    reading: ClassificationState,
+    items: readonly Item[],
+  ): Promise<void> {
+    // The items are taken now rather than when the write runs: the offset and
+    // the fold describe this moment, and a list still being appended to would
+    // put records past the offset into an entry that claims to end at it.
+    const held = [...items];
+    this.#writing = this.#writing.then(() => this.#write(path, offset, fold, reading, held));
+    return this.#writing;
+  }
+
+  /** Forget what was folded out of this file, for a transcript that turned out
+   * to be another one. */
+  drop(path: string): Promise<void> {
+    this.#writing = this.#writing.then(async () => {
+      await unlink(this.#fileFor(path)).catch(() => undefined);
+    });
+    return this.#writing;
+  }
+
+  async #write(
+    path: string,
+    offset: number,
+    fold: FoldState,
+    reading: ClassificationState,
+    items: readonly Item[],
+  ): Promise<void> {
+    let known: Awaited<ReturnType<typeof stat>>;
+    try {
+      known = await stat(path);
+    } catch {
+      return;
+    }
+    const entry: FoldCacheEntry = {
+      version: FOLD_CACHE_VERSION,
+      path,
+      dev: known.dev,
+      ino: known.ino,
+      offset,
+      fold,
+      reading,
+      items: [...items],
+    };
+    const file = this.#fileFor(path);
+    const temporary = `${file}.${process.pid}.tmp`;
+    try {
+      await mkdir(this.dir, { recursive: true });
+      await writeFile(temporary, JSON.stringify(entry));
+      await rename(temporary, file);
+    } catch {
+      await unlink(temporary).catch(() => undefined);
+    }
+  }
+
+  /** One file per transcript, named by a digest of its path: the path is what
+   * identifies the transcript, and a digest of it is a name every filesystem
+   * takes however the path was spelled. */
+  #fileFor(path: string): string {
+    return join(this.dir, `${createHash("sha256").update(path).digest("hex").slice(0, 32)}.json`);
+  }
+}
+
+/** Whether what was read back is an entry this build can take up.
+ *
+ * Everything the fold and the reading are restored from is checked, because
+ * restoring walks it: a field of the wrong shape is a file that says nothing
+ * this build can use, which is the same as no file at all. What is not checked
+ * is what nothing walks — the contents of an item, of a call's arguments, of a
+ * todo — since those are carried whole and stated as they were written. */
+function describes(entry: unknown, path: string): entry is FoldCacheEntry {
+  if (!isObject(entry)) return false;
+  if (entry["version"] !== FOLD_CACHE_VERSION || entry["path"] !== path) return false;
+  if (!counted(entry["offset"]) || !counted(entry["dev"]) || !counted(entry["ino"])) return false;
+  return states(entry["fold"]) && reads(entry["reading"]) && Array.isArray(entry["items"]);
+}
+
+function states(fold: unknown): boolean {
+  if (!isObject(fold)) return false;
+  for (const key of ["files", "todos", "teammates", "background", "workflows", "agents", "calls"]) {
+    const held = fold[key];
+    if (!Array.isArray(held)) return false;
+    for (const pair of held) {
+      if (!Array.isArray(pair) || pair.length !== 2 || typeof pair[0] !== "string") return false;
+    }
+  }
+  return true;
+}
+
+function reads(reading: unknown): boolean {
+  if (!isObject(reading)) return false;
+  if (!counted(reading["turn"]) || typeof reading["subject"] !== "string") return false;
+  if (!Array.isArray(reading["calls"])) return false;
+  for (const pair of reading["calls"]) {
+    if (!Array.isArray(pair) || pair.length !== 2 || typeof pair[0] !== "string") return false;
+    const call: unknown = pair[1];
+    if (!isObject(call) || typeof call["tool"] !== "string" || typeof call["name"] !== "string") {
+      return false;
+    }
+  }
+  return true;
+}
+
+function isObject(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function counted(value: unknown): value is number {
+  return typeof value === "number" && Number.isInteger(value) && value >= 0;
+}

package/src/transcript/files.ts

@@ -1,4 +1,4 @@
-import { readdirSync, type Stats, statSync } from "node:fs";
+import type { Stats } from "node:fs";
 import { readdir, readFile, stat } from "node:fs/promises";
 import { basename, dirname, join } from "node:path";
 import type { Sid, TranscriptSubject } from "@ccmsg/protocol";
@@ -95,16 +95,16 @@ export class TranscriptFiles {
    * greeted or is no longer running. Both stay inside this harness's own
    * transcript tree — the announced path because it was taken only if it was
    * inside it, the walk because that tree is what it walks (M6). */
-  path(sid: Sid): string | undefined {
+  async path(sid: Sid): Promise<string | undefined> {
     const announced = this.deps.announced(sid);
-    if (announced !== undefined && isFile(announced)) return announced;
-    return this.find(sid);
+    if (announced !== undefined && (await stated(announced)) !== undefined) return announced;
+    return await this.find(sid);
   }
 
   /** The session's own transcript, for an op that has nothing to answer
    * without one. */
-  session(sid: Sid): string {
-    const found = this.path(sid);
+  async session(sid: Sid): Promise<string> {
+    const found = await this.path(sid);
     if (found === undefined) throw new OpError("not_found", `no transcript is held for ${sid}`);
     return found;
   }
@@ -116,7 +116,7 @@ export class TranscriptFiles {
    * cannot be combined — a request carrying both names two files and is a
    * caller's mistake rather than a choice this makes for them. */
   async locate(sid: Sid, names: AgentNames = {}): Promise<string> {
-    const file = this.session(sid);
+    const file = await this.session(sid);
     if (names.agent_id !== undefined && names.teammate !== undefined) {
       throw new OpError("invalid_args", "agent_id and teammate name two different transcripts");
     }
@@ -238,21 +238,23 @@ export class TranscriptFiles {
    * which is one `stat` per directory instead of a listing; one whose name
    * carries more than the sid is walked, because the rest of the name is
    * exactly what this does not know. */
-  private find(sid: Sid): string | undefined {
+  private async find(sid: Sid): Promise<string | undefined> {
     const layout = LAYOUTS[this.deps.harness];
-    const dirs = directories(this.#root(), layout.depth);
+    const dirs = await walked(this.#root(), layout.depth);
     const nameOf = layout.nameOf;
     if (nameOf !== undefined) {
       if (!SID.test(sid)) return undefined;
       for (const dir of dirs) {
         const file = join(dir, nameOf(sid));
-        if (isFile(file)) return file;
+        if ((await stated(file)) !== undefined) return file;
       }
       return undefined;
     }
     for (const dir of dirs) {
-      for (const entry of names(dir)) {
-        if (layout.sidOf(entry) === sid && isFile(join(dir, entry))) return join(dir, entry);
+      for (const entry of await listed(dir)) {
+        if (layout.sidOf(entry) !== sid) continue;
+        const file = join(dir, entry);
+        if ((await stated(file)) !== undefined) return file;
       }
     }
     return undefined;
@@ -265,13 +267,9 @@ export class TranscriptFiles {
  * wrote, and a tree with a directory nobody expected is one whose files are
  * still found.
  *
- * There are two of this walk, and of the two readings below it, because the
- * two callers are not alike: `all()` runs from an op and reads the tree
- * without holding the instance, while `path()` also runs from a subscription
- * being opened, which states its value in the turn it is opened in and so
- * cannot wait (DESIGN §6.2). The one that cannot wait is the one CT-Q8 and the
- * issue `fold-from-head-with-versioned-cache` settle, and the two become one
- * reading again when they do. */
+ * One walk for every caller: enumerating the tree and finding one session's
+ * file in it are the same reading, and a subscription that opens on it waits
+ * for it like any other caller (CT-Q8). */
 async function walked(root: string, depth: number): Promise<string[]> {
   let level = [root];
   for (let step = 0; step < depth; step += 1) {
@@ -284,14 +282,6 @@ async function walked(root: string, depth: number): Promise<string[]> {
   return level;
 }
 
-function directories(root: string, depth: number): string[] {
-  let level = [root];
-  for (let step = 0; step < depth; step += 1) {
-    level = level.flatMap((dir) => names(dir).map((entry) => join(dir, entry)));
-  }
-  return level;
-}
-
 export interface AgentNames {
   readonly agent_id?: string;
   readonly run_id?: string;
@@ -343,14 +333,6 @@ async function listed(dir: string): Promise<string[]> {
   }
 }
 
-function names(dir: string): string[] {
-  try {
-    return readdirSync(dir);
-  } catch {
-    return [];
-  }
-}
-
 async function stated(file: string): Promise<Stats | undefined> {
   try {
     const known = await stat(file);
@@ -359,16 +341,3 @@ async function stated(file: string): Promise<Stats | undefined> {
     return undefined;
   }
 }
-
-function statOf(file: string) {
-  try {
-    const known = statSync(file);
-    return known.isFile() ? known : undefined;
-  } catch {
-    return undefined;
-  }
-}
-
-function isFile(file: string): boolean {
-  return statOf(file) !== undefined;
-}

package/src/transcript/fold.ts

@@ -61,6 +61,29 @@ export const NO_FACTS: TranscriptFacts = {
   agent_tree: { teammates: [], agents: [], workflows: [] },
 };
 
+/** Everything the fold carries from one record to the next, as data.
+ *
+ * The facts are what a consumer reads; this is what reading further needs. The
+ * two are not the same — a todo is held under the key a later record updates it
+ * by, a call is held until its result arrives — so a fold resumed from its
+ * facts alone would answer the next record differently from one that had read
+ * every record before it. What is kept here is the whole of that difference,
+ * which is what makes resuming from it the same reading rather than a similar
+ * one. */
+export interface FoldState {
+  readonly api_error?: SessionApiError;
+  readonly last_user_input_at?: Timestamp;
+  readonly model?: string;
+  readonly effort?: string;
+  readonly files: readonly (readonly [string, ExternalFile])[];
+  readonly todos: readonly (readonly [string, SessionTodo])[];
+  readonly teammates: readonly (readonly [string, Teammate])[];
+  readonly background: readonly (readonly [string, SessionBackgroundStatus])[];
+  readonly workflows: readonly (readonly [string, SessionWorkflowStatus])[];
+  readonly agents: readonly (readonly [string, AgentTreeNode])[];
+  readonly calls: readonly (readonly [string, PendingCall])[];
+}
+
 /** The one place a transcript line is interpreted.
  *
  * Nothing outside this module parses a transcript record. A line arrives, the
@@ -104,6 +127,43 @@ export class TranscriptFold {
     };
   }
 
+  /** Everything the reading carries forward, so that it can be taken up again
+   * where it stopped. */
+  get held(): FoldState {
+    return {
+      ...(this.#apiError === undefined ? {} : { api_error: this.#apiError }),
+      ...(this.#lastUserInputAt === undefined ? {} : { last_user_input_at: this.#lastUserInputAt }),
+      ...(this.#model === undefined ? {} : { model: this.#model }),
+      ...(this.#effort === undefined ? {} : { effort: this.#effort }),
+      files: [...this.#files],
+      todos: [...this.#todos],
+      teammates: [...this.#teammates],
+      background: [...this.#background],
+      workflows: [...this.#workflows],
+      agents: [...this.#agents],
+      calls: [...this.#calls],
+    };
+  }
+
+  /** Take up a reading somebody else left off, replacing whatever this fold
+   * held. The state is trusted as the record of records already read — what
+   * makes it trustworthy is decided where it is kept (`FOLD_CACHE_VERSION`),
+   * not here. */
+  restore(state: FoldState): void {
+    this.reset();
+    this.#apiError = state.api_error;
+    this.#lastUserInputAt = state.last_user_input_at;
+    this.#model = state.model;
+    this.#effort = state.effort;
+    for (const [key, value] of state.files) this.#files.set(key, value);
+    for (const [key, value] of state.todos) this.#todos.set(key, value);
+    for (const [key, value] of state.teammates) this.#teammates.set(key, value);
+    for (const [key, value] of state.background) this.#background.set(key, { ...value });
+    for (const [key, value] of state.workflows) this.#workflows.set(key, { ...value });
+    for (const [key, value] of state.agents) this.#agents.set(key, { ...value });
+    for (const [key, value] of state.calls) this.#calls.set(key, value);
+  }
+
   reset(): void {
     this.#apiError = undefined;
     this.#lastUserInputAt = undefined;
@@ -648,13 +708,13 @@ type Mutable<T> = { -readonly [K in keyof T]: T[K] };
 
 /** A teammate as the fold holds it: what the contract states about it, and the
  * two facts only the tree needs. */
-interface Teammate {
+export interface Teammate {
   readonly status: Mutable<SessionTeammate>;
   agent_id?: string;
   team_name?: string;
 }
 
-interface PendingCall {
+export interface PendingCall {
   readonly name: string;
   readonly input: Record<string, unknown>;
   readonly at?: Timestamp;

package/src/transcript/index.ts

@@ -4,7 +4,9 @@ export {
   TranscriptFiles,
   type TranscriptFilesDeps,
 } from "./files.ts";
+export { FOLD_CACHE_VERSION, FoldCache, type FoldCacheEntry } from "./cache.ts";
 export {
+  type FoldState,
   NO_FACTS,
   readRecord,
   type TranscriptFacts,
@@ -12,5 +14,5 @@ export {
   type TranscriptRecord,
 } from "./fold.ts";
 export { READ_LIMIT, readSlice } from "./read.ts";
-export { type Appended, FOLD_TAIL_BYTES, TranscriptTail } from "./tail.ts";
+export { type Appended, READ_CHUNK_BYTES, TranscriptTail } from "./tail.ts";
 export { ITEMS_SNAPSHOT, Transcripts, type TranscriptsDeps } from "./transcripts.ts";

package/src/transcript/items/classify.ts

@@ -97,6 +97,27 @@ type Draft = Record<string, unknown> & {
   at: number;
 };
 
+/** Everything a reading carries from one record to the next, as data.
+ *
+ * The items already handed over are not here — a reader holds those — but what
+ * decides how the next record reads is: which turn the file is in, which slash
+ * command output belongs to, whose file it is, and the calls still waiting for
+ * an answer. A reading taken up from this answers the next record exactly as
+ * the reading that produced it would have.
+ *
+ * The outstanding calls name their items by id rather than carrying them: the
+ * items are what a reader already holds, and naming them is what lets a result
+ * point back at the call in the very list the reader has. */
+export interface ClassificationState {
+  readonly turn: number;
+  readonly slash?: string;
+  readonly subject: TranscriptSubject;
+  readonly calls: readonly (readonly [
+    string,
+    { readonly tool: string; readonly message?: string; readonly name: string },
+  ])[];
+}
+
 /** A whole transcript read as items, in the order the file holds them.
  *
  * The file is read through once and the links are filled in as the answers
@@ -144,6 +165,62 @@ export class Classification {
     this.#subject = subject;
   }
 
+  /** What this reading would carry into the next record. */
+  get held(): ClassificationState {
+    const calls: ClassificationState["calls"] = [...this.#calls].map(([id, call]) => [
+      id,
+      {
+        tool: call.tool.id,
+        name: call.name,
+        ...optional("message", call.message?.id),
+      },
+    ]);
+    return {
+      turn: this.#turn,
+      subject: this.#subject,
+      calls,
+      ...optional("slash", this.#slash),
+    };
+  }
+
+  /** Take up a reading somebody else left off.
+   *
+   * `known` is the items that reading handed over and this one still holds, so
+   * an outstanding call is matched back to the item a reader has rather than to
+   * a copy of it: what the answer writes into the call then lands where the
+   * reader will look for it. A call whose item the reader no longer holds is
+   * kept by name and id alone — the result still says which tool answered and
+   * which item it belongs to, and there is no item left to write into. */
+  restore(state: ClassificationState, known: Iterable<Item>): void {
+    this.#items = [];
+    this.#calls.clear();
+    this.#turn = state.turn;
+    this.#subject = state.subject;
+    this.#slash = state.slash;
+    const held = new Map<string, Draft>();
+    for (const item of known) {
+      const draft = item as unknown as Draft;
+      if (typeof draft.id === "string") held.set(draft.id, draft);
+    }
+    const detached = (id: string): Draft =>
+      held.get(id) ??
+      ({
+        id,
+        uuid: "",
+        subject: this.#subject,
+        source: { offset: 0, bytes: 0 },
+        type: "",
+        at: 0,
+      } as Draft);
+    for (const [id, call] of state.calls) {
+      this.#calls.set(id, {
+        tool: detached(call.tool),
+        name: call.name,
+        ...optional("message", call.message === undefined ? undefined : detached(call.message)),
+      });
+    }
+  }
+
   /** The records of one chunk as the items they were read as, oldest first.
    *
    * What comes back is this chunk's items alone. A call the chunk before it

package/src/transcript/items/index.ts

@@ -1,4 +1,4 @@
-export { classify, Classification } from "./classify.ts";
+export { classify, Classification, type ClassificationState } from "./classify.ts";
 export { document, type DumpView } from "./document.ts";
 export { fields, type Item } from "./item.ts";
 export { ledger } from "./ids.ts";