@ccmsg/cli 0.12.0 → 0.14.0

package/src/transcript/files.ts

@@ -1,4 +1,5 @@
-import { readdirSync, readFileSync, statSync } from "node:fs";
+import { readdirSync, type Stats, statSync } from "node:fs";
+import { readdir, readFile, stat } from "node:fs/promises";
 import { basename, dirname, join } from "node:path";
 import type { Sid, TranscriptSubject } from "@ccmsg/protocol";
 import { type Harness, HARNESS } from "../harness/index.ts";
@@ -114,7 +115,7 @@ export class TranscriptFiles {
    * `agent_id` and `teammate` are two ways of naming the same kind of file and
    * cannot be combined — a request carrying both names two files and is a
    * caller's mistake rather than a choice this makes for them. */
-  locate(sid: Sid, names: AgentNames = {}): string {
+  async locate(sid: Sid, names: AgentNames = {}): Promise<string> {
     const file = this.session(sid);
     if (names.agent_id !== undefined && names.teammate !== undefined) {
       throw new OpError("invalid_args", "agent_id and teammate name two different transcripts");
@@ -127,9 +128,11 @@ export class TranscriptFiles {
     }
     const under = agentsDir(file, names.run_id);
     if (names.agent_id !== undefined) {
-      return existing(join(under, `agent-${name(names.agent_id, AGENT_ID, "agent_id")}${SUFFIX}`));
+      return await existing(
+        join(under, `agent-${name(names.agent_id, AGENT_ID, "agent_id")}${SUFFIX}`),
+      );
     }
-    return this.teammate(under, name(names.teammate ?? "", TEAMMATE, "teammate"));
+    return await this.teammate(under, name(names.teammate ?? "", TEAMMATE, "teammate"));
   }
 
   /** Which standing a transcript was written from, which every item read out
@@ -148,13 +151,13 @@ export class TranscriptFiles {
    * nothing goes on standing, nobody is addressed by name — so a teammate read
    * as one loses a name it might have been drawn under, where the reverse would
    * have a reader write back to something that is already gone. */
-  subjectOf(file: string): TranscriptSubject {
+  async subjectOf(file: string): Promise<TranscriptSubject> {
     const name = basename(file);
     if (!name.startsWith(AGENT_PREFIX) || !name.endsWith(SUFFIX)) return "main";
     let note: unknown;
     try {
       note = JSON.parse(
-        readFileSync(join(dirname(file), `${name.slice(0, -SUFFIX.length)}.meta.json`), "utf8"),
+        await readFile(join(dirname(file), `${name.slice(0, -SUFFIX.length)}.meta.json`), "utf8"),
       );
     } catch {
       return "sub";
@@ -168,10 +171,10 @@ export class TranscriptFiles {
    * The name a teammate carries in conversation is not its filename, so the
    * directory's own records are read for it rather than the name being
    * substituted into a path. */
-  private teammate(under: string, wanted: string): string {
+  private async teammate(under: string, wanted: string): Promise<string> {
     let names: string[];
     try {
-      names = readdirSync(under);
+      names = await readdir(under);
     } catch {
       throw new OpError("not_found", `no agent has run under this session`);
     }
@@ -179,13 +182,13 @@ export class TranscriptFiles {
       if (!each.endsWith(".meta.json")) continue;
       let document: unknown;
       try {
-        document = JSON.parse(readFileSync(join(under, each), "utf8"));
+        document = JSON.parse(await readFile(join(under, each), "utf8"));
       } catch {
         continue;
       }
       const named = (document as { name?: unknown } | null)?.name;
       if (named !== wanted) continue;
-      return existing(join(under, `${each.slice(0, -".meta.json".length)}${SUFFIX}`));
+      return await existing(join(under, `${each.slice(0, -".meta.json".length)}${SUFFIX}`));
     }
     throw new OpError("not_found", `no teammate of this session is addressed as ${wanted}`);
   }
@@ -195,16 +198,16 @@ export class TranscriptFiles {
    * The one enumeration a search and a fork sweep both start from. It states
    * the file and what a `stat` already said about it, so neither has to stat
    * again to decide whether to open it. */
-  all(): TranscriptFile[] {
+  async all(): Promise<TranscriptFile[]> {
     const layout = LAYOUTS[this.deps.harness];
     const found: TranscriptFile[] = [];
-    for (const dir of directories(this.#root(), layout.depth)) {
-      for (const entry of names(dir)) {
+    for (const dir of await walked(this.#root(), layout.depth)) {
+      for (const entry of await listed(dir)) {
         const sid = layout.sidOf(entry);
         if (sid === undefined) continue;
         const file = join(dir, entry);
-        const stat = statOf(file);
-        if (stat === undefined) continue;
+        const known = await stated(file);
+        if (known === undefined) continue;
         found.push({
           sid,
           file,
@@ -213,9 +216,9 @@ export class TranscriptFiles {
           // a session ran narrows nothing, and the transcript's own `cwd`
           // decides as it already does.
           ...(this.deps.harness === "claude" ? { project: basename(dir) } : {}),
-          size: stat.size,
-          created_at: Math.round(stat.birthtimeMs || stat.ctimeMs),
-          updated_at: Math.round(stat.mtimeMs),
+          size: known.size,
+          created_at: Math.round(known.birthtimeMs || known.ctimeMs),
+          updated_at: Math.round(known.mtimeMs),
         });
       }
     }
@@ -260,7 +263,27 @@ export class TranscriptFiles {
  *
  * Names are read rather than dates computed: what is there is what the harness
  * wrote, and a tree with a directory nobody expected is one whose files are
- * still found. */
+ * still found.
+ *
+ * There are two of this walk, and of the two readings below it, because the
+ * two callers are not alike: `all()` runs from an op and reads the tree
+ * without holding the instance, while `path()` also runs from a subscription
+ * being opened, which states its value in the turn it is opened in and so
+ * cannot wait (DESIGN §6.2). The one that cannot wait is the one CT-Q8 and the
+ * issue `fold-from-head-with-versioned-cache` settle, and the two become one
+ * reading again when they do. */
+async function walked(root: string, depth: number): Promise<string[]> {
+  let level = [root];
+  for (let step = 0; step < depth; step += 1) {
+    const below: string[] = [];
+    for (const dir of level) {
+      for (const entry of await listed(dir)) below.push(join(dir, entry));
+    }
+    level = below;
+  }
+  return level;
+}
+
 function directories(root: string, depth: number): string[] {
   let level = [root];
   for (let step = 0; step < depth; step += 1) {
@@ -305,11 +328,21 @@ function name(value: string, shape: RegExp, field: string): string {
   return value;
 }
 
-function existing(file: string): string {
-  if (!isFile(file)) throw new OpError("not_found", "no transcript is held for that agent");
+async function existing(file: string): Promise<string> {
+  if ((await stated(file)) === undefined) {
+    throw new OpError("not_found", "no transcript is held for that agent");
+  }
   return file;
 }
 
+async function listed(dir: string): Promise<string[]> {
+  try {
+    return await readdir(dir);
+  } catch {
+    return [];
+  }
+}
+
 function names(dir: string): string[] {
   try {
     return readdirSync(dir);
@@ -318,10 +351,19 @@ function names(dir: string): string[] {
   }
 }
 
+async function stated(file: string): Promise<Stats | undefined> {
+  try {
+    const known = await stat(file);
+    return known.isFile() ? known : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
 function statOf(file: string) {
   try {
-    const stat = statSync(file);
-    return stat.isFile() ? stat : undefined;
+    const known = statSync(file);
+    return known.isFile() ? known : undefined;
   } catch {
     return undefined;
   }

package/src/transcript/read.ts

@@ -1,4 +1,4 @@
-import { closeSync, openSync, readSync, statSync } from "node:fs";
+import { open, stat } from "node:fs/promises";
 import type { TranscriptReadResult, Sid } from "@ccmsg/protocol";
 import { OpError } from "../dispatch/index.ts";
 
@@ -20,15 +20,15 @@ export const READ_LIMIT = 512 * 1024;
  *
  * The offsets are the ones the `transcript` topic's frames carry, so what a
  * client reads and what arrives live stitch together without overlap. */
-export function readSlice(
+export async function readSlice(
   sid: Sid,
   file: string,
   before?: number,
   maxBytes?: number,
-): TranscriptReadResult {
+): Promise<TranscriptReadResult> {
   let size: number;
   try {
-    size = statSync(file).size;
+    size = (await stat(file)).size;
   } catch {
     throw new OpError("not_found", `the transcript of ${sid} could not be read`);
   }
@@ -45,7 +45,7 @@ export function readSlice(
   // arithmetic puts them — inside a character as readily as before one — and
   // decoding first would turn those bytes into replacement characters of a
   // different length, moving every offset derived from them.
-  const bytes = slice(file, probe, until);
+  const bytes = await slice(file, probe, until);
   // What of the read is whole records: everything up to the last newline. A
   // record the writer has not finished ends the file without one.
   const lastNewline = bytes.lastIndexOf(NEWLINE);
@@ -69,14 +69,14 @@ const NEWLINE = 0x0a;
 
 /** The bytes in a range. A range that reads short — the file was truncated
  * between the stat and the read — yields what was actually there. */
-function slice(file: string, from: number, to: number): Buffer {
+async function slice(file: string, from: number, to: number): Promise<Buffer> {
   if (to <= from) return Buffer.alloc(0);
-  const handle = openSync(file, "r");
+  const handle = await open(file, "r");
   try {
     const buffer = Buffer.alloc(to - from);
-    const read = readSync(handle, buffer, 0, buffer.length, from);
-    return buffer.subarray(0, read);
+    const { bytesRead } = await handle.read(buffer, 0, buffer.length, from);
+    return buffer.subarray(0, bytesRead);
   } finally {
-    closeSync(handle);
+    await handle.close();
   }
 }

package/src/transcript/scan.ts

@@ -0,0 +1,49 @@
+import type { TranscriptSubject } from "@ccmsg/protocol";
+import { Classification, type Item } from "./items/index.ts";
+import { positioned, span } from "./items/record.ts";
+
+/** How many records one pass reads before it hands the loop back.
+ *
+ * A transcript is read whole by the ops that classify one, and reading it is
+ * CPU rather than IO: the file arrives in one `await` and every record in it is
+ * then parsed. Nothing else on the instance runs while that happens, so the
+ * pass is cut into turns — small enough that the wait an op elsewhere sees is
+ * one turn of parsing, large enough that the turns themselves cost nothing
+ * measurable against the parsing they carry. */
+const RECORDS_PER_TURN = 2000;
+
+/** Hand the event loop back, so that whatever else is waiting on it runs.
+ *
+ * The pause of a macrotask rather than of a microtask: a promise resolved with
+ * nothing to wait for is drained before the loop is reached at all, which would
+ * make the yield a shape in the code and nothing in the behaviour. */
+export function breathe(): Promise<void> {
+  return new Promise((resume) => {
+    setImmediate(resume);
+  });
+}
+
+/** Whether this many records have gone by since the last pause. */
+export function due(read: number): boolean {
+  return read > 0 && read % RECORDS_PER_TURN === 0;
+}
+
+/** A whole transcript as the items it was read into, yielding as it goes.
+ *
+ * The same reading `classify` does over the whole text at once, cut into turns:
+ * one `Classification` is fed chunk after chunk, so a call in one chunk is
+ * still known when its result arrives in a later one, and the items come back
+ * in the order the records are in. */
+export async function classified(text: string, subject: TranscriptSubject): Promise<Item[]> {
+  const reading = new Classification(subject);
+  const lines = text.split("\n");
+  const items: Item[] = [];
+  let offset = 0;
+  for (let from = 0; from < lines.length; from += RECORDS_PER_TURN) {
+    const chunk = lines.slice(from, from + RECORDS_PER_TURN);
+    for (const item of reading.readAll(positioned(chunk, offset))) items.push(item);
+    for (const line of chunk) offset += span(line);
+    if (from + RECORDS_PER_TURN < lines.length) await breathe();
+  }
+  return items;
+}