@ccmsg/cli 0.14.0 → 0.15.0

package/src/transcript/tail.ts

@@ -1,22 +1,19 @@
-import { closeSync, type FSWatcher, openSync, readSync, statSync, watch } from "node:fs";
+import { type FSWatcher, watch } from "node:fs";
 import { open, stat } from "node:fs/promises";
 import { CONFIRM_POLL_MS } from "../sessions/harness.ts";
+import { breathe } from "./scan.ts";
 
-/** How much of an existing transcript is read when a tail starts.
+/** How much of a transcript one read takes at a time.
  *
- * The fold's two values both describe the present — the error the latest turn
- * ended on, and the last time a person spoke — so what a tail needs on opening
- * is the recent end of the file, not its history. A megabyte is a few hundred
- * records at the sizes the harness writes, which reaches back past the current
- * turn by a wide margin while costing one read of fixed size however large the
- * file has grown (DESIGN §2.3: a transcript of any size is read from its end).
- *
- * A person who has not spoken within it is reported as having no known input
- * rather than as having spoken long ago, which is what the contract's absent
- * `last_user_input_at` already means. */
-export const FOLD_TAIL_BYTES = 1024 * 1024;
-
-/** What the tail found appended, with the offsets that place it.
+ * The reading of a whole file is cut into reads of this size and the loop is
+ * handed back between them, so following a transcript of any size costs the
+ * instance a read of fixed size rather than a pause proportional to the file.
+ * A megabyte is a few hundred records at the sizes the harness writes, which
+ * is large enough that the turns themselves cost nothing measurable against
+ * the parsing they carry. */
+export const READ_CHUNK_BYTES = 1024 * 1024;
+
+/** What the tail read, with the offsets that place it.
  *
  * The offsets are the ones a transcript read pages by, so what arrives live
  * and what was read stitch together without reading anything twice. `end` is
@@ -32,10 +29,10 @@ export interface Appended {
 export interface TailDeps {
   /** Complete lines only; a record still being written waits for its end. */
   readonly onAppended: (appended: Appended) => void;
-  /** The end of the file as it stood when the tail opened, oldest first, with
-   * the offsets that place it. The seed of what is folded and of what is
+  /** What was already in the file when the tail opened, oldest first, in the
+   * reads it arrived in. The input of what is folded and of what is
    * classified, not something a subscriber is sent. */
-  readonly onSeed: (seeded: Appended) => void;
+  readonly onExisting: (read: Appended) => void;
   /** The file is not the one the tail was reading: it shrank, so what was
    * folded out of the old contents no longer describes it. */
   readonly onTruncated: () => void;
@@ -63,17 +60,7 @@ export class TranscriptTail {
   constructor(
     private readonly path: string,
     private readonly deps: TailDeps,
-  ) {
-    // Where the file ends, read before anything can ask. A subscription's
-    // snapshot states this and is answered in the same turn the tail is
-    // created, so a size that only the awaited seed had filled in would be
-    // reported as zero and every byte already written would look appended.
-    // Reading it here also fixes what the seed reads: the seed takes this size
-    // rather than stating a newer one, so nothing lands between the size the
-    // subscriber was given and the first frame it is sent.
-    this.#size = sizeNow(path);
-    this.#offset = this.#size;
-  }
+  ) {}
 
   /** The transcript's size as last observed, which is what a subscription's
    * snapshot states and where the frames after it begin. */
@@ -81,21 +68,29 @@ export class TranscriptTail {
     return this.#size;
   }
 
+  /** Just past the last record read, which is what a reading resumed later
+   * starts from. */
+  get offset(): number {
+    return this.#offset;
+  }
+
   get running(): boolean {
     return this.#watcher !== undefined || this.#timer !== undefined;
   }
 
-  /** Begin following, seeding the fold from the end of what is already there.
+  /** Read what is already there, then begin following.
+   *
+   * `from` is where a reading of this same file left off — nothing for a file
+   * being read for the first time, which is read from its beginning. Everything
+   * before the tail's own first append is therefore accounted for, whether it
+   * was read now or read once before and remembered.
    *
-   * The seed is read before this returns rather than awaited, for the reason
-   * the size is read in the constructor: a subscription's snapshot is answered
-   * in the same turn the tail is started, and what the seed settles — the
-   * fold's values, and the items a subscriber opens on — would otherwise be
-   * stated as empty and the whole existing end of the file would arrive later
-   * as though it had just been appended. */
-  async start(): Promise<void> {
+   * This is awaited by whoever states a value derived from the file, so a
+   * subscriber is told what the whole transcript says rather than what its end
+   * says (CT-Q8). */
+  async start(from = 0): Promise<void> {
     if (this.running) return;
-    this.#seed();
+    await this.#catchUp(from);
     try {
       this.#watcher = watch(this.path, () => void this.refresh());
     } catch {
@@ -121,20 +116,17 @@ export class TranscriptTail {
     return this.#reading;
   }
 
-  #seed(): void {
-    const size = this.#size;
-    if (size === 0) return;
-    const from = Math.max(0, size - FOLD_TAIL_BYTES);
-    const complete = whole(this.#sliceSync(from, size));
-    this.#offset = from + complete.byteLength;
-    // The first line is half a record whenever the read began mid-file, so it
-    // is dropped: what is read are whole records or nothing.
-    const at = from === 0 ? 0 : complete.indexOf(NEWLINE) + 1;
-    this.deps.onSeed({ lines: split(complete, at), start: from + at, end: this.#offset, size });
+  /** The file as it already stands, read to the end it had when this began.
+   * What is written while it runs is left to the first append, so the size a
+   * subscription opens on and the first frame after it meet exactly. */
+  async #catchUp(from: number): Promise<void> {
+    this.#offset = from;
+    this.#size = await this.#stat(from);
+    await this.#consume(this.#size, this.deps.onExisting);
   }
 
   async #read(): Promise<void> {
-    const size = await this.#stat();
+    const size = await this.#stat(this.#offset);
     if (size < this.#offset) {
       // Shorter than what was already consumed: the file was replaced, so what
       // was folded out of it describes nothing, and reading resumes from its
@@ -143,21 +135,36 @@ export class TranscriptTail {
       this.deps.onTruncated();
     }
     this.#size = size;
-    if (size === this.#offset) return;
-    const complete = whole(await this.#slice(this.#offset, size));
-    if (complete.byteLength === 0) return;
-    const start = this.#offset;
-    const end = start + complete.byteLength;
-    this.#offset = end;
-    this.deps.onAppended({ lines: split(complete, 0), start, end, size });
+    await this.#consume(size, this.deps.onAppended);
   }
 
-  async #stat(): Promise<number> {
+  /** Everything up to `size`, in reads of a fixed size with the loop handed
+   * back between them. A record still being written ends the pass: it is left
+   * for the read that finds its end. */
+  async #consume(size: number, state: (read: Appended) => void): Promise<void> {
+    while (this.#offset < size) {
+      // A record longer than one read is taken in one piece rather than in
+      // halves, so the window grows until it holds a whole one.
+      let to = Math.min(this.#offset + READ_CHUNK_BYTES, size);
+      let complete = whole(await this.#slice(this.#offset, to));
+      while (complete.byteLength === 0 && to < size) {
+        to = Math.min(to + READ_CHUNK_BYTES, size);
+        complete = whole(await this.#slice(this.#offset, to));
+      }
+      if (complete.byteLength === 0) return;
+      const start = this.#offset;
+      this.#offset = start + complete.byteLength;
+      state({ lines: split(complete), start, end: this.#offset, size });
+      if (this.#offset < size) await breathe();
+    }
+  }
+
+  async #stat(fallback: number): Promise<number> {
     try {
       return (await stat(this.path)).size;
     } catch {
       // Not there. Nothing was appended, and the poll keeps looking.
-      return this.#offset;
+      return fallback;
     }
   }
 
@@ -169,26 +176,6 @@ export class TranscriptTail {
    * arithmetic puts it, inside a character as readily as before one, and
    * decoding first would turn those bytes into a replacement character of a
    * different length and move every offset derived from it. */
-  /** The same range, read without yielding, which is what the seed is read
-   * through: the turn that starts a tail is the turn that answers a
-   * subscription, and it has to hold the end of the file by then. Bounded by
-   * `FOLD_TAIL_BYTES` however large the transcript is. */
-  #sliceSync(from: number, to: number): Buffer {
-    if (to <= from) return Buffer.alloc(0);
-    let handle: number;
-    try {
-      handle = openSync(this.path, "r");
-    } catch {
-      return Buffer.alloc(0);
-    }
-    try {
-      const buffer = Buffer.alloc(to - from);
-      return buffer.subarray(0, readSync(handle, buffer, 0, buffer.length, from));
-    } finally {
-      closeSync(handle);
-    }
-  }
-
   async #slice(from: number, to: number): Promise<Buffer> {
     if (to <= from) return Buffer.alloc(0);
     const handle = await open(this.path, "r").catch(() => undefined);
@@ -213,19 +200,13 @@ function whole(bytes: Buffer): Buffer {
   return last < 0 ? Buffer.alloc(0) : bytes.subarray(0, last + 1);
 }
 
-/** The records in what was read, from a byte that begins one. */
-function split(complete: Buffer, at: number): string[] {
+/** The records in what was read, which begins at a record. */
+function split(complete: Buffer): string[] {
   const lines: string[] = [];
-  for (let from = at; from < complete.byteLength;) {
+  for (let from = 0; from < complete.byteLength;) {
     const newline = complete.indexOf(NEWLINE, from);
     lines.push(complete.toString("utf8", from, newline));
     from = newline + 1;
   }
   return lines;
 }
-
-/** How large the file is right now, or zero for one that is not there yet.
- * Synchronous because the value is wanted before the first await. */
-function sizeNow(path: string): number {
-  return statSync(path, { throwIfNoEntry: false })?.size ?? 0;
-}

package/src/transcript/transcripts.ts

@@ -1,17 +1,17 @@
 import type { InstanceId, Sid } from "@ccmsg/protocol";
 import { topicParam, type TopicValue, type UpstreamResource } from "../topics/index.ts";
+import type { FoldCache } from "./cache.ts";
 import { NO_FACTS, type TranscriptFacts, TranscriptFold } from "./fold.ts";
 import { Classification, type Item, positioned } from "./items/index.ts";
 import { type Appended, TranscriptTail } from "./tail.ts";
 
 /** How many items a subscription to `transcript.items:<sid>` opens with.
  *
- * The tail of the same megabyte the fold is seeded from, bounded by a count
- * because that read is bounded by bytes: a file of many small records would
- * otherwise make the opening frame as large as the read that produced it. Two
- * hundred items is several turns at the sizes the harness writes, which is
- * more than a live view shows at once — and a client that wants further back
- * asks for it by range rather than waiting for a snapshot to grow. */
+ * The end of a reading that covers the whole file, bounded by a count because
+ * the whole file is what was read: an opening frame is what a live view draws,
+ * and two hundred items is several turns at the sizes the harness writes —
+ * more than such a view shows at once. A client that wants further back asks
+ * for it by range rather than waiting for a snapshot to grow. */
 export const ITEMS_SNAPSHOT = 200;
 
 /** Which of the two topics a name is. Both are fed by one tail, so the
@@ -26,7 +26,11 @@ export interface TranscriptsDeps {
    * (DESIGN §4.2), or the `<sid>.jsonl` under this instance's `projects/` that
    * carries its name. A sid neither names nor is named by a file there has
    * none, and nothing is guessed for it. */
-  readonly pathOf: (sid: Sid) => string | undefined;
+  readonly pathOf: (sid: Sid) => Promise<string | undefined>;
+  /** Where a reading of a transcript is kept so the next one resumes from it.
+   * Absent leaves every reading starting from the file's beginning, which is
+   * the same answer at the price of reading it again. */
+  readonly cache?: FoldCache;
   /** The one way a value reaches subscribers (DESIGN §6.1). */
   readonly publish: (topic: string, data: unknown) => void;
   /** The fold now says something different about this session. What the fold
@@ -70,16 +74,20 @@ export class Transcripts implements UpstreamResource {
    * there, which is the whole of the snapshot for a topic whose frames are an
    * append rather than a value (DESIGN §6.2). A session whose transcript this
    * instance cannot find has nothing to state, and the subscriber begins at
-   * the first thing appended after one appears. */
-  snapshot(topic: string): readonly TopicValue[] {
+   * the first thing appended after one appears.
+   *
+   * Answered once the transcript has been read, which is what makes the size
+   * and the items it states describe the same whole file the fold does. */
+  async snapshot(topic: string): Promise<readonly TopicValue[]> {
     const sid = topicParam(topic);
-    const followed = sid === undefined ? undefined : this.#followed.get(sid);
-    if (sid === undefined || followed === undefined) return [];
+    if (sid === undefined) return [];
+    await this.ready(sid);
+    const followed = this.#followed.get(sid);
+    const tail = followed?.tail;
+    if (followed === undefined || tail === undefined) return [];
     // The items topic holds a list that is only appended to, so its snapshot
     // is the end of that list rather than a place to start from.
-    const data = isItems(topic)
-      ? { sid, items: [...followed.recent] }
-      : { sid, size: followed.tail.size };
+    const data = isItems(topic) ? { sid, items: [...followed.recent] } : { sid, size: tail.size };
     return [{ instance: this.deps.self, data }];
   }
 
@@ -89,10 +97,17 @@ export class Transcripts implements UpstreamResource {
     return this.#followed.get(sid)?.fold.facts ?? NO_FACTS;
   }
 
+  /** When the transcript held for a session has been read. Whoever states a
+   * value the fold settles waits on this, so what is stated describes the
+   * whole file rather than the part of it read so far (CT-Q8). */
+  ready(sid: Sid): Promise<void> {
+    return this.#followed.get(sid)?.ready ?? Promise.resolve();
+  }
+
   /** Whether a session's transcript is being followed, which is how "the
    * subscription drives the resource" is observable from outside. */
   following(sid: Sid): boolean {
-    return this.#followed.get(sid)?.tail.running === true;
+    return this.#followed.get(sid)?.tail?.running === true;
   }
 
   /** Ask for a session's transcript to be followed. Each hold is released
@@ -103,11 +118,15 @@ export class Transcripts implements UpstreamResource {
       held.holds += 1;
       return;
     }
-    const path = this.deps.pathOf(sid);
-    if (path === undefined) return;
-    const followed = this.#follow(sid, path);
+    const followed: Followed = {
+      holds: 1,
+      fold: new TranscriptFold(),
+      reading: new Classification(),
+      recent: [],
+      ready: Promise.resolve(),
+    };
     this.#followed.set(sid, followed);
-    void followed.tail.start();
+    followed.ready = this.#open(sid, followed);
   }
 
   release(sid: Sid): void {
@@ -116,7 +135,8 @@ export class Transcripts implements UpstreamResource {
     held.holds -= 1;
     if (held.holds > 0) return;
     this.#followed.delete(sid);
-    held.tail.stop();
+    held.tail?.stop();
+    void this.#remember(held);
     // What the fold held goes with it: the values it derived describe a file
     // this instance is no longer reading, and stating them from memory would
     // outlive the reading that justified them.
@@ -129,36 +149,100 @@ export class Transcripts implements UpstreamResource {
     const followed = new Map(this.#followed);
     this.#followed.clear();
     for (const [sid, entry] of followed) {
-      entry.tail.stop();
+      entry.tail?.stop();
+      void this.#remember(entry);
       this.deps.onFacts(sid);
     }
   }
 
-  #follow(sid: Sid, path: string): Followed {
-    const fold = new TranscriptFold();
-    const followed: Followed = {
-      holds: 1,
-      fold,
-      reading: new Classification(),
-      recent: [],
-      tail: new TranscriptTail(path, {
-        onSeed: (seeded) => {
-          // The end of the file as it already stood: it settles what the fold
-          // says and opens the reading that classifies what comes next, and it
-          // is not an append, so nothing is published for it.
-          this.#keep(sid, seeded);
-          if (foldAll(fold, seeded.lines)) this.deps.onFacts(sid);
+  /** Find the file, take up whatever reading of it was left behind, and read
+   * the rest of it.
+   *
+   * Every await here is a window in which the last hold may be released, so
+   * what was true before each one is asked again after it (DR-0015 §2.5): a
+   * reading nobody wants any more stops where it is, rather than going on to
+   * put a watch on a file and states about it into a session that has since
+   * been opened afresh. */
+  async #open(sid: Sid, followed: Followed): Promise<void> {
+    try {
+      const path = await this.deps.pathOf(sid);
+      if (!this.#holds(sid, followed)) return;
+      if (path === undefined) {
+        // Nothing to follow. The entry goes rather than standing as a session
+        // with an empty fold, so a later hold looks for the file again.
+        this.#followed.delete(sid);
+        return;
+      }
+      followed.path = path;
+      const kept = await this.deps.cache?.read(path);
+      if (!this.#holds(sid, followed)) return;
+      let settled = false;
+      if (kept !== undefined) {
+        followed.fold.restore(kept.fold);
+        followed.recent.push(...kept.items);
+        // The reading resumes as the same reading: the turn it had reached and
+        // the calls it was still waiting on are taken up with the items they
+        // belong to, so a result answered now names the call a reader holds.
+        followed.reading.restore(kept.reading, followed.recent);
+        settled = true;
+      }
+      const tail = new TranscriptTail(path, {
+        onExisting: (read) => {
+          // What was already in the file: it settles what the fold says and
+          // opens the reading that classifies what comes next, and it is not an
+          // append, so nothing is published for it.
+          this.#keep(followed, read);
+          if (foldAll(followed.fold, read.lines)) settled = true;
         },
-        onAppended: (appended) => this.#appended(sid, fold, appended),
+        onAppended: (appended) => this.#appended(sid, followed, appended),
         onTruncated: () => {
-          fold.reset();
-          this.#reset(sid);
+          followed.fold.reset();
+          this.#reset(followed);
+          void this.deps.cache?.drop(path);
           this.deps.onFacts(sid);
         },
         ...(this.deps.pollMs === undefined ? {} : { pollMs: this.deps.pollMs }),
-      }),
-    };
-    return followed;
+      });
+      followed.tail = tail;
+      await tail.start(kept?.offset ?? 0);
+      if (!this.#holds(sid, followed)) {
+        // Released while the file was being read: the watch this just put on it
+        // is the only thing left of the reading, and it goes with it.
+        tail.stop();
+        return;
+      }
+      await this.#remember(followed);
+      // A file that said nothing says nothing: the reading is finished either
+      // way, and only a reading that settled something is news to the domain.
+      if (settled) this.deps.onFacts(sid);
+    } catch {
+      // The reading could not be made. The entry goes rather than standing as
+      // one whose `ready` will never settle, since every later hold would join
+      // the same failure instead of trying the file again.
+      if (this.#followed.get(sid) === followed) this.#followed.delete(sid);
+      followed.tail?.stop();
+    }
+  }
+
+  /** Whether this is still the reading that session is being followed by. A
+   * release during an await takes the entry out, and a hold after it puts a
+   * different one in; neither is this one. */
+  #holds(sid: Sid, followed: Followed): boolean {
+    return this.#followed.get(sid) === followed;
+  }
+
+  /** Write down where the reading has reached, so the next one starts there. */
+  async #remember(followed: Followed): Promise<void> {
+    const path = followed.path;
+    const tail = followed.tail;
+    if (path === undefined || tail === undefined) return;
+    await this.deps.cache?.save(
+      path,
+      tail.offset,
+      followed.fold.held,
+      followed.reading.held,
+      followed.recent,
+    );
   }
 
   /** The one pass over what was appended: the lines go to the fold, to the
@@ -170,8 +254,8 @@ export class Transcripts implements UpstreamResource {
    * now was made in bytes that went past long ago, and a reading started when
    * somebody subscribed would not know it. What the memory holds is bounded —
    * the calls still outstanding, and the items of the opening frame. */
-  #appended(sid: Sid, fold: TranscriptFold, appended: Appended): void {
-    const changed = foldAll(fold, appended.lines);
+  #appended(sid: Sid, followed: Followed, appended: Appended): void {
+    const changed = foldAll(followed.fold, appended.lines);
     this.deps.publish(`transcript:${sid}`, {
       sid,
       lines: [...appended.lines],
@@ -179,7 +263,7 @@ export class Transcripts implements UpstreamResource {
       end: appended.end,
       size: appended.size,
     });
-    const items = this.#keep(sid, appended);
+    const items = this.#keep(followed, appended);
     // A record still being written was not read, so there is nothing to say
     // about it yet; a chunk whose records were all the interface's own
     // bookkeeping says nothing either.
@@ -191,9 +275,7 @@ export class Transcripts implements UpstreamResource {
    * subscription to open on. A result that fills in a call already handed over
    * is not sent again: the call named nothing to wait for and the result names
    * the call, so a reader ties the two together from what it already has. */
-  #keep(sid: Sid, chunk: Appended): readonly Item[] {
-    const followed = this.#followed.get(sid);
-    if (followed === undefined) return [];
+  #keep(followed: Followed, chunk: Appended): readonly Item[] {
     const items = followed.reading.readAll(positioned(chunk.lines, chunk.start));
     followed.recent.push(...items);
     if (followed.recent.length > ITEMS_SNAPSHOT) {
@@ -204,9 +286,7 @@ export class Transcripts implements UpstreamResource {
 
   /** The file is not the one that was being read, so neither the reading nor
    * what it produced describes it. */
-  #reset(sid: Sid): void {
-    const followed = this.#followed.get(sid);
-    if (followed === undefined) return;
+  #reset(followed: Followed): void {
     followed.reading = new Classification();
     followed.recent.length = 0;
   }
@@ -219,7 +299,10 @@ interface Followed {
   reading: Classification;
   /** The end of what has been read, which is what a subscription opens on. */
   readonly recent: Item[];
-  readonly tail: TranscriptTail;
+  /** Settled once the file has been found and read. */
+  ready: Promise<void>;
+  path?: string;
+  tail?: TranscriptTail;
 }
 
 function foldAll(fold: TranscriptFold, lines: readonly string[]): boolean {