@reticlehq/server 3.1.0 → 3.3.0

package/dist/memory/journal/session-end.d.ts

@@ -1,8 +1,9 @@
 import { type DriveProgram, type TapeStep } from './drive-flow.js';
 import { type OnboardingStep } from '@reticlehq/core/telemetry';
 import type { AmbientCounts } from '@reticlehq/engine/window/ambient.js';
-import { type JournalAction } from '@reticlehq/core';
+import { type ProjectId, type JournalAction } from '@reticlehq/core';
 import type { FileSystemPort } from '../project/fs/fs-port.js';
+import { type PruneWorkspaceOptions } from './on-disk/startup-maintenance.js';
 /**
  * Session teardown: the durable half of ending a session. Two things must happen when a tab disconnects,
  * and neither did before — the journal batches events and only writes at its flush threshold, so the tail
@@ -39,13 +40,15 @@ export interface SessionEndTarget {
     readJournalActions?(): Promise<JournalAction[]>;
     /** The project's own `.reticle`, so a run lands in the repo it is about. */
     readonly artifactRoot?: string | undefined;
-    readonly projectId?: string | undefined;
+    readonly projectId?: ProjectId | undefined;
 }
 interface SessionEndDeps {
     fs: FileSystemPort;
     reticleRoot: string;
-    /** Journaling/persistence off (opt-out) → teardown is a no-op. */
+    /** Journaling/persistence off (opt-out) → teardown is a no-op. Retention still runs. */
     enabled: boolean;
+    /** What this project is willing to keep — the `retain` block of its `.reticle.json`. */
+    retain?: PruneWorkspaceOptions;
     /**
      * The one clock in this file, injected — the run artifact stamps `createdAt` from it.
      *
@@ -76,7 +79,7 @@ interface SessionEndDeps {
      * this handler follows.
      */
     flows?: {
-        save: (program: DriveProgram, annotations?: undefined, projectId?: string) => Promise<unknown>;
+        save: (program: DriveProgram, annotations?: undefined, projectId?: ProjectId) => Promise<unknown>;
     };
     /**
      * Called when a run artifact is written, so cloud sync can cycle instead of waiting for its timer.
@@ -84,6 +87,16 @@ interface SessionEndDeps {
      * Optional, like everything else here: absent simply means the next scheduled cycle picks it up.
      */
     onRunPersisted?: () => void;
+    /**
+     * The ids of the sessions still OPEN, read at each teardown so retention never deletes a journal
+     * that is still being written. A getter rather than a snapshot: this handler is built once at
+     * wiring and fires for every session for the life of the daemon.
+     *
+     * Injected, like the clock and the reporter — teardown needs the answer, not the registry.
+     * Optional so every existing construction keeps compiling, and absent means "none known", which
+     * is exactly the old behaviour.
+     */
+    liveSessionIds?: () => ReadonlySet<string>;
 }
 /**
  * Build the teardown handler the bridge fires when a session is removed. Flushes the journal first (so no

package/dist/memory/journal/session-end.js

@@ -1,8 +1,10 @@
 import { driveFlowsFrom } from './drive-flow.js';
+import { journalActionsPath, sessionDirPath } from '../project/dir/reticle-dir.js';
+import { asSessionId } from '@reticlehq/core';
 import { OnboardingPhase, OnboardingStepStatus, } from '@reticlehq/core/telemetry';
 import { AmbientStore } from './ambient-store.js';
 import { subjectOf } from '@reticlehq/core';
-import { pruneSessions } from './on-disk/retention.js';
+import { pruneWorkspace } from './on-disk/startup-maintenance.js';
 import { buildVerificationRun } from '../../judgement/runs/artifact/build-verification-run.js';
 import { driveRunFrom, driveRunId } from '../../judgement/runs/drive-run.js';
 import { RunStore } from '../../judgement/runs/artifact/run-store.js';
@@ -13,8 +15,28 @@ import { RunStore } from '../../judgement/runs/artifact/run-store.js';
  */
 export function makeSessionEnd(deps) {
     return async (session) => {
-        if (!deps.enabled)
+        /*
+         * Retention is maintenance of a directory, not a part of journalling, so it runs either way.
+         *
+         * Teardown used to return here, and this is the one setting under which that mattered most:
+         * somebody switches journalling off BECAUSE `.reticle/` got too big, and switching it off was
+         * what stopped anything ever deleting what was already there. It also stopped the sweep of
+         * visual diffs, feedback copies and run artifacts, none of which need the journal to be written
+         * at all. The daemon's own start-path sweep was gated on the same flag, so there was no second
+         * site still running — both are ungated now.
+         */
+        const sweep = async () => {
+            try {
+                await pruneWorkspace(deps.fs, session.artifactRoot ?? deps.reticleRoot, deps.liveSessionIds?.() ?? new Set(), deps.retain ?? {});
+            }
+            catch {
+                // retention is best-effort maintenance; never surface at teardown
+            }
+        };
+        if (!deps.enabled) {
+            await sweep();
             return;
+        }
         try {
             await session.flushJournal();
         }
@@ -22,7 +44,10 @@ export function makeSessionEnd(deps) {
             // a failed flush must not block ambient persistence, and must never throw at teardown
         }
         try {
-            const store = new AmbientStore(deps.fs, deps.reticleRoot);
+            // The session's own root, like the run artifact below: a churn map learned from one app is
+            // not about another, and writing it to the daemon's cwd both mixed two apps' maps together and
+            // left an `ambient.json` in a tree the user never instrumented.
+            const store = new AmbientStore(deps.fs, session.artifactRoot ?? deps.reticleRoot);
             const persisted = await store.load();
             // Accumulate history + what is NEW. `ownAmbientCounts` excludes the map this session was
             // seeded from at startup; `ambientCounts` includes it, and adding that onto `persisted` wrote
@@ -81,13 +106,48 @@ export function makeSessionEnd(deps) {
             // because it is precisely when a new directory has just been created, which makes this amortized
             // rather than periodic (no timer to leak) and mirrors what RunStore already does.
             //
-            // Safe against deleting the session that just ended: pruning selects the OLDEST by mtime, and the
-            // directory written moments ago is the newest.
-            await pruneSessions(deps.fs, deps.reticleRoot);
+            // What makes this safe is the `live` set, NOT the order.
+            //
+            // This used to read "safe against deleting the session that just ended: pruning selects the
+            // OLDEST by mtime, and the directory written moments ago is the newest". That is true of the
+            // session that ENDED and says nothing about the ones still open — and it is not even true of
+            // the byte budget, which has no count floor and will evict the single oldest entry the moment
+            // the tier is over, however few there are.
+            //
+            // A session directory's mtime is stamped at creation and never moves again (appending to a
+            // file inside it advances the FILE's mtime), so the longest-running open session is the
+            // oldest-looking thing on disk and was the first candidate for both bounds. Passing the open
+            // ids is what actually holds. The ended session is already out of the registry by the time
+            // this runs — the bridge removes it before firing teardown — so it is eligible again, which
+            // is correct: it is finished, and its journal was flushed above.
+            // The root this session actually journalled into. Pruning the daemon's tree instead meant a
+            // per-project workspace was never swept at all, so the one place journals really accumulate
+            // was the one place retention never ran.
+            /*
+             * A session nobody ever drove keeps no journal.
+             *
+             * The observers run from the moment the SDK attaches, so an idle tab on a busy page
+             * accumulates megabytes without a single tool call. The cost is not the disk: those
+             * directories occupy retention SLOTS, so a journal that could answer a verdict question is
+             * evicted by one that was never asked one.
+             *
+             * Decided at the END and never by gating CAPTURE. Whether a tool call will happen is not
+             * knowable while the events that would answer it are being recorded, and gating capture on it
+             * would leave the first assertion of a session with nothing to read.
+             */
+            await dropUndrivenJournal(deps.fs, session.artifactRoot ?? deps.reticleRoot, asSessionId(session.id));
         }
         catch {
-            // retention is best-effort maintenance; never surface at teardown
+            // teardown must never throw: the tab is already gone
         }
+        // EVERY tier, not just sessions. Visual diffs and feedback copies were pruned only at daemon
+        // start, against the DAEMON's root — which for a globally registered daemon is `$HOME` and not
+        // the project at all, so the two tiers that only ever grow in a project workspace were the two
+        // never swept there. Same defect as the one this line already fixed for sessions, one move
+        // behind. The byte budget rides along for the same reason: it was wired at daemon start too.
+        //
+        // Outside the try above, so a failure anywhere in teardown still leaves the directory swept.
+        await sweep();
     };
 }
 /**
@@ -157,3 +217,21 @@ async function saveDrivenFlow(deps, session) {
         ...(outcome.unprovenSteps === undefined ? {} : { reason: 'no_declared_consequence' }),
     });
 }
+/**
+ * Remove this session's journal when it served no tool call.
+ *
+ * "Served no tool call" is read off the actions ledger, which is written one line per call: absent
+ * or empty means nothing was ever driven here. Best-effort like every other maintenance step —
+ * a session that cannot be tidied is not a session that failed.
+ */
+async function dropUndrivenJournal(fs, root, sessionId) {
+    try {
+        const actions = await fs.readFile(journalActionsPath(root, sessionId)).catch(() => '');
+        if (actions.trim().length > 0)
+            return;
+        await fs.rm(sessionDirPath(root, sessionId));
+    }
+    catch {
+        // tidying is never the reason a teardown fails
+    }
+}

package/dist/memory/journal/session-journal.d.ts

@@ -1,5 +1,61 @@
-import { type JournalAction, type ReticleEvent } from '@reticlehq/core';
+import { type JournalAction, type JournalWriteLoss, type ReticleEvent } from '@reticlehq/core';
 import type { FileSystemPort } from '../project/fs/fs-port.js';
+import type { JournalLedgerSize, JournalReadLoss } from './journal-recorder.js';
+/**
+ * The two ceilings a durable read is held to.
+ *
+ * Neither is a tuning number. `events.jsonl` is append-only and nothing prunes a single session's
+ * file (`pruneSessions` caps how many session DIRECTORIES survive, never how large one grows), and
+ * the session id lives in the page's sessionStorage, so a reload reconnects under the same id and
+ * builds a fresh journal over the same file with its byte cursor back at zero. That first read
+ * asked for the whole accumulated file, and V8 cannot build a string longer than 0x1fffffe8
+ * characters: `Buffer.prototype.toString` throws rather than returning a shorter one. The throw came
+ * out of the daemon as a raw exception on `reticle_assert` and `reticle_act_and_wait` — the only
+ * tools that reach this fall-through, and the only two that produce a verdict at all.
+ *
+ * `MAX_READ_BYTES` bounds ONE read. That alone bounds nothing over the life of a session: reads are
+ * incremental, so a thousand reads each well under the ceiling still accumulate every record they
+ * parsed. `MAX_RETAINED_*` bound what the parse-cache keeps, which is the quantity that actually
+ * grows with session age.
+ *
+ * The numbers, and what they are measured against:
+ *
+ *   - 32 MiB is four times the ring buffer's own byte budget (`RING_BUFFER_DEFAULTS.MAX_BYTES`), so
+ *     the durable half still reaches meaningfully further back than the hot cache it backs, and
+ *     roughly a sixteenth of the string ceiling above (0x1fffffe8 characters, about 512 MB of
+ *     ASCII) — a margin, not an order of magnitude, and enough of one.
+ *   - Retention is bounded in journal BYTES and in RECORDS, because neither alone bounds memory:
+ *     bytes track the file, records track the parsed objects, and an object costs several times the
+ *     JSONL it came from (see the parse-cache note below). 100k records is fifty times the ring
+ *     buffer's event budget, so the fall-through is still the long-memory half by a wide margin.
+ */
+export declare const JOURNAL_READ_LIMITS: {
+    /** What ONE read materialises as a single string. */
+    readonly MAX_READ_BYTES: number;
+    /** What the parse-cache keeps, in the journal bytes those records were parsed from. */
+    readonly MAX_RETAINED_BYTES: number;
+    /** What the parse-cache keeps, in records. Whichever bound binds first wins. */
+    readonly MAX_RETAINED_EVENTS: 100000;
+};
+/** Ceiling overrides. Injected so a test can drive a cut without writing 32 MiB of fixture. */
+export interface SessionJournalOptions {
+    maxReadBytes?: number;
+    maxRetainedBytes?: number;
+    maxRetainedEvents?: number;
+}
+/**
+ * Default byte budget for one serialized writer's event ledger (#986).
+ *
+ * A ledger may exceed this by one bounded truncation marker. Existing oversized files are not
+ * deleted or rewritten; they stop accepting event batches. This is a per-event-file limit, not a
+ * workspace-wide quota, and does not bound action journals or concurrent independent writers.
+ */
+export declare const JOURNAL_EVENT_BYTES_CAP: number;
+/** Construction-time knobs. The cap is injected so a test can prove the bound at a few kilobytes. */
+export interface SessionJournalOptions {
+    /** Byte ceiling for this session's event ledger. Defaults to {@link JOURNAL_EVENT_BYTES_CAP}. */
+    eventBytesCap?: number;
+}
 /**
  * The durable per-session journal: append-only JSONL for events and actions, the ledger the ring
  * buffer becomes a hot cache over. Writes are batched (the caller flushes ring-buffer windows), so a
@@ -10,15 +66,76 @@ import type { FileSystemPort } from '../project/fs/fs-port.js';
  * journal stores redacted payloads; the ledger is local-only. A server-side second-pass event redactor
  * is defense-in-depth for a later commit, not a correctness gate here.
  *
- * ponytail: append-per-batch, no retention yet. Bounded-disk pruning (cap session dirs / file size,
- * "pruned like runs/") is a dedicated follow-up. Perf ceiling: if main-thread overhead becomes
- * visible at high event rates, coalesce batches behind a flush timer.
+ * Event READS are BOUNDED at both ends — one read, and what the parse-cache keeps across reads.
+ * Both keep the NEWEST records and declare what they removed through `readLoss()`. Without the read
+ * ceiling a long session's file outgrew what a JavaScript string can hold and the read threw
+ * instead of answering; without the retention ceiling the cache grew for the life of the session.
+ * See `JOURNAL_READ_LIMITS` and `session-journal.lossy-conformance.test.ts`.
+ *
+ * The event ledger is also bounded on the WRITE side — see {@link JOURNAL_EVENT_BYTES_CAP} and
+ * `#closeAtCap`. Session directories are pruned by recency in `on-disk/retention.ts`; this is the
+ * per-file half of the same question, and the half a single runaway session needs. The two ceilings
+ * answer different questions and neither substitutes for the other: one stops a read from throwing,
+ * the other stops the file from growing without end.
+ *
+ * `readActions` is NOT bounded: it reads the whole action ledger as one string and would throw the
+ * same V8 string-length error on a large enough one. Left that way deliberately, and the reason is
+ * the loss channel rather than the read — the action ledger is one record per tool call, so it is
+ * orders of magnitude smaller than the event stream, and it is read by the run fold rather than by
+ * a verdict. Bounding it needs a loss report of its own: folding an action-ledger cut into
+ * `readLoss` would impeach page verdicts (through `Session.lostSince`) over a ledger that is not
+ * page evidence. Do that when an action ledger is measured large, not before.
+ *
+ * ponytail: append-per-batch. Perf ceiling: if main-thread overhead becomes visible at high event
+ * rates, coalesce batches behind a flush timer.
  */
 export declare class SessionJournal {
     #private;
-    constructor(fs: FileSystemPort, root: string, sessionId: string);
+    constructor(fs: FileSystemPort, root: string, sessionId: string, options?: SessionJournalOptions);
+    /**
+     * Append a batch, or refuse it at the byte cap and say so.
+     *
+     * A batch is accepted whole or not at all. This preserves the batch's ordering and avoids
+     * representing only part of a refused batch as successfully persisted evidence.
+     *
+     * REFUSAL, not rotation. Rotation keeps the newest evidence, which is the better policy in the
+     * abstract, but it shrinks the file — and `readEvents` tracks a byte offset into it, so a rotation
+     * resets that cursor and the reader silently returns FEWER events than it did a call earlier, with
+     * nothing to say why. Refusal keeps the file monotonic, so every cursor stays valid, no evidence
+     * already on disk is deleted, and the loss is at the end where it can be declared.
+     */
     appendEvents(events: readonly ReticleEvent[]): Promise<void>;
+    /**
+     * What this session's durable ledger lost on the WRITE side, or `undefined` if it lost nothing.
+     *
+     * The half the in-band marker cannot cover. That record carries the `t` of the last refused event
+     * and every journal-backed query filters on `t`, so a window opened after the ceiling was reached
+     * holds no marker and no events — which is indistinguishable from a complete window in which
+     * nothing happened. Named `readWriteLoss` because it is the WRITER's loss: what never reached the
+     * file, as opposed to what a bounded read declined to hand back.
+     */
+    readWriteLoss(): Promise<JournalWriteLoss | undefined>;
+    /**
+     * The ledger's size against its ceiling, once the size is known.
+     *
+     * `undefined` before the first append, because `#eventBytes` is seeded from `stat` at that point
+     * and a zero reported before it is measured would be a claim rather than a reading.
+     */
+    ledger(): JournalLedgerSize | undefined;
     appendAction(action: JournalAction): Promise<void>;
     readEvents(): Promise<ReticleEvent[]>;
+    /**
+     * What these reads could not produce, or `undefined` when nothing was lost.
+     *
+     * A report beside the value, not inside it: the events keep the exact shape their consumers parse,
+     * and a caller holding a short list can still tell it apart from a complete one. `Session.lostSince`
+     * is what carries this to a verdict.
+     */
+    readLoss(): JournalReadLoss | undefined;
+    /**
+     * The action ledger, whole — the one read here that is still UNBOUNDED, on purpose. See the class
+     * note: it holds one record per tool call rather than one per observed event, and bounding it
+     * needs a loss channel that does not impeach page verdicts.
+     */
     readActions(): Promise<JournalAction[]>;
 }