@reticlehq/server 2.14.0 → 3.2.0

package/dist/memory/journal/session-end.d.ts

@@ -0,0 +1,112 @@
+import { type DriveProgram, type TapeStep } from './drive-flow.js';
+import { type OnboardingStep } from '@reticlehq/core/telemetry';
+import type { AmbientCounts } from '@reticlehq/engine/window/ambient.js';
+import { type ProjectId, type JournalAction } from '@reticlehq/core';
+import type { FileSystemPort } from '../project/fs/fs-port.js';
+/**
+ * Session teardown: the durable half of ending a session. Two things must happen when a tab disconnects,
+ * and neither did before — the journal batches events and only writes at its flush threshold, so the tail
+ * of every session (< one batch) was silently lost from disk, and the learned ambient map was never
+ * persisted at all, so every session re-learned the page's churn from zero.
+ *
+ * Both are best-effort: a teardown failure must never surface as a session error (the tab is already gone).
+ */
+/** The minimal Session surface teardown needs (Session satisfies it structurally). */
+export interface SessionEndTarget {
+    readonly id: string;
+    /**
+     * Enough of the session to name the subject in the protocol's terms.
+     *
+     * Optional for the same reason `readJournalActions` is: an existing double keeps compiling and
+     * simply produces a run with no subject, which is the honest outcome rather than an invented
+     * one. See `subjectOf` -- there is one definition of what identifies a subject, and this reads
+     * it rather than assembling a second.
+     */
+    readonly url?: string | undefined;
+    readonly runtime?: string | undefined;
+    readonly currentDocumentId?: string | undefined;
+    readonly currentEditEpoch?: number | undefined;
+    /** Write any batched journal events to disk. */
+    flushJournal(): Promise<void>;
+    /** The ambient-churn counts learned during this session. */
+    ambientCounts(): AmbientCounts;
+    ownAmbientCounts(): AmbientCounts;
+    /**
+     * This session's journal, for the run fold below. Optional so every existing double keeps
+     * compiling and simply produces no run — the safe direction for an artifact whose only failure
+     * mode is being absent.
+     */
+    readJournalActions?(): Promise<JournalAction[]>;
+    /** The project's own `.reticle`, so a run lands in the repo it is about. */
+    readonly artifactRoot?: string | undefined;
+    readonly projectId?: ProjectId | undefined;
+}
+interface SessionEndDeps {
+    fs: FileSystemPort;
+    reticleRoot: string;
+    /** Journaling/persistence off (opt-out) → teardown is a no-op. */
+    enabled: boolean;
+    /**
+     * The one clock in this file, injected — the run artifact stamps `createdAt` from it.
+     *
+     * Optional so every existing construction keeps working; defaulted at the single call site rather
+     * than reached for inside the fold, which stays pure.
+     */
+    now?: () => number;
+    /**
+     * Take this session's ambient tape, closing it. Bound by the CALLER, which is the half that
+     * already holds the recorder — teardown needs the data, not the recorder, and reaching two
+     * directories into `language/flows/recording/tape/` for a constant and two types was a dependency
+     * nobody needed. `directory-reach.test.ts` refused it, and its advice was right.
+     */
+    takeAmbientTape?: () => {
+        steps: readonly TapeStep[];
+        startPath?: string;
+    } | undefined;
+    /**
+     * Report one funnel step. Injected rather than imported, because this module is teardown and the
+     * reach guard is right that it should not grow a dependency on the telemetry tree to say so.
+     */
+    reportStep?: (step: OnboardingStep) => Promise<boolean>;
+    /**
+     * The flow store, so a drive can be saved as a flow at teardown.
+     *
+     * Optional, like `now`, so every existing construction keeps working — and absent means the
+     * capture simply does not run rather than throwing at teardown, which is the rule every step in
+     * this handler follows.
+     */
+    flows?: {
+        save: (program: DriveProgram, annotations?: undefined, projectId?: ProjectId) => Promise<unknown>;
+    };
+    /**
+     * Called when a run artifact is written, so cloud sync can cycle instead of waiting for its timer.
+     *
+     * Optional, like everything else here: absent simply means the next scheduled cycle picks it up.
+     */
+    onRunPersisted?: () => void;
+    /**
+     * The ids of the sessions still OPEN, read at each teardown so retention never deletes a journal
+     * that is still being written. A getter rather than a snapshot: this handler is built once at
+     * wiring and fires for every session for the life of the daemon.
+     *
+     * Injected, like the clock and the reporter — teardown needs the answer, not the registry.
+     * Optional so every existing construction keeps compiling, and absent means "none known", which
+     * is exactly the old behaviour.
+     */
+    liveSessionIds?: () => ReadonlySet<string>;
+}
+/**
+ * Build the teardown handler the bridge fires when a session is removed. Flushes the journal first (so no
+ * evidence is lost), then folds this session's ambient counts into the persisted map so the NEXT session
+ * starts already knowing which regions churn.
+ */
+export declare function makeSessionEnd(deps: SessionEndDeps): (session: SessionEndTarget) => Promise<void>;
+/**
+ * Persist this session's drive as a run artifact, when it proved anything.
+ *
+ * Scoped to the session's OWN `artifactRoot`, never the daemon's cwd: a daemon registered globally
+ * stands in `$HOME`, and writing there is how one app's evidence once reached another account's
+ * dashboard.
+ */
+export declare function recordDriveRun(deps: SessionEndDeps, session: SessionEndTarget): Promise<void>;
+export {};

package/dist/memory/journal/session-end.js

@@ -0,0 +1,179 @@
+import { driveFlowsFrom } from './drive-flow.js';
+import { OnboardingPhase, OnboardingStepStatus, } from '@reticlehq/core/telemetry';
+import { AmbientStore } from './ambient-store.js';
+import { subjectOf } from '@reticlehq/core';
+import { pruneSessions } from './on-disk/retention.js';
+import { buildVerificationRun } from '../../judgement/runs/artifact/build-verification-run.js';
+import { driveRunFrom, driveRunId } from '../../judgement/runs/drive-run.js';
+import { RunStore } from '../../judgement/runs/artifact/run-store.js';
+/**
+ * Build the teardown handler the bridge fires when a session is removed. Flushes the journal first (so no
+ * evidence is lost), then folds this session's ambient counts into the persisted map so the NEXT session
+ * starts already knowing which regions churn.
+ */
+export function makeSessionEnd(deps) {
+    return async (session) => {
+        if (!deps.enabled)
+            return;
+        try {
+            await session.flushJournal();
+        }
+        catch {
+            // a failed flush must not block ambient persistence, and must never throw at teardown
+        }
+        try {
+            // The session's own root, like the run artifact below: a churn map learned from one app is
+            // not about another, and writing it to the daemon's cwd both mixed two apps' maps together and
+            // left an `ambient.json` in a tree the user never instrumented.
+            const store = new AmbientStore(deps.fs, session.artifactRoot ?? deps.reticleRoot);
+            const persisted = await store.load();
+            // Accumulate history + what is NEW. `ownAmbientCounts` excludes the map this session was
+            // seeded from at startup; `ambientCounts` includes it, and adding that onto `persisted` wrote
+            // `2 x persisted + own` on every teardown — a doubling per session that had driven the
+            // committed map to ~9.1e23.
+            const merged = { ...persisted };
+            for (const [ref, count] of Object.entries(session.ownAmbientCounts())) {
+                merged[ref] = (merged[ref] ?? 0) + count;
+            }
+            await store.save(merged);
+        }
+        catch {
+            // ambient learning is an optimization; a disk failure never breaks teardown
+        }
+        // A DRIVE IS A VERIFICATION, and it used to leave no artifact.
+        //
+        // `.reticle/runs/` is what the cloud sync reads, and its only writer was the flow-replay path —
+        // so a session that drove the app and produced a hundred verdicts synced nothing, and the
+        // dashboard stayed empty with `lastPushAt: null`. This folds what the journal already recorded
+        // into the same artifact replay writes; the sync daemon needs no change to pick it up.
+        //
+        // AFTER the flush above, because the fold reads the ledger the flush just completed. Silent when
+        // nothing was proved either way — see drive-run.ts on why a green "nothing measured" row is
+        // worse than no row.
+        //
+        // Not nudged: the sync daemon re-reads `.reticle/runs/` from disk on every cycle, so this lands
+        // within one interval and survives a daemon that exits first. Threading a wake-up through here
+        // would buy under a minute of latency for a mutable handle held across two wiring sites.
+        // A DRIVE IS A REGRESSION TEST, and it used to need somebody to remember to say so.
+        //
+        // `record{start}` + `flow_save` is the agent-facing route and agents do not take it: the corpus
+        // measures 3 of 33 flows mutation-testable and 6 of 112 steps declaring a consequence, while the
+        // engine catches 84 of the 86 bugs it structurally can. The ceiling is how many flows EXIST that
+        // could go red, so a drive that is never saved is a regression test paid for and thrown away.
+        //
+        // Saved only when the tape declared a consequence — see drive-flow.ts for why the alternative is
+        // a false-green factory running once per session.
+        try {
+            await saveDrivenFlow(deps, session);
+        }
+        catch {
+            // a flow is a by-product of the session; failing to write one never fails the teardown
+        }
+        try {
+            await recordDriveRun(deps, session);
+        }
+        catch {
+            // an artifact is a report ABOUT the session; failing to write one never fails the teardown
+        }
+        try {
+            // Bound the journal on disk HERE, not only at daemon start.
+            //
+            // Pruning ran exactly once, during wiring. A daemon that stays up — which is the normal case for
+            // a dev session, and the whole point of the pool — therefore never pruned again, so session
+            // directories accumulated without bound for as long as it lived. Session end is the right moment
+            // because it is precisely when a new directory has just been created, which makes this amortized
+            // rather than periodic (no timer to leak) and mirrors what RunStore already does.
+            //
+            // What makes this safe is the `live` set, NOT the order.
+            //
+            // This used to read "safe against deleting the session that just ended: pruning selects the
+            // OLDEST by mtime, and the directory written moments ago is the newest". That is true of the
+            // session that ENDED and says nothing about the ones still open — and it is not even true of
+            // the byte budget, which has no count floor and will evict the single oldest entry the moment
+            // the tier is over, however few there are.
+            //
+            // A session directory's mtime is stamped at creation and never moves again (appending to a
+            // file inside it advances the FILE's mtime), so the longest-running open session is the
+            // oldest-looking thing on disk and was the first candidate for both bounds. Passing the open
+            // ids is what actually holds. The ended session is already out of the registry by the time
+            // this runs — the bridge removes it before firing teardown — so it is eligible again, which
+            // is correct: it is finished, and its journal was flushed above.
+            // The root this session actually journalled into. Pruning the daemon's tree instead meant a
+            // per-project workspace was never swept at all, so the one place journals really accumulate
+            // was the one place retention never ran.
+            await pruneSessions(deps.fs, session.artifactRoot ?? deps.reticleRoot, {
+                live: deps.liveSessionIds?.(),
+            });
+        }
+        catch {
+            // retention is best-effort maintenance; never surface at teardown
+        }
+    };
+}
+/**
+ * Persist this session's drive as a run artifact, when it proved anything.
+ *
+ * Scoped to the session's OWN `artifactRoot`, never the daemon's cwd: a daemon registered globally
+ * stands in `$HOME`, and writing there is how one app's evidence once reached another account's
+ * dashboard.
+ */
+export async function recordDriveRun(deps, session) {
+    // Called through the object, not lifted into a local: a lifted method loses its `this`, and the
+    // real Session's reader closes over the journal it was constructed with.
+    if (session.readJournalActions === undefined)
+        return;
+    const actions = await session.readJournalActions();
+    const input = driveRunFrom(actions, {
+        // Derived from the session, NOT random. Teardown fires on every socket close, and a reconnecting
+        // tab keeps its session id and appends to the same ledger — so a drive across two page reloads
+        // would fold the whole ledger twice and publish two overlapping rows, the second a superset of
+        // the first. A stable id makes the artifact idempotent: the same session rewrites its own run,
+        // and the cloud diffs by runId, so a re-push supersedes rather than duplicates.
+        runId: driveRunId(session.id),
+        ...(session.projectId === undefined ? {} : { projectId: session.projectId }),
+    });
+    if (input === undefined)
+        return;
+    // The protocol reaching the artifact a person actually reads. Only when the session could say
+    // where it was: a subject with no locator is not a weaker subject, it is a guess, and the
+    // schema makes the field optional so that absence can be told from invention.
+    const subject = session.url === undefined
+        ? undefined
+        : subjectOf({
+            id: session.id,
+            url: session.url,
+            runtime: session.runtime,
+            currentDocumentId: session.currentDocumentId,
+            currentEditEpoch: session.currentEditEpoch,
+        });
+    const store = new RunStore(deps.fs, session.artifactRoot ?? deps.reticleRoot, 
+    // Tell the sync daemon a run landed, rather than letting it find out on its next tick. Teardown
+    // used to write this artifact with no callback at all, so the evidence from a whole session sat
+    // on disk until a timer noticed — which, for the last session of the day, could be never.
+    deps.onRunPersisted === undefined ? undefined : { onWrote: deps.onRunPersisted });
+    await store.write(buildVerificationRun({ ...input, ...(subject === undefined ? {} : { subject }) }, deps.now ?? (() => Date.now())));
+}
+/** Persist what this session drove as a replayable flow, when it declared anything provable. */
+async function saveDrivenFlow(deps, session) {
+    const takeTape = deps.takeAmbientTape;
+    const flows = deps.flows;
+    if (takeTape === undefined || flows === undefined)
+        return;
+    // One flow per journey the session contained, not one flow per session — see drive-flow.ts.
+    const { programs, outcome } = driveFlowsFrom(session.id, takeTape());
+    for (const program of programs) {
+        await flows.save(program, undefined, session.projectId);
+    }
+    // The funnel step that makes the SECOND run cheap.
+    //
+    // SKIPPED, not failed, when a drive proved nothing: the agent drove and declared no consequence,
+    // which is a real outcome and not our bug. Counting it as a failure would blame the product for a
+    // choice the agent made, and counting it as success would claim a regression test that cannot go
+    // red. `unprovenSteps` is what separates "drove and proved nothing" from "drove nothing at all".
+    await deps.reportStep?.({
+        phase: OnboardingPhase.FIRST_RUN,
+        step: 'flow_recorded',
+        status: 0 === programs.length ? OnboardingStepStatus.SKIPPED : OnboardingStepStatus.COMPLETED,
+        ...(outcome.unprovenSteps === undefined ? {} : { reason: 'no_declared_consequence' }),
+    });
+}

package/dist/memory/journal/session-journal.d.ts

@@ -0,0 +1,134 @@
+import { type JournalAction, type JournalWriteLoss, type ReticleEvent } from '@reticlehq/core';
+import type { FileSystemPort } from '../project/fs/fs-port.js';
+import type { JournalReadLoss } from './journal-recorder.js';
+/**
+ * The two ceilings a durable read is held to.
+ *
+ * Neither is a tuning number. `events.jsonl` is append-only and nothing prunes a single session's
+ * file (`pruneSessions` caps how many session DIRECTORIES survive, never how large one grows), and
+ * the session id lives in the page's sessionStorage, so a reload reconnects under the same id and
+ * builds a fresh journal over the same file with its byte cursor back at zero. That first read
+ * asked for the whole accumulated file, and V8 cannot build a string longer than 0x1fffffe8
+ * characters: `Buffer.prototype.toString` throws rather than returning a shorter one. The throw came
+ * out of the daemon as a raw exception on `reticle_assert` and `reticle_act_and_wait` — the only
+ * tools that reach this fall-through, and the only two that produce a verdict at all.
+ *
+ * `MAX_READ_BYTES` bounds ONE read. That alone bounds nothing over the life of a session: reads are
+ * incremental, so a thousand reads each well under the ceiling still accumulate every record they
+ * parsed. `MAX_RETAINED_*` bound what the parse-cache keeps, which is the quantity that actually
+ * grows with session age.
+ *
+ * The numbers, and what they are measured against:
+ *
+ *   - 32 MiB is four times the ring buffer's own byte budget (`RING_BUFFER_DEFAULTS.MAX_BYTES`), so
+ *     the durable half still reaches meaningfully further back than the hot cache it backs, and
+ *     roughly a sixteenth of the string ceiling above (0x1fffffe8 characters, about 512 MB of
+ *     ASCII) — a margin, not an order of magnitude, and enough of one.
+ *   - Retention is bounded in journal BYTES and in RECORDS, because neither alone bounds memory:
+ *     bytes track the file, records track the parsed objects, and an object costs several times the
+ *     JSONL it came from (see the parse-cache note below). 100k records is fifty times the ring
+ *     buffer's event budget, so the fall-through is still the long-memory half by a wide margin.
+ */
+export declare const JOURNAL_READ_LIMITS: {
+    /** What ONE read materialises as a single string. */
+    readonly MAX_READ_BYTES: number;
+    /** What the parse-cache keeps, in the journal bytes those records were parsed from. */
+    readonly MAX_RETAINED_BYTES: number;
+    /** What the parse-cache keeps, in records. Whichever bound binds first wins. */
+    readonly MAX_RETAINED_EVENTS: 100000;
+};
+/** Ceiling overrides. Injected so a test can drive a cut without writing 32 MiB of fixture. */
+export interface SessionJournalOptions {
+    maxReadBytes?: number;
+    maxRetainedBytes?: number;
+    maxRetainedEvents?: number;
+}
+/**
+ * Default byte budget for one serialized writer's event ledger (#986).
+ *
+ * A ledger may exceed this by one bounded truncation marker. Existing oversized files are not
+ * deleted or rewritten; they stop accepting event batches. This is a per-event-file limit, not a
+ * workspace-wide quota, and does not bound action journals or concurrent independent writers.
+ */
+export declare const JOURNAL_EVENT_BYTES_CAP: number;
+/** Construction-time knobs. The cap is injected so a test can prove the bound at a few kilobytes. */
+export interface SessionJournalOptions {
+    /** Byte ceiling for this session's event ledger. Defaults to {@link JOURNAL_EVENT_BYTES_CAP}. */
+    eventBytesCap?: number;
+}
+/**
+ * The durable per-session journal: append-only JSONL for events and actions, the ledger the ring
+ * buffer becomes a hot cache over. Writes are batched (the caller flushes ring-buffer windows), so a
+ * batch is one syscall — not one per event. Reads never throw: a missing file is `[]`, a malformed or
+ * schema-invalid line is skipped, matching the never-throw discipline of the run store.
+ *
+ * Events are already browser-edge-redacted (network/storage/DOM) before they reach the wire, so the
+ * journal stores redacted payloads; the ledger is local-only. A server-side second-pass event redactor
+ * is defense-in-depth for a later commit, not a correctness gate here.
+ *
+ * Event READS are BOUNDED at both ends — one read, and what the parse-cache keeps across reads.
+ * Both keep the NEWEST records and declare what they removed through `readLoss()`. Without the read
+ * ceiling a long session's file outgrew what a JavaScript string can hold and the read threw
+ * instead of answering; without the retention ceiling the cache grew for the life of the session.
+ * See `JOURNAL_READ_LIMITS` and `session-journal.lossy-conformance.test.ts`.
+ *
+ * The event ledger is also bounded on the WRITE side — see {@link JOURNAL_EVENT_BYTES_CAP} and
+ * `#closeAtCap`. Session directories are pruned by recency in `on-disk/retention.ts`; this is the
+ * per-file half of the same question, and the half a single runaway session needs. The two ceilings
+ * answer different questions and neither substitutes for the other: one stops a read from throwing,
+ * the other stops the file from growing without end.
+ *
+ * `readActions` is NOT bounded: it reads the whole action ledger as one string and would throw the
+ * same V8 string-length error on a large enough one. Left that way deliberately, and the reason is
+ * the loss channel rather than the read — the action ledger is one record per tool call, so it is
+ * orders of magnitude smaller than the event stream, and it is read by the run fold rather than by
+ * a verdict. Bounding it needs a loss report of its own: folding an action-ledger cut into
+ * `readLoss` would impeach page verdicts (through `Session.lostSince`) over a ledger that is not
+ * page evidence. Do that when an action ledger is measured large, not before.
+ *
+ * ponytail: append-per-batch. Perf ceiling: if main-thread overhead becomes visible at high event
+ * rates, coalesce batches behind a flush timer.
+ */
+export declare class SessionJournal {
+    #private;
+    constructor(fs: FileSystemPort, root: string, sessionId: string, options?: SessionJournalOptions);
+    /**
+     * Append a batch, or refuse it at the byte cap and say so.
+     *
+     * A batch is accepted whole or not at all. This preserves the batch's ordering and avoids
+     * representing only part of a refused batch as successfully persisted evidence.
+     *
+     * REFUSAL, not rotation. Rotation keeps the newest evidence, which is the better policy in the
+     * abstract, but it shrinks the file — and `readEvents` tracks a byte offset into it, so a rotation
+     * resets that cursor and the reader silently returns FEWER events than it did a call earlier, with
+     * nothing to say why. Refusal keeps the file monotonic, so every cursor stays valid, no evidence
+     * already on disk is deleted, and the loss is at the end where it can be declared.
+     */
+    appendEvents(events: readonly ReticleEvent[]): Promise<void>;
+    /**
+     * What this session's durable ledger lost on the WRITE side, or `undefined` if it lost nothing.
+     *
+     * The half the in-band marker cannot cover. That record carries the `t` of the last refused event
+     * and every journal-backed query filters on `t`, so a window opened after the ceiling was reached
+     * holds no marker and no events — which is indistinguishable from a complete window in which
+     * nothing happened. Named `readWriteLoss` because it is the WRITER's loss: what never reached the
+     * file, as opposed to what a bounded read declined to hand back.
+     */
+    readWriteLoss(): Promise<JournalWriteLoss | undefined>;
+    appendAction(action: JournalAction): Promise<void>;
+    readEvents(): Promise<ReticleEvent[]>;
+    /**
+     * What these reads could not produce, or `undefined` when nothing was lost.
+     *
+     * A report beside the value, not inside it: the events keep the exact shape their consumers parse,
+     * and a caller holding a short list can still tell it apart from a complete one. `Session.lostSince`
+     * is what carries this to a verdict.
+     */
+    readLoss(): JournalReadLoss | undefined;
+    /**
+     * The action ledger, whole — the one read here that is still UNBOUNDED, on purpose. See the class
+     * note: it holds one record per tool call rather than one per observed event, and bounding it
+     * needs a loss channel that does not impeach page verdicts.
+     */
+    readActions(): Promise<JournalAction[]>;
+}