@reticlehq/server 2.14.0 → 3.2.0

package/dist/memory/cloud/sync-daemon.js

@@ -0,0 +1,297 @@
+/**
+ * Automatic sync, for the session nobody thinks about.
+ *
+ * Somebody runs Reticle all morning. Every tool call rewrites the impact record; every verification
+ * writes a run. If sync is a command a human types, the dashboard is stale for exactly as long as
+ * they are busy — which is precisely when the numbers on it would have been worth looking at.
+ *
+ * So the daemon does it. Not a new process, not a thing to remember: the daemon is already running
+ * for the whole session (it is what the agent talks to), so it is the only place where "sync while
+ * work is happening" costs nobody anything.
+ *
+ * ── THE THREE RULES THAT KEEP IT INVISIBLE ────────────────────────────────────────────────────
+ * QUIET WHEN IDLE. A cycle on an unchanged session is one small GET and nothing else. That is what
+ * makes a one-minute timer affordable; it is also why the timer is not adaptive — a scheme that
+ * "backs off when idle" is complexity paid for a cost that is already near zero.
+ *
+ * NEVER IN THE WAY. Nothing here is awaited by a tool call. A sync that can slow down or fail a
+ * verification would be a sync worth switching off, and the local record is authoritative anyway.
+ *
+ * SILENT UNLESS IT MATTERS. A failed cycle is logged once per NEW error, not once per minute: a
+ * laptop on a train would otherwise fill the log with the same line four hundred times and teach
+ * everyone to ignore it.
+ */
+import { log } from '../../log.js';
+import { emitSyncHook } from '../../hooks/hook-emit.js';
+import { describeSync, runSyncCycle } from './sync-cycle.js';
+import { diskSink, diskSource, readCloudState } from './sync-disk.js';
+import { resolveProjectCloud } from './cloud-config.js';
+/** How often the daemon cycles. See the note above on why this is a constant and not a curve. */
+const DAEMON_SYNC_INTERVAL_MS = 60_000;
+/**
+ * The cadence while something is actually happening.
+ *
+ * A fixed minute is the wrong shape for both states it has to cover. During a drive the ledger
+ * changes on every tool call and a minute of lag is what makes a dashboard feel dead; idle, a minute
+ * is already more often than nothing has changed deserves.
+ *
+ * So the interval follows the work: a cycle that MOVED something earns the fast rate, a cycle that
+ * moved nothing backs off to the slow one. No session plumbing and no new configuration — activity
+ * is inferred from what the last cycle actually sent, which is the only honest evidence available.
+ *
+ * Deliberately not per-tool-call. A push in the tool path would put the network in the agent's inner
+ * loop, and verification working with the network down is a promise this product makes.
+ */
+const DAEMON_SYNC_ACTIVE_INTERVAL_MS = 5_000;
+/** Given to the first cycle so a freshly-started daemon does not race the session that woke it. */
+const FIRST_CYCLE_DELAY_MS = 5_000;
+/**
+ * How long a nudge waits before cycling.
+ *
+ * Long enough to COALESCE — a battery that writes six runs in a second must cause one cycle, not
+ * six — and short enough that finishing a verification and looking at the dashboard feels like one
+ * action. Not zero: a run artifact is renamed into place and the sibling records it implies are
+ * written just after, so cycling on the very first byte would ship a bundle that is missing them.
+ */
+const NUDGE_DELAY_MS = 1_500;
+const defaultRequest = async (url, init) => {
+    const res = await fetch(url, init);
+    return { status: res.status, text: await res.text() };
+};
+/**
+ * Start the loop. Safe to call for an UNLINKED project: it resolves the link on every tick and does
+ * nothing until one appears, which is what lets `reticle link` take effect without a restart.
+ */
+export function startSyncDaemon(deps) {
+    const intervalMs = deps.intervalMs ?? DAEMON_SYNC_INTERVAL_MS;
+    // Never slower than the idle rate: a deployment that lengthens the interval means "sync less", and
+    // an active burst must not quietly re-introduce the cost it was lowering.
+    const activeIntervalMs = Math.min(DAEMON_SYNC_ACTIVE_INTERVAL_MS, intervalMs);
+    const now = deps.now ?? (() => Date.now());
+    const request = deps.request ?? defaultRequest;
+    let timer;
+    let stopped = false;
+    let running = false;
+    /** The last error already reported. Repeats are swallowed so an offline laptop stays quiet. */
+    let reportedError;
+    /**
+     * Whether the last cycle found a link — undefined until the first one has looked.
+     *
+     * An unlinked project used to sync nothing and say nothing, which is indistinguishable from a
+     * working sync until somebody notices the dashboard is hours behind. Observed for real: the
+     * daemon was started from a sibling repo that had never been linked, so every call for an entire
+     * session was recorded into THAT repo's ledger and pushed nowhere, and the first evidence was a
+     * stale number on a dashboard nobody had reason to distrust.
+     *
+     * Announced on the FIRST cycle and on every change after it, so `reticle link` mid-session says
+     * so too. Once per transition, never per tick — a line every minute is a line people stop reading.
+     */
+    let wasLinked;
+    /** One root's bundle. The state, source and sink are already per-root; only the caller was not. */
+    const pushRoot = async (root, cloud) => {
+        if (null === cloud.config)
+            return undefined;
+        const full = diskSource(root);
+        return runSyncCycle({
+            config: cloud.config,
+            source: {
+                runs: () => (cloud.policy.runs ? full.runs() : []),
+                flows: () => (cloud.policy.flows ? full.flows() : []),
+                capsules: () => (cloud.policy.capsules ? full.capsules() : []),
+                derived: (kind) => (cloud.policy.memory ? full.derived(kind) : undefined),
+            },
+            sink: diskSink(root),
+            state: readCloudState(root),
+            now,
+            request,
+        });
+    };
+    /**
+     * Push every other linked root this machine knows about.
+     *
+     * Isolated per root on purpose. A revoked credential, a deleted directory or an unreachable
+     * self-hosted server in ONE repo is a fact about that repo; letting it throw would take the
+     * daemon's own push down with it and turn one broken link into a machine-wide outage.
+     */
+    const syncOtherRoots = async () => {
+        const roots = deps.otherRoots;
+        const cloudFor = deps.cloudFor;
+        if (roots === undefined || cloudFor === undefined)
+            return;
+        let list = [];
+        try {
+            list = await roots();
+        }
+        catch {
+            // Enumeration is best-effort: a registry that cannot be read must not stop this daemon
+            // syncing the root it is standing in.
+            return;
+        }
+        for (const root of list) {
+            if (root === deps.reticleRoot)
+                continue;
+            try {
+                const cloud = await cloudFor(root);
+                if (null === cloud.config)
+                    continue;
+                const report = await pushRoot(root, cloud);
+                if (report === undefined)
+                    continue;
+                if (report.error !== undefined) {
+                    log('reticle_cloud_sync_failed', { root, error: report.error });
+                    continue;
+                }
+                const moved = report.runsSent > 0 ||
+                    report.flowsSent > 0 ||
+                    report.capsulesSent > 0 ||
+                    report.derivedSent.length > 0 ||
+                    report.pulled > 0;
+                // The root is NAMED here and not in the single-root log below, because with several repos
+                // reporting, "synced 3 runs" without a directory is not an answer to "synced from where".
+                if (moved)
+                    log('reticle_cloud_synced', { root, summary: describeSync(report) });
+            }
+            catch (error) {
+                log('reticle_cloud_sync_failed', {
+                    root,
+                    error: error instanceof Error ? error.message : String(error),
+                });
+            }
+        }
+    };
+    const cycle = async () => {
+        // Overlap guard: a slow cycle must not have a second one started on top of it, or two bundles
+        // race and the cursor written by the loser silently rewinds the winner's progress.
+        if (running)
+            return undefined;
+        running = true;
+        try {
+            const cloud = await deps.cloud();
+            const linked = null !== cloud.config;
+            if (linked !== wasLinked) {
+                wasLinked = linked;
+                if (linked)
+                    log('reticle_cloud_linked', {
+                        projectId: cloud.projectId,
+                        root: deps.reticleRoot,
+                    });
+                // Names the ROOT, because the answer is almost always "the daemon is not where you think it
+                // is" — and a message that omits the directory sends people to check their key instead.
+                else
+                    log('reticle_cloud_unlinked', {
+                        root: deps.reticleRoot,
+                        fix: 'run `reticle link` in this directory, or start the daemon in the linked one — nothing is being synced from here',
+                    });
+            }
+            // Every OTHER linked root, first. Their failures are logged and never abort this daemon's own
+            // push: one repo whose credential was revoked must not silence the rest of the machine.
+            await syncOtherRoots();
+            if (null === cloud.config)
+                return undefined;
+            const report = await pushRoot(deps.reticleRoot, cloud);
+            // `pushRoot` returns undefined only for an unlinked root, which the guard above has ruled out.
+            if (report === undefined)
+                return undefined;
+            // Said out loud on EVERY cycle, including the ones that moved nothing and the ones that
+            // failed. The log below is deliberately quiet when idle, which is right for a log and wrong
+            // for a hook: from outside, "nothing changed" and "sync has been broken since Tuesday" are
+            // the same silence, and a consumer that cannot tell them apart has to guess.
+            emitSyncHook({
+                runsPushed: report.runsSent,
+                ok: report.error === undefined,
+                ...(report.error === undefined ? {} : { error: report.error }),
+            });
+            if (report.error !== undefined) {
+                if (report.error !== reportedError) {
+                    reportedError = report.error;
+                    log('reticle_cloud_sync_failed', { error: report.error });
+                }
+            }
+            else {
+                reportedError = undefined;
+                // Only when something actually moved. A per-minute "nothing to send" is noise that trains
+                // people to stop reading the log.
+                const moved = report.runsSent > 0 ||
+                    report.flowsSent > 0 ||
+                    report.capsulesSent > 0 ||
+                    report.derivedSent.length > 0 ||
+                    report.pulled > 0;
+                if (moved)
+                    log('reticle_cloud_synced', { summary: describeSync(report) });
+                /*
+                 * A cycle that moved something means a drive is in progress, so the next one comes sooner;
+                 * one that moved nothing means the machine is idle, so it backs straight off. The rate
+                 * follows the work without needing to be told about sessions, and an idle laptop settles at
+                 * the same cost it had before this existed.
+                 */
+                nextDelay = moved ? activeIntervalMs : intervalMs;
+            }
+            return report;
+        }
+        catch (error) {
+            // Belt and braces: runSyncCycle already swallows, and a throw here would kill the timer.
+            const message = error instanceof Error ? error.message : String(error);
+            if (message !== reportedError) {
+                reportedError = message;
+                log('reticle_cloud_sync_failed', { error: message });
+            }
+            return undefined;
+        }
+        finally {
+            running = false;
+        }
+    };
+    /** The delay for the NEXT cycle: fast while the last one moved something, slow once it stops. */
+    let nextDelay = intervalMs;
+    const schedule = (delay) => {
+        if (stopped)
+            return;
+        // Replace, never stack. `nudge` and the interval both schedule, and leaving the old timer armed
+        // would let every nudge add a permanent extra cycle per minute for the life of the process.
+        if (timer !== undefined)
+            clearTimeout(timer);
+        timer = setTimeout(() => {
+            void cycle().finally(() => schedule(nextDelay));
+        }, delay);
+        // Unref'd: a pending sync must never be the reason a process refuses to exit.
+        timer.unref?.();
+    };
+    schedule(FIRST_CYCLE_DELAY_MS);
+    return {
+        syncNow: cycle,
+        nudge: () => {
+            // A cycle already in flight will not pick this up, so the nudge is still scheduled behind it
+            // rather than dropped — otherwise the run that arrived during a slow cycle waits a full tick.
+            schedule(NUDGE_DELAY_MS);
+        },
+        stop: () => {
+            stopped = true;
+            if (timer !== undefined)
+                clearTimeout(timer);
+            timer = undefined;
+        },
+    };
+}
+/**
+ * Start cloud sync for a daemon and hand back the two hooks the daemon wires to it.
+ *
+ * The wiring lived in `index.ts`, which sits exactly on the thousand-line cap — so the next thing
+ * anyone needed there could not be added without a split, and this is the block that most obviously
+ * did not belong in a bootstrap file. Four calls that are all one concern: what to sync, where else
+ * to look, how a landed run wakes it, and what the panel's button does.
+ *
+ * Returns the daemon itself: the caller still owns when to nudge, when to sync now, and when to
+ * stop. Only the CONSTRUCTION moved, which is the part that was four interdependent arguments about
+ * resolving a credential — and the part a bootstrap file has no business spelling out.
+ */
+export function attachCloudSync(deps) {
+    return startSyncDaemon({
+        reticleRoot: deps.reticleRoot,
+        cloud: () => resolveProjectCloud(deps.fs, deps.reticleRoot, deps.homeDir, deps.env),
+        // Every OTHER linked repo on this machine, not just the directory the daemon was started in.
+        // One daemon serves many projects, and pushing only its own root left the rest silently
+        // reporting nothing — indistinguishable, on the dashboard, from nobody having verified anything.
+        otherRoots: () => Promise.resolve(deps.otherRoots()),
+        cloudFor: (root) => resolveProjectCloud(deps.fs, root, deps.homeDir, deps.env),
+    });
+}

package/dist/memory/cloud/sync-disk.js

@@ -0,0 +1,193 @@
+/**
+ * The filesystem half of a sync cycle: what `.reticle/` looks like to the protocol.
+ *
+ * Kept apart from `sync-cycle.ts` on purpose. The cycle is the interesting part — what to send, what
+ * to skip, what to do when the network is down — and it stays provable only because it never touches
+ * a disk. This is the boring part, and boring is what it should be: read a file, tolerate its
+ * absence, never throw.
+ *
+ * EVERY READ IS FORGIVING. A half-written impact.json from a process that died mid-flush must cost
+ * one unsynced record, not a crashed sync. There is no state here worth defending against a parse
+ * error — the file will be rewritten on the next tool call anyway.
+ */
+import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, writeFileSync, } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { ReticleDir } from '@reticlehq/core';
+import { subjectFor } from '../intent/intent-subject.js';
+const JSON_SUFFIX = '.json';
+/** Read + parse, or undefined for absent/unreadable/malformed. Never throws. */
+function readJson(path) {
+    try {
+        return existsSync(path) ? JSON.parse(readFileSync(path, 'utf8')) : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * Write atomically. tmp + rename, because two daemons can serve one repo at once and a half-written
+ * cursor file is one the next read discards — which would silently re-pull the whole decision
+ * history, or worse, skip it.
+ */
+function writeJson(path, value) {
+    try {
+        mkdirSync(dirname(path), { recursive: true });
+        const tmp = `${path}.${String(process.pid)}.tmp`;
+        writeFileSync(tmp, `${JSON.stringify(value, null, 2)}\n`, 'utf8');
+        renameSync(tmp, path);
+    }
+    catch {
+        // Bookkeeping that cannot be written must never break a sync. The next cycle re-derives it; the
+        // worst case is one redundant upload, which the server dedupes anyway.
+    }
+}
+/** Every `*.json` directly inside a directory, parsed. Absent directory reads as empty. */
+function readJsonDir(dir) {
+    try {
+        if (!existsSync(dir))
+            return [];
+        return readdirSync(dir)
+            .filter((f) => f.endsWith(JSON_SUFFIX))
+            .map((f) => readJson(join(dir, f)))
+            .filter((v) => v !== undefined);
+    }
+    catch {
+        return [];
+    }
+}
+/** Flows live one directory deeper, under the app's own project id. */
+function readFlows(root) {
+    const dir = join(root, ReticleDir.FLOWS_SUBDIR);
+    try {
+        if (!existsSync(dir))
+            return [];
+        return readdirSync(dir).flatMap((scope) => readJsonDir(join(dir, scope)));
+    }
+    catch {
+        return [];
+    }
+}
+const DERIVED_FILE = {
+    impact: ReticleDir.IMPACT_FILE,
+    flake: ReticleDir.FLAKE_FILE,
+    intent: ReticleDir.INTENT_FILE,
+};
+/** The directory the sharded intent store writes into, beside the legacy flat file. */
+const INTENT_SUBDIR = 'intent';
+const INTENT_INDEX = 'index.json';
+/**
+ * Every intent this project holds, flat file and shards merged.
+ *
+ * The store was split into one file per subject so that writing one record stops rewriting all of
+ * them. Sync was not told, and kept reading `intent.json` alone — which the migration deliberately
+ * does not delete. The result was the worst shape a sync bug can take: not an error, but a corpus
+ * that looked healthy on the dashboard and had silently stopped moving, because every intent
+ * captured after the migration stayed on the engineer's laptop.
+ *
+ * Shards WIN on a collision, matching the store's own rule: an id present in both has been migrated
+ * and possibly edited since, and letting the flat copy overwrite that would revert the edit on every
+ * cycle. `index.json` is derived from the shards and is skipped — sending it would duplicate every
+ * record as a summary of itself.
+ *
+ * Returns undefined when there is no memory at all, so a project with nothing to say sends nothing
+ * rather than an empty envelope the server has to store.
+ */
+function readIntent(reticleRoot) {
+    const legacy = readJson(join(reticleRoot, ReticleDir.INTENT_FILE));
+    const merged = {
+        ...(legacy?.intents ?? {}),
+    };
+    let shards = 0;
+    for (const file of safeReaddir(join(reticleRoot, INTENT_SUBDIR))) {
+        if (!file.endsWith(JSON_SUFFIX) || INTENT_INDEX === file)
+            continue;
+        const shard = readJson(join(reticleRoot, INTENT_SUBDIR, file));
+        // A hand-edited or half-written shard costs one subject, never the whole sync.
+        if (shard?.intents === undefined)
+            continue;
+        shards++;
+        for (const [id, record] of Object.entries(shard.intents))
+            merged[id] = record;
+    }
+    if (legacy === undefined && 0 === shards)
+        return undefined;
+    /*
+     * Stamp the SUBJECT before sending, on anything that does not already carry one.
+     *
+     * The subject ladder lives here, in the engine, and knows how to read a flow, a route and a
+     * binding predicate. The server had no access to it and had reimplemented a far weaker version —
+     * flow-or-nothing — so records this ladder files correctly arrived at the dashboard as
+     * `unsorted`. Measured: most of the records the engine can place were landing in the bucket of
+     * last resort, which is what made a coverage map read as one pile with six labels.
+     *
+     * Computed HERE rather than duplicated there, because two rules for one question is how the
+     * answers start disagreeing — and the one with the evidence should be the one that decides.
+     * Records that already name a subject keep it: an agent's explicit choice outranks inference.
+     */
+    const placed = Object.fromEntries(Object.entries(merged).map(([id, record]) => {
+        const held = record;
+        if ('string' === typeof held.subject && held.subject.length > 0)
+            return [id, record];
+        const subject = subjectFor({
+            surface: held.surface,
+            binding: held.binding,
+        });
+        return [id, { ...held, subject }];
+    }));
+    return { version: 1, intents: placed };
+}
+/** Directory listing that tolerates absence — an unmigrated project has no `intent/`. */
+function safeReaddir(dir) {
+    try {
+        return readdirSync(dir);
+    }
+    catch {
+        return [];
+    }
+}
+/** What the cycle reads, backed by a real `.reticle` directory. */
+export function diskSource(reticleRoot) {
+    return {
+        runs: () => readJsonDir(join(reticleRoot, ReticleDir.RUNS_SUBDIR))
+            .map((payload) => {
+            const id = payload?.runId;
+            return 'string' === typeof id ? { runId: id, payload } : undefined;
+        })
+            // A run artifact with no id cannot be diffed against the server's list, so sending it would
+            // mean re-sending it every cycle forever. Dropped rather than uploaded repeatedly.
+            .filter((r) => r !== undefined),
+        flows: () => readFlows(reticleRoot),
+        // Flat directory of `<id>.json`, which is exactly what readJsonDir walks — unlike flows, which
+        // sit one level deeper under the app's project id.
+        capsules: () => readJsonDir(join(reticleRoot, ReticleDir.CAPSULES_SUBDIR)),
+        // Intent is the only derived record that is no longer one file. See readIntent.
+        derived: (kind) => 'intent' === kind ? readIntent(reticleRoot) : readJson(join(reticleRoot, DERIVED_FILE[kind])),
+    };
+}
+/** What the cycle writes back into `.reticle/`. */
+export function diskSink(reticleRoot) {
+    return {
+        writeIssues: (issues) => {
+            // MERGED, not replaced. A pull returns only what changed since the cursor, so overwriting
+            // would drop every earlier decision the moment one new decision arrived.
+            const held = readCloudIssues(reticleRoot);
+            writeJson(join(reticleRoot, ReticleDir.ISSUES_FILE), {
+                triage: { ...held.triage, ...issues.triage },
+            });
+        },
+        writeState: (state) => {
+            writeJson(join(reticleRoot, ReticleDir.CLOUD_STATE_FILE), state);
+        },
+    };
+}
+/** This machine's side of the conversation, or a blank one on first run. */
+export function readCloudState(reticleRoot) {
+    const raw = readJson(join(reticleRoot, ReticleDir.CLOUD_STATE_FILE));
+    return 'object' === typeof raw && null !== raw ? raw : {};
+}
+/** The decisions pulled back from the dashboard, for the HUD and for the next run to read. */
+export function readCloudIssues(reticleRoot) {
+    const raw = readJson(join(reticleRoot, ReticleDir.ISSUES_FILE));
+    const triage = 'object' === typeof raw && null !== raw ? raw.triage : undefined;
+    return { triage: 'object' === typeof triage && null !== triage ? triage : {} };
+}

package/dist/memory/impact/impact-recorder.d.ts

@@ -0,0 +1,54 @@
+import { type ImpactCounts, type ImpactDefect, type ImpactSnapshot } from '@reticlehq/core';
+import type { ConfigSource } from '../cloud/harness-config.js';
+import { ImpactStore, type ImpactFoldMeta } from './impact-store.js';
+/**
+ * Wire the DEFAULT root — the daemon's own. Idempotent; the first one wins.
+ *
+ * Still idempotent, and still first-wins, because the daemon's own root genuinely does not change
+ * while it runs. What changed is that this is no longer the ONLY root: a call that knows its
+ * project passes one, and this is the fallback for calls that do not.
+ */
+export declare function initImpact(opts: {
+    /**
+     * The harness-config source, supplied by whoever wired the daemon.
+     *
+     * Passed in rather than built here: reading the platform lives in `features/harness`, and this
+     * directory may not reach for it. The daemon owns both sides.
+     */
+    config?: ConfigSource;
+    reticleRoot: string | undefined;
+    projectName?: string;
+    now?: () => number;
+}): ImpactStore | undefined;
+/** The live store for a root (default when omitted), if one has been initialised. */
+export declare function impactStore(root?: string): ImpactStore | undefined;
+/** Test seam: drop the stores so a suite can start from a clean record. */
+export declare function resetImpactForTest(): void;
+/**
+ * Record one delta against a project's ledger. Never throws: stats must not break a tool call.
+ *
+ * `root` is the `.reticle` directory this call's evidence belongs to. Omitted means "could not tell
+ * which project", which falls back to the daemon's own root — the old behaviour, kept for the paths
+ * that genuinely have no session to ask.
+ */
+export declare function recordImpact(delta: Partial<ImpactCounts>, meta?: ImpactFoldMeta, root?: string): void;
+export declare function impactSnapshot(root?: string): ImpactSnapshot | undefined;
+/**
+ * What one tool call did, as counters.
+ *
+ * Pure and exported so the shape is testable without a dispatch: given a result and a duration, it
+ * says what the record should gain. `verified` is the only field that can mint a verdict, which is
+ * the same rule the product states to agents - everything else moves or reads the app.
+ */
+export declare function deltaForToolResult(raw: unknown, durationMs: number, refused: boolean): Partial<ImpactCounts>;
+/**
+ * What one FAILED verdict was, in a line.
+ *
+ * Only a verified:"no" produces one. That is the same rule the counters follow and the same rule
+ * the product states to agents: an "unknown" is not a defect, it is Reticle admitting it could not
+ * tell — recording those here would fill the user's short list with the tool's own blind spots.
+ *
+ * Everything is read from the result the tool already returned, so this costs nothing extra and
+ * cannot disagree with the verdict it describes.
+ */
+export declare function defectForToolResult(raw: unknown, at: number): ImpactDefect | undefined;