@reticlehq/server 2.12.0 → 2.13.1

package/dist/intent/intent-shard-store.js

@@ -0,0 +1,203 @@
+/**
+ * Reading and writing the sharded intent store, and migrating the flat one into it.
+ *
+ * The operations an agent actually performs, and what each is allowed to cost:
+ *
+ *   index()            every id + one line, always affordable — the session-start read
+ *   subject(name)      the full records for one subject, when it is about to work on that
+ *   get(id)            one record, via the index rather than by scanning shards
+ *   record()/update()  a write that touches ONE shard file plus the index
+ *
+ * The last one is the reason for the whole design. Under the flat file, changing one intent rewrote
+ * all 141, so two sessions touching unrelated subjects raced each other and the loser's write
+ * vanished. Sharded, they only collide when they genuinely edit the same subject.
+ *
+ * ## Migration is automatic, and it does not delete
+ *
+ * `.reticle/intent.json` is read on every load and folded in when the sharded copy has nothing for
+ * an id. That makes the move invisible: an agent on the new store sees old intents immediately, and
+ * an older build still reading the flat file keeps working, because nothing removes it. Deleting it
+ * is a separate decision somebody can take once they believe the migration.
+ */
+import { parseIntentFile } from '@reticlehq/core';
+import { reticleDirPaths } from '../project/reticle-dir.js';
+import { withFileLock } from '../project/file-lock.js';
+import { subjectFor } from './intent-subject.js';
+import { emptyShard, indexFrom, IntentShardSchema, IntentStatus, recordFromIntent, serialise, shardsFrom, readJsonFile, } from './intent-shard.js';
+const INTENT_DIR = 'intent';
+const INDEX_FILE = 'index.json';
+const SHARD_SUFFIX = '.json';
+export class IntentShardStore {
+    #fs;
+    #root;
+    #clock;
+    constructor(fs, root, clock) {
+        this.#fs = fs;
+        this.#root = root;
+        this.#clock = clock;
+    }
+    #dir() {
+        return `${reticleDirPaths(this.#root).root}/${INTENT_DIR}`;
+    }
+    #indexPath() {
+        return `${this.#dir()}/${INDEX_FILE}`;
+    }
+    #shardPath(subject) {
+        return `${this.#dir()}/${subject}${SHARD_SUFFIX}`;
+    }
+    /** Legacy records, keyed by id. Empty when there is no flat file, which is the eventual case. */
+    async #legacy() {
+        const file = await readJsonFile(this.#fs, reticleDirPaths(this.#root).intent, (raw) => parseIntentFile(raw), { version: 1, intents: {} });
+        return new Map(Object.values(file.intents).map((i) => [i.id, recordFromIntent(i)]));
+    }
+    async #readShard(subject) {
+        return readJsonFile(this.#fs, this.#shardPath(subject), (raw) => IntentShardSchema.parse(raw), emptyShard(subject));
+    }
+    /**
+     * Which subjects exist, read from the DIRECTORY rather than the index.
+     *
+     * The index cannot be the source here. It is derived from the shards, so discovering shards
+     * through it is circular — and the circle is not theoretical: a brand-new subject was invisible to
+     * the very write that created it, because its shard existed on disk while the index did not
+     * mention it yet. Listing the directory also makes the index self-healing: delete it and the next
+     * read rebuilds it from what is actually there.
+     */
+    async #subjectsOnDisk() {
+        try {
+            const entries = await this.#fs.readdir(this.#dir());
+            return entries
+                .filter((e) => e.endsWith(SHARD_SUFFIX) && INDEX_FILE !== e)
+                .map((e) => e.slice(0, -SHARD_SUFFIX.length));
+        }
+        catch {
+            // No directory yet — an unmigrated project, which the legacy fold below still answers for.
+            return [];
+        }
+    }
+    /**
+     * Every record: the shards, plus any legacy intent the shards have not absorbed.
+     *
+     * Sharded wins on a collision. An id present in both has been migrated and possibly edited since,
+     * and letting the flat copy overwrite that would silently revert the edit.
+     */
+    async #all() {
+        const shards = await Promise.all((await this.#subjectsOnDisk()).map((s) => this.#readShard(s)));
+        const merged = new Map();
+        for (const [id, record] of await this.#legacy())
+            merged.set(id, record);
+        for (const shard of shards) {
+            for (const record of Object.values(shard.intents))
+                merged.set(record.id, record);
+        }
+        return [...merged.values()];
+    }
+    /**
+     * The cheap read: one line per intent, and the call an agent makes at the start of a session.
+     *
+     * Includes anything still only in the legacy flat file, so the very first call on an unmigrated
+     * project already answers completely — which is what makes the migration invisible rather than a
+     * step somebody has to remember to run.
+     */
+    async index() {
+        // Derived from what is on disk, never from the stored copy. The stored index is a cache for
+        // readers outside this process; inside it, trusting it would let a stale one hide a real record.
+        return indexFrom(shardsFrom(await this.#all()));
+    }
+    /** Records already in shards, ignoring the legacy file — what migration diffs against. */
+    async #storedRecords() {
+        const shards = await Promise.all((await this.#subjectsOnDisk()).map((s) => this.#readShard(s)));
+        return shards.flatMap((s) => Object.values(s.intents));
+    }
+    /** Every record for one subject — the working read, once an agent knows what it is touching. */
+    async subject(name) {
+        const all = await this.#all();
+        return all.filter((r) => name === r.subject);
+    }
+    /** One record by id, or null. */
+    async get(id) {
+        const all = await this.#all();
+        return all.find((r) => id === r.id) ?? null;
+    }
+    /**
+     * Write one intent, creating or updating it.
+     *
+     * Merges onto whatever is already stored, so a caller adding a `why` to a migrated record does not
+     * have to restate its binding — and cannot silently drop it by omission, which is the failure mode
+     * that makes agents afraid to touch a store.
+     */
+    async record(input) {
+        return withFileLock(this.#indexPath(), async () => {
+            const existing = await this.get(input.id);
+            const now = this.#clock.now();
+            const subject = input.subject ??
+                existing?.subject ??
+                subjectFor({ surface: input.surface, binding: input.binding });
+            const next = {
+                ...(existing ?? {
+                    id: input.id,
+                    statement: input.statement,
+                    state: 'declared',
+                    declaredAt: now,
+                    subject,
+                    status: IntentStatus.PROPOSED,
+                }),
+                statement: input.statement,
+                subject,
+                status: input.status ?? existing?.status ?? IntentStatus.PROPOSED,
+                updatedAt: now,
+                ...(input.why === undefined ? {} : { why: input.why }),
+                ...(input.source === undefined ? {} : { source: input.source }),
+                ...(input.surface === undefined ? {} : { surface: input.surface }),
+                ...(input.binding === undefined ? {} : { binding: input.binding }),
+            };
+            await this.#write(next, existing?.subject);
+            return next;
+        });
+    }
+    /** Persist one record, rewriting its shard and the index — and the OLD shard if it moved. */
+    async #write(record, previousSubject) {
+        await this.#fs.mkdir(this.#dir());
+        if (previousSubject !== undefined && record.subject !== previousSubject) {
+            const old = await this.#readShard(previousSubject);
+            if (old.intents[record.id] !== undefined) {
+                const { [record.id]: _moved, ...rest } = old.intents;
+                await this.#fs.writeFile(this.#shardPath(previousSubject), serialise({ ...old, intents: rest }));
+            }
+        }
+        const shard = await this.#readShard(record.subject);
+        shard.intents[record.id] = record;
+        await this.#fs.writeFile(this.#shardPath(record.subject), serialise(shard));
+        // The index is DERIVED, never edited in place: two sources of truth would disagree, and the one
+        // that is cheap to read is the one people would trust.
+        const all = await this.#all();
+        await this.#fs.writeFile(this.#indexPath(), serialise(indexFrom(shardsFrom(all))));
+    }
+    /**
+     * Fold the flat file into shards on disk, so the move is visible in a diff rather than implicit.
+     *
+     * Idempotent, and it does not delete `.reticle/intent.json`: an older build still reads it, and a
+     * migration that removes its own source cannot be checked afterwards.
+     */
+    async migrate() {
+        return withFileLock(this.#indexPath(), async () => {
+            const legacy = await this.#legacy();
+            if (0 === legacy.size)
+                return { migrated: 0, subjects: [] };
+            const stored = await this.#storedRecords();
+            const known = new Set(stored.map((r) => r.id));
+            const incoming = [...legacy.values()].filter((r) => !known.has(r.id));
+            if (0 === incoming.length)
+                return { migrated: 0, subjects: [] };
+            const shards = shardsFrom([...stored, ...incoming]);
+            await this.#fs.mkdir(this.#dir());
+            for (const shard of shards) {
+                await this.#fs.writeFile(this.#shardPath(shard.subject), serialise(shard));
+            }
+            await this.#fs.writeFile(this.#indexPath(), serialise(indexFrom(shards)));
+            return {
+                migrated: incoming.length,
+                subjects: [...new Set(incoming.map((r) => r.subject))].sort(),
+            };
+        });
+    }
+}

package/dist/intent/intent-shard.d.ts

@@ -0,0 +1,411 @@
+/**
+ * The sharded intent store: one small file per subject, plus an index cheap enough to always load.
+ *
+ * ## Why this exists
+ *
+ * `.reticle/intent.json` was a single object that reached 141 entries and 109KB. Parsing that is
+ * nothing; READING it is the cost, and reading is what this store is for. An agent wanting the two
+ * rules about checkout pulled 139 irrelevant ones through its context to find them, and an agent
+ * changing one rewrote the whole file — which is both slow to review and a lost write whenever two
+ * sessions touch different subjects at once.
+ *
+ * ## The shape
+ *
+ *   .reticle/intent/index.json     every id, its subject and one line of prose
+ *   .reticle/intent/<subject>.json the full records for that subject
+ *
+ * The index is the whole point: it answers "what do we know, and where does it live?" without
+ * opening anything, and detail is then fetched for the one subject in play. Measured on the real
+ * corpus, that is 109KB all-or-nothing becoming an index plus one shard — and the largest shard is
+ * under 10KB once the unsorted bucket is set aside.
+ *
+ * The index is NOT tiny, and measuring beat guessing twice over. With full statements it came to
+ * 36KB for 141 entries; summarising them saved 7%, because the bulk is not the prose but the ids —
+ * `inline:signing-in-with-the-email-password-created-earli-6ca804e9` is 63 characters before the
+ * statement starts. So the honest numbers are: 109KB all-or-nothing becomes a 33KB index plus one
+ * ~10KB shard, and the real win is that a READ and a WRITE are both bounded to one subject rather
+ * than the whole corpus.
+ *
+ * The index is therefore the part that does not scale forever. At ten times this size it needs a
+ * second tier — subjects and counts, with ids fetched per subject — rather than a longer single
+ * read. Recorded here because the next person will hit it, and the shape of the fix is already
+ * visible.
+ *
+ * ## What the records carry that the old ones did not
+ *
+ * The legacy file was written BY verification, AT verification time, in verification's vocabulary: a
+ * statement, a predicate, and whether a verdict discharged it. Of 141 entries, four named a surface.
+ * There was nowhere to put the reason a thing must be true, who decided it, or when — so the
+ * business case somebody states once and then forgets had no home and was never captured.
+ *
+ * `why`, `source` and `subject` are that home. They are optional because a migrated record cannot
+ * invent them, and a store that rejected incomplete records would simply not be written to.
+ */
+import { z } from 'zod';
+import { type Intent } from '@reticlehq/core';
+import type { FileSystemPort } from '../project/fs-port.js';
+export declare const INTENT_SHARD_VERSION = 1;
+/**
+ * How settled a record is.
+ *
+ * Separate from the legacy `state`, which only ever described the VERIFICATION lifecycle
+ * (declared → bound → proved). A rule can be agreed with the customer months before anything can
+ * prove it, and collapsing those two axes is what made the old file an assertion log rather than a
+ * memory.
+ */
+export declare const IntentStatus: {
+    /** Captured, not yet confirmed by anyone. What an agent writes while building. */
+    readonly PROPOSED: "proposed";
+    /** Confirmed as something that must hold. May have no way to prove it yet. */
+    readonly AGREED: "agreed";
+    /** A verdict has actually shown it holds. */
+    readonly PROVED: "proved";
+    /** Believed out of date — kept, because deleting the record loses the fact it was ever true. */
+    readonly STALE: "stale";
+};
+export type IntentStatus = (typeof IntentStatus)[keyof typeof IntentStatus];
+export declare const IntentRecordSchema: z.ZodObject<{
+    id: z.ZodString;
+    statement: z.ZodString;
+    state: z.ZodEnum<["declared", "bound", "proved"]>;
+    declaredAt: z.ZodNumber;
+    binding: z.ZodOptional<z.ZodUnknown>;
+    surface: z.ZodOptional<z.ZodObject<{
+        route: z.ZodOptional<z.ZodString>;
+        flow: z.ZodOptional<z.ZodString>;
+        files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    }, "strip", z.ZodTypeAny, {
+        flow?: string | undefined;
+        route?: string | undefined;
+        files?: string[] | undefined;
+    }, {
+        flow?: string | undefined;
+        route?: string | undefined;
+        files?: string[] | undefined;
+    }>>;
+    provenBy: z.ZodOptional<z.ZodObject<{
+        verdictId: z.ZodString;
+        grade: z.ZodString;
+        at: z.ZodNumber;
+    }, "strip", z.ZodTypeAny, {
+        at: number;
+        verdictId: string;
+        grade: string;
+    }, {
+        at: number;
+        verdictId: string;
+        grade: string;
+    }>>;
+    amended: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        statement: z.ZodString;
+        at: z.ZodNumber;
+    }, "strip", z.ZodTypeAny, {
+        at: number;
+        statement: string;
+    }, {
+        at: number;
+        statement: string;
+    }>, "many">>;
+} & {
+    /** Which shard this lives in. Stored as well as implied, so a file read alone is self-describing. */
+    subject: z.ZodString;
+    status: z.ZodEnum<["proposed", "agreed", "proved", "stale"]>;
+    /** WHY it must be true. The half the old file had no room for. */
+    why: z.ZodOptional<z.ZodString>;
+    /** Where it came from — "the user, in conversation" beats an anonymous assertion in six months. */
+    source: z.ZodOptional<z.ZodString>;
+    /** When it was last touched, so a stale-looking record can be told from an untouched one. */
+    updatedAt: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    status: "proved" | "stale" | "proposed" | "agreed";
+    state: "proved" | "declared" | "bound";
+    subject: string;
+    id: string;
+    statement: string;
+    declaredAt: number;
+    surface?: {
+        flow?: string | undefined;
+        route?: string | undefined;
+        files?: string[] | undefined;
+    } | undefined;
+    source?: string | undefined;
+    why?: string | undefined;
+    updatedAt?: number | undefined;
+    binding?: unknown;
+    provenBy?: {
+        at: number;
+        verdictId: string;
+        grade: string;
+    } | undefined;
+    amended?: {
+        at: number;
+        statement: string;
+    }[] | undefined;
+}, {
+    status: "proved" | "stale" | "proposed" | "agreed";
+    state: "proved" | "declared" | "bound";
+    subject: string;
+    id: string;
+    statement: string;
+    declaredAt: number;
+    surface?: {
+        flow?: string | undefined;
+        route?: string | undefined;
+        files?: string[] | undefined;
+    } | undefined;
+    source?: string | undefined;
+    why?: string | undefined;
+    updatedAt?: number | undefined;
+    binding?: unknown;
+    provenBy?: {
+        at: number;
+        verdictId: string;
+        grade: string;
+    } | undefined;
+    amended?: {
+        at: number;
+        statement: string;
+    }[] | undefined;
+}>;
+export type IntentRecord = z.infer<typeof IntentRecordSchema>;
+export declare const IntentShardSchema: z.ZodObject<{
+    version: z.ZodLiteral<1>;
+    subject: z.ZodString;
+    intents: z.ZodRecord<z.ZodString, z.ZodObject<{
+        id: z.ZodString;
+        statement: z.ZodString;
+        state: z.ZodEnum<["declared", "bound", "proved"]>;
+        declaredAt: z.ZodNumber;
+        binding: z.ZodOptional<z.ZodUnknown>;
+        surface: z.ZodOptional<z.ZodObject<{
+            route: z.ZodOptional<z.ZodString>;
+            flow: z.ZodOptional<z.ZodString>;
+            files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        }, "strip", z.ZodTypeAny, {
+            flow?: string | undefined;
+            route?: string | undefined;
+            files?: string[] | undefined;
+        }, {
+            flow?: string | undefined;
+            route?: string | undefined;
+            files?: string[] | undefined;
+        }>>;
+        provenBy: z.ZodOptional<z.ZodObject<{
+            verdictId: z.ZodString;
+            grade: z.ZodString;
+            at: z.ZodNumber;
+        }, "strip", z.ZodTypeAny, {
+            at: number;
+            verdictId: string;
+            grade: string;
+        }, {
+            at: number;
+            verdictId: string;
+            grade: string;
+        }>>;
+        amended: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            statement: z.ZodString;
+            at: z.ZodNumber;
+        }, "strip", z.ZodTypeAny, {
+            at: number;
+            statement: string;
+        }, {
+            at: number;
+            statement: string;
+        }>, "many">>;
+    } & {
+        /** Which shard this lives in. Stored as well as implied, so a file read alone is self-describing. */
+        subject: z.ZodString;
+        status: z.ZodEnum<["proposed", "agreed", "proved", "stale"]>;
+        /** WHY it must be true. The half the old file had no room for. */
+        why: z.ZodOptional<z.ZodString>;
+        /** Where it came from — "the user, in conversation" beats an anonymous assertion in six months. */
+        source: z.ZodOptional<z.ZodString>;
+        /** When it was last touched, so a stale-looking record can be told from an untouched one. */
+        updatedAt: z.ZodOptional<z.ZodNumber>;
+    }, "strip", z.ZodTypeAny, {
+        status: "proved" | "stale" | "proposed" | "agreed";
+        state: "proved" | "declared" | "bound";
+        subject: string;
+        id: string;
+        statement: string;
+        declaredAt: number;
+        surface?: {
+            flow?: string | undefined;
+            route?: string | undefined;
+            files?: string[] | undefined;
+        } | undefined;
+        source?: string | undefined;
+        why?: string | undefined;
+        updatedAt?: number | undefined;
+        binding?: unknown;
+        provenBy?: {
+            at: number;
+            verdictId: string;
+            grade: string;
+        } | undefined;
+        amended?: {
+            at: number;
+            statement: string;
+        }[] | undefined;
+    }, {
+        status: "proved" | "stale" | "proposed" | "agreed";
+        state: "proved" | "declared" | "bound";
+        subject: string;
+        id: string;
+        statement: string;
+        declaredAt: number;
+        surface?: {
+            flow?: string | undefined;
+            route?: string | undefined;
+            files?: string[] | undefined;
+        } | undefined;
+        source?: string | undefined;
+        why?: string | undefined;
+        updatedAt?: number | undefined;
+        binding?: unknown;
+        provenBy?: {
+            at: number;
+            verdictId: string;
+            grade: string;
+        } | undefined;
+        amended?: {
+            at: number;
+            statement: string;
+        }[] | undefined;
+    }>>;
+}, "strip", z.ZodTypeAny, {
+    subject: string;
+    intents: Record<string, {
+        status: "proved" | "stale" | "proposed" | "agreed";
+        state: "proved" | "declared" | "bound";
+        subject: string;
+        id: string;
+        statement: string;
+        declaredAt: number;
+        surface?: {
+            flow?: string | undefined;
+            route?: string | undefined;
+            files?: string[] | undefined;
+        } | undefined;
+        source?: string | undefined;
+        why?: string | undefined;
+        updatedAt?: number | undefined;
+        binding?: unknown;
+        provenBy?: {
+            at: number;
+            verdictId: string;
+            grade: string;
+        } | undefined;
+        amended?: {
+            at: number;
+            statement: string;
+        }[] | undefined;
+    }>;
+    version: 1;
+}, {
+    subject: string;
+    intents: Record<string, {
+        status: "proved" | "stale" | "proposed" | "agreed";
+        state: "proved" | "declared" | "bound";
+        subject: string;
+        id: string;
+        statement: string;
+        declaredAt: number;
+        surface?: {
+            flow?: string | undefined;
+            route?: string | undefined;
+            files?: string[] | undefined;
+        } | undefined;
+        source?: string | undefined;
+        why?: string | undefined;
+        updatedAt?: number | undefined;
+        binding?: unknown;
+        provenBy?: {
+            at: number;
+            verdictId: string;
+            grade: string;
+        } | undefined;
+        amended?: {
+            at: number;
+            statement: string;
+        }[] | undefined;
+    }>;
+    version: 1;
+}>;
+export type IntentShard = z.infer<typeof IntentShardSchema>;
+export declare const summarise: (statement: string) => string;
+/** One line per intent: enough to decide whether to open the shard, and no more. */
+export declare const IntentIndexEntrySchema: z.ZodObject<{
+    id: z.ZodString;
+    subject: z.ZodString;
+    statement: z.ZodString;
+    status: z.ZodEnum<["proposed", "agreed", "proved", "stale"]>;
+}, "strip", z.ZodTypeAny, {
+    status: "proved" | "stale" | "proposed" | "agreed";
+    subject: string;
+    id: string;
+    statement: string;
+}, {
+    status: "proved" | "stale" | "proposed" | "agreed";
+    subject: string;
+    id: string;
+    statement: string;
+}>;
+export type IntentIndexEntry = z.infer<typeof IntentIndexEntrySchema>;
+export declare const IntentIndexSchema: z.ZodObject<{
+    version: z.ZodLiteral<1>;
+    entries: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        subject: z.ZodString;
+        statement: z.ZodString;
+        status: z.ZodEnum<["proposed", "agreed", "proved", "stale"]>;
+    }, "strip", z.ZodTypeAny, {
+        status: "proved" | "stale" | "proposed" | "agreed";
+        subject: string;
+        id: string;
+        statement: string;
+    }, {
+        status: "proved" | "stale" | "proposed" | "agreed";
+        subject: string;
+        id: string;
+        statement: string;
+    }>, "many">;
+}, "strip", z.ZodTypeAny, {
+    entries: {
+        status: "proved" | "stale" | "proposed" | "agreed";
+        subject: string;
+        id: string;
+        statement: string;
+    }[];
+    version: 1;
+}, {
+    entries: {
+        status: "proved" | "stale" | "proposed" | "agreed";
+        subject: string;
+        id: string;
+        statement: string;
+    }[];
+    version: 1;
+}>;
+export type IntentIndex = z.infer<typeof IntentIndexSchema>;
+export declare const emptyShard: (subject: string) => IntentShard;
+export declare const emptyIndex: () => IntentIndex;
+/** The verification lifecycle a legacy record carried, mapped onto the settledness axis. */
+export declare const statusFromState: (state: Intent["state"]) => IntentStatus;
+/**
+ * Turn a legacy intent into a record, inferring only what can be inferred.
+ *
+ * `why` and `source` are left ABSENT rather than filled with the statement or a placeholder. A
+ * record that claims to carry a reason it does not have is worse than one that admits the gap: the
+ * gap is a prompt to write the reason down, and a placeholder is a reason to stop looking.
+ */
+export declare const recordFromIntent: (intent: Intent) => IntentRecord;
+/** Build the index from the shards. Derived, never hand-maintained — two sources would disagree. */
+export declare const indexFrom: (shards: readonly IntentShard[]) => IntentIndex;
+/** Group records into shards by their own subject. */
+export declare const shardsFrom: (records: readonly IntentRecord[]) => IntentShard[];
+/** Serialise byte-stably, with keys sorted so a re-save of unchanged data is a no-op diff. */
+export declare const serialise: (value: unknown) => string;
+/** Read + validate one JSON file, failing soft: a hand-merged file with a conflict marker is real. */
+export declare const readJsonFile: <T>(fs: FileSystemPort, path: string, parse: (raw: unknown) => T, fallback: T) => Promise<T>;