@reticlehq/server 2.14.0 → 3.2.0

package/dist/memory/intent/inline-intent.js

@@ -0,0 +1,206 @@
+import { createHash } from 'node:crypto';
+import { IntentStore } from './intent-store.js';
+import { sessionRoot } from '../project/session-root.js';
+/**
+ * Intent declared INLINE, on the tool that is already drawing the verdict.
+ *
+ * `reticle_intent` is on the extended surface, so declaring intent costs an agent a discovery, a
+ * decision and a round trip before it has done anything — three places the capture silently fails,
+ * and it does. `reticle_act_and_wait` and `reticle_assert` are in every agent's tool list already,
+ * so one optional argument there makes the declaration free and discoverable by construction.
+ *
+ * This is a shortcut into the EXISTING ledger, never a second one. It writes `.reticle/intent.json`
+ * through `IntentStore` exactly as `reticle_intent { action: "declare" }` does, so a reviewer reads
+ * one file in one vocabulary and cannot tell from the row which door the intent came in by. The
+ * pattern is `flows/flow-intent.ts`, which does the same job for a saved flow.
+ */
+/** Namespaced like `flow:`, so an inline id never collides with one an agent chose by hand. */
+const INLINE_INTENT_ID_PREFIX = 'inline:';
+const ID_SEPARATOR = ':';
+const SLUG_SEPARATOR = '-';
+const SLUG_MAX = 48;
+const HASH_LENGTH = 8;
+const HASH_ALGORITHM = 'sha1';
+const NON_SLUG = /[^a-z0-9]+/g;
+const SLUG_EDGES = /^-+|-+$/g;
+/**
+ * The ledger id a piece of inline prose is declared under.
+ *
+ * Derived from the statement rather than minted, so the same sentence declared on five verdicts is
+ * one row rather than five, and a DIFFERENT sentence is a different row rather than a false
+ * amendment of the first. The slug is there because the id is what an agent later types to reference
+ * the intent, and a bare digest is unreadable; the digest is there because the slug is truncated and
+ * two long statements sharing a prefix must not share a row.
+ */
+function inlineIntentId(statement) {
+    const slug = statement
+        .toLowerCase()
+        .replace(NON_SLUG, SLUG_SEPARATOR)
+        .replace(SLUG_EDGES, '')
+        .slice(0, SLUG_MAX)
+        .replace(SLUG_EDGES, '');
+    const digest = createHash(HASH_ALGORITHM).update(statement).digest('hex').slice(0, HASH_LENGTH);
+    return `${INLINE_INTENT_ID_PREFIX}${slug}${SLUG_SEPARATOR}${digest}`;
+}
+/** What `provenBy` records: which tool drew the verdict, and when. */
+export function inlineVerdictId(tool, at) {
+    return `${tool}${ID_SEPARATOR}${String(at)}`;
+}
+/**
+ * Strip a trailing `:line` from a `file:line` label.
+ *
+ * Anchored to the END and to digits, because a Windows path carries its own colon: splitting on the
+ * first one turns `C:\\app\\src\\cart.tsx:12` into `C`, which is a path to nothing and would be
+ * filed as though it were real.
+ */
+const SOURCE_LINE_SUFFIX = /:\d+$/;
+/**
+ * The surface an inline intent was captured on, so the store can FILE it.
+ *
+ * Without one, almost everything a project knew landed in `unsorted`: `act_and_wait({ intent })`
+ * declared a statement and nothing else, so the subject ladder had no flow, no route and no explicit
+ * subject to work from. A coverage map that is one pile with six labels reads as a team that knows
+ * nothing, when the truth is that it knows a great deal and none of it is filed.
+ *
+ * The ROUTE is what makes this work at all, because it is always available: an agent asserting
+ * something is always somewhere. A flow name is better when there is one, and `subjectFor` already
+ * prefers it — this only has to supply both and let that ladder decide.
+ *
+ * Query and hash are dropped: `/issues?category=severe` and `/issues` are the same subject seen
+ * twice, and keeping the query would shard one area across every filter anybody used.
+ *
+ * Returns undefined rather than an empty object when there is nothing to record, so the result reads
+ * as a capture that looked for a location and found none.
+ */
+export function surfaceForInlineIntent(url, flow, 
+/** `file` or `file:line` for the element the verdict acted on, when it had one. */
+sourceLabel) {
+    let route;
+    if (url !== undefined) {
+        try {
+            const path = new URL(url).pathname;
+            // A bare origin has no path worth filing under: `/` would put every record in one bucket,
+            // which is the problem this exists to fix rather than a solution to it.
+            if (path.length > 1)
+                route = path;
+        }
+        catch {
+            route = undefined;
+        }
+    }
+    /*
+     * The file, without its line.
+     *
+     * A record is about a FILE; the line is where one verdict happened to touch it, and keeping it
+     * would make the same rule look like a different one every time the file was edited above it.
+     */
+    const file = sourceLabel === undefined || 0 === sourceLabel.length
+        ? undefined
+        : sourceLabel.replace(SOURCE_LINE_SUFFIX, '');
+    if (route === undefined && flow === undefined && file === undefined)
+        return undefined;
+    return {
+        ...(route === undefined ? {} : { route }),
+        ...(flow === undefined ? {} : { flow }),
+        ...(file === undefined ? {} : { files: [file] }),
+    };
+}
+/**
+ * The session's live URL, or undefined when no session can be named.
+ *
+ * `sessions.resolve` throws for every "cannot tell which tab" case, and none of them is a reason to
+ * fail the caller's verdict — a missing surface costs a filing, not a result.
+ */
+function sessionUrl(deps, sessionId) {
+    try {
+        return deps.sessions.resolve(sessionId).url;
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * Put an inline intent in the ledger and attach the predicate about to be evaluated. Returns the id
+ * the verdict may later discharge, or undefined when there was no intent to record.
+ *
+ * `intent` carries EITHER prose OR the id of an intent already in the ledger. One field rather than
+ * two because the surface is the scarce thing here, and the two cases are told apart by the ledger
+ * itself: a string that names an existing row is a reference, anything else is prose.
+ *
+ * A referenced intent is never re-declared, which would overwrite the agent's own words with an id,
+ * and its binding is left alone when it already has one — a predicate bound deliberately through
+ * `reticle_intent { action: "bind" }` is a stronger statement than whatever this one call asserts.
+ *
+ * Best-effort by construction: a ledger that cannot be written is a small problem, and a verdict
+ * that fails to return because of one is a large problem.
+ */
+export async function linkInlineIntent(deps, sessionId, intent, 
+/** The predicate this verdict will evaluate, or undefined when it proves nothing (a bare settle). */
+binding) {
+    if (intent === undefined || 0 === intent.trim().length)
+        return undefined;
+    try {
+        const store = new IntentStore(deps.fs, sessionRoot(deps, sessionId), { now: deps.now });
+        const existing = (await store.read()).find((row) => row.id === intent);
+        const id = existing?.id ?? inlineIntentId(intent);
+        // Declared WITHOUT a surface on purpose: see dischargeInlineIntent for why the route that
+        // describes this record only exists after the action it is about.
+        const declaredHere = existing === undefined;
+        if (declaredHere)
+            await store.declare([{ id, statement: intent }]);
+        /*
+         * A binding may be INVENTED only for a row this call also declared.
+         *
+         * Inline, the agent wrote the statement and the predicate in one breath, so pairing them is
+         * recording the agent's own act. A row that ALREADY existed was paired by nothing: the prose was
+         * written at one moment about one thing, and the predicate belongs to whatever this call happens
+         * to assert. Binding them is the ledger inventing a relationship nobody claimed.
+         *
+         * Measured against a live daemon: an intent reading "applying a discount code reduces the order
+         * total shown at checkout" was declared separately, then referenced by id from an act asserting
+         * an unrelated element on another route. It was recorded `proved` at presence grade, with that
+         * unrelated predicate retro-fitted as its binding — a false green in the one layer where prose
+         * meets evidence, and the only place in this product with no honesty check on the join.
+         *
+         * An existing row stays open until `reticle_intent { action: "bind" }` says what would prove it.
+         * That is a real cost — one call — and it is the cost of the ledger meaning anything.
+         */
+        if (binding !== undefined && declaredHere)
+            await store.bind(id, binding);
+        return id;
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * Record that this verdict proved the intent. Call only for a verdict that actually proved something.
+ *
+ * The ledger's own rule does the rest: `dischargeIntent` refuses an intent with no binding, so an
+ * intent declared inline on a wait that asserted no consequence stays open and honest instead of
+ * collecting a proof nothing earned.
+ */
+export async function dischargeInlineIntent(deps, sessionId, id, proof, 
+/** `file:line` of the element this verdict acted on, when there was one. */
+sourceLabel) {
+    if (id === undefined)
+        return;
+    try {
+        const store = new IntentStore(deps.fs, sessionRoot(deps, sessionId), { now: deps.now });
+        /*
+         * File it HERE, not at declaration.
+         *
+         * The intent is declared before the action so a verdict can see it open, and at that instant the
+         * agent is still on the page it is leaving — clicking through to /issues from / would have filed
+         * the record under the page it left, or under nothing at all. The route that describes what the
+         * intent is about is the one that exists once the consequence has landed.
+         */
+        const surface = surfaceForInlineIntent(sessionUrl(deps, sessionId), undefined, sourceLabel);
+        if (surface !== undefined)
+            await store.place(id, surface);
+        await store.discharge(id, proof);
+    }
+    catch {
+        // The verdict already stood. The proof is simply not recorded — see dischargeFlowIntent.
+    }
+}

package/dist/memory/intent/intent-shard-store.d.ts

@@ -0,0 +1,87 @@
+/**
+ * Reading and writing the sharded intent store, and migrating the flat one into it.
+ *
+ * The operations an agent actually performs, and what each is allowed to cost:
+ *
+ *   index()            every id + one line, always affordable — the session-start read
+ *   subject(name)      the full records for one subject, when it is about to work on that
+ *   get(id)            one record, via the index rather than by scanning shards
+ *   record()/update()  a write that touches ONE shard file plus the index
+ *
+ * The last one is the reason for the whole design. Under the flat file, changing one intent rewrote
+ * all 141, so two sessions touching unrelated subjects raced each other and the loser's write
+ * vanished. Sharded, they only collide when they genuinely edit the same subject.
+ *
+ * ## Migration is automatic, and it does not delete
+ *
+ * `.reticle/intent.json` is read on every load and folded in when the sharded copy has nothing for
+ * an id. That makes the move invisible: an agent on the new store sees old intents immediately, and
+ * an older build still reading the flat file keeps working, because nothing removes it. Deleting it
+ * is a separate decision somebody can take once they believe the migration.
+ */
+import { type Intent } from '@reticlehq/core/artifacts';
+import type { FileSystemPort } from '../project/fs/fs-port.js';
+import type { Clock } from '../../machine/clock.js';
+import { IntentStatus, type IntentIndex, type IntentRecord } from './intent-shard.js';
+/** What a caller supplies to write an intent. Everything optional is genuinely optional. */
+interface IntentInput {
+    id: string;
+    statement: string;
+    subject?: string | undefined;
+    why?: string | undefined;
+    source?: string | undefined;
+    status?: IntentStatus | undefined;
+    surface?: Intent['surface'];
+    binding?: unknown;
+}
+export declare class IntentShardStore {
+    #private;
+    constructor(fs: FileSystemPort, root: string, clock: Clock);
+    /**
+     * Every record: the shards, plus any legacy intent the shards have not absorbed.
+     *
+     * Sharded wins on a collision. An id present in both has been migrated and possibly edited since,
+     * and letting the flat copy overwrite that would silently revert the edit.
+     */
+    /**
+     * Every record this ledger holds, from BOTH files, deduped by id.
+     *
+     * Public because the ledger is one logical thing kept in two places, and the tool that offers
+     * `list` has to be able to see all of it. It was private while `record` wrote to shards and `list`
+     * read the flat file, which meant an intent written through one action was invisible to the other
+     * — and an agent told its own recorded intent does not exist concludes the intent does not exist,
+     * not that there are two stores.
+     */
+    all(): Promise<IntentRecord[]>;
+    /**
+     * The cheap read: one line per intent, and the call an agent makes at the start of a session.
+     *
+     * Includes anything still only in the legacy flat file, so the very first call on an unmigrated
+     * project already answers completely — which is what makes the migration invisible rather than a
+     * step somebody has to remember to run.
+     */
+    index(): Promise<IntentIndex>;
+    /** Every record for one subject — the working read, once an agent knows what it is touching. */
+    subject(name: string): Promise<IntentRecord[]>;
+    /** One record by id, or null. */
+    get(id: string): Promise<IntentRecord | null>;
+    /**
+     * Write one intent, creating or updating it.
+     *
+     * Merges onto whatever is already stored, so a caller adding a `why` to a migrated record does not
+     * have to restate its binding — and cannot silently drop it by omission, which is the failure mode
+     * that makes agents afraid to touch a store.
+     */
+    record(input: IntentInput): Promise<IntentRecord>;
+    /**
+     * Fold the flat file into shards on disk, so the move is visible in a diff rather than implicit.
+     *
+     * Idempotent, and it does not delete `.reticle/intent.json`: an older build still reads it, and a
+     * migration that removes its own source cannot be checked afterwards.
+     */
+    migrate(): Promise<{
+        migrated: number;
+        subjects: string[];
+    }>;
+}
+export {};

package/dist/memory/intent/intent-shard-store.js

@@ -0,0 +1,215 @@
+/**
+ * Reading and writing the sharded intent store, and migrating the flat one into it.
+ *
+ * The operations an agent actually performs, and what each is allowed to cost:
+ *
+ *   index()            every id + one line, always affordable — the session-start read
+ *   subject(name)      the full records for one subject, when it is about to work on that
+ *   get(id)            one record, via the index rather than by scanning shards
+ *   record()/update()  a write that touches ONE shard file plus the index
+ *
+ * The last one is the reason for the whole design. Under the flat file, changing one intent rewrote
+ * all 141, so two sessions touching unrelated subjects raced each other and the loser's write
+ * vanished. Sharded, they only collide when they genuinely edit the same subject.
+ *
+ * ## Migration is automatic, and it does not delete
+ *
+ * `.reticle/intent.json` is read on every load and folded in when the sharded copy has nothing for
+ * an id. That makes the move invisible: an agent on the new store sees old intents immediately, and
+ * an older build still reading the flat file keeps working, because nothing removes it. Deleting it
+ * is a separate decision somebody can take once they believe the migration.
+ */
+import { parseIntentFile } from '@reticlehq/core/artifacts';
+import { ReticleDir } from '@reticlehq/core';
+import { reticleDirPaths } from '../project/dir/reticle-dir.js';
+import { withFileLock } from '../project/file-lock.js';
+import { subjectFor } from './intent-subject.js';
+import { emptyShard, indexFrom, IntentShardSchema, IntentStatus, recordFromIntent, serialise, shardsFrom, readJsonFile, } from './intent-shard.js';
+const INDEX_FILE = 'index.json';
+const SHARD_SUFFIX = '.json';
+export class IntentShardStore {
+    #fs;
+    #root;
+    #clock;
+    constructor(fs, root, clock) {
+        this.#fs = fs;
+        this.#root = root;
+        this.#clock = clock;
+    }
+    #dir() {
+        return `${reticleDirPaths(this.#root).root}/${ReticleDir.INTENT_SUBDIR}`;
+    }
+    #indexPath() {
+        return `${this.#dir()}/${INDEX_FILE}`;
+    }
+    #shardPath(subject) {
+        return `${this.#dir()}/${subject}${SHARD_SUFFIX}`;
+    }
+    /** Legacy records, keyed by id. Empty when there is no flat file, which is the eventual case. */
+    async #legacy() {
+        const file = await readJsonFile(this.#fs, reticleDirPaths(this.#root).intent, (raw) => parseIntentFile(raw), { version: 1, intents: {} });
+        return new Map(Object.values(file.intents).map((i) => [i.id, recordFromIntent(i)]));
+    }
+    async #readShard(subject) {
+        return readJsonFile(this.#fs, this.#shardPath(subject), (raw) => IntentShardSchema.parse(raw), emptyShard(subject));
+    }
+    /**
+     * Which subjects exist, read from the DIRECTORY rather than the index.
+     *
+     * The index cannot be the source here. It is derived from the shards, so discovering shards
+     * through it is circular — and the circle is not theoretical: a brand-new subject was invisible to
+     * the very write that created it, because its shard existed on disk while the index did not
+     * mention it yet. Listing the directory also makes the index self-healing: delete it and the next
+     * read rebuilds it from what is actually there.
+     */
+    async #subjectsOnDisk() {
+        try {
+            const entries = await this.#fs.readdir(this.#dir());
+            return entries
+                .filter((e) => e.endsWith(SHARD_SUFFIX) && INDEX_FILE !== e)
+                .map((e) => e.slice(0, -SHARD_SUFFIX.length));
+        }
+        catch {
+            // No directory yet — an unmigrated project, which the legacy fold below still answers for.
+            return [];
+        }
+    }
+    /**
+     * Every record: the shards, plus any legacy intent the shards have not absorbed.
+     *
+     * Sharded wins on a collision. An id present in both has been migrated and possibly edited since,
+     * and letting the flat copy overwrite that would silently revert the edit.
+     */
+    /**
+     * Every record this ledger holds, from BOTH files, deduped by id.
+     *
+     * Public because the ledger is one logical thing kept in two places, and the tool that offers
+     * `list` has to be able to see all of it. It was private while `record` wrote to shards and `list`
+     * read the flat file, which meant an intent written through one action was invisible to the other
+     * — and an agent told its own recorded intent does not exist concludes the intent does not exist,
+     * not that there are two stores.
+     */
+    async all() {
+        return this.#all();
+    }
+    async #all() {
+        const shards = await Promise.all((await this.#subjectsOnDisk()).map((s) => this.#readShard(s)));
+        const merged = new Map();
+        for (const [id, record] of await this.#legacy())
+            merged.set(id, record);
+        for (const shard of shards) {
+            for (const record of Object.values(shard.intents))
+                merged.set(record.id, record);
+        }
+        return [...merged.values()];
+    }
+    /**
+     * The cheap read: one line per intent, and the call an agent makes at the start of a session.
+     *
+     * Includes anything still only in the legacy flat file, so the very first call on an unmigrated
+     * project already answers completely — which is what makes the migration invisible rather than a
+     * step somebody has to remember to run.
+     */
+    async index() {
+        // Derived from what is on disk, never from the stored copy. The stored index is a cache for
+        // readers outside this process; inside it, trusting it would let a stale one hide a real record.
+        return indexFrom(shardsFrom(await this.#all()));
+    }
+    /** Records already in shards, ignoring the legacy file — what migration diffs against. */
+    async #storedRecords() {
+        const shards = await Promise.all((await this.#subjectsOnDisk()).map((s) => this.#readShard(s)));
+        return shards.flatMap((s) => Object.values(s.intents));
+    }
+    /** Every record for one subject — the working read, once an agent knows what it is touching. */
+    async subject(name) {
+        const all = await this.#all();
+        return all.filter((r) => name === r.subject);
+    }
+    /** One record by id, or null. */
+    async get(id) {
+        const all = await this.#all();
+        return all.find((r) => id === r.id) ?? null;
+    }
+    /**
+     * Write one intent, creating or updating it.
+     *
+     * Merges onto whatever is already stored, so a caller adding a `why` to a migrated record does not
+     * have to restate its binding — and cannot silently drop it by omission, which is the failure mode
+     * that makes agents afraid to touch a store.
+     */
+    async record(input) {
+        return withFileLock(this.#indexPath(), async () => {
+            const existing = await this.get(input.id);
+            const now = this.#clock.now();
+            const subject = input.subject ??
+                existing?.subject ??
+                subjectFor({ surface: input.surface, binding: input.binding });
+            const next = {
+                ...(existing ?? {
+                    id: input.id,
+                    statement: input.statement,
+                    state: 'declared',
+                    declaredAt: now,
+                    subject,
+                    status: IntentStatus.PROPOSED,
+                }),
+                statement: input.statement,
+                subject,
+                status: input.status ?? existing?.status ?? IntentStatus.PROPOSED,
+                updatedAt: now,
+                ...(input.why === undefined ? {} : { why: input.why }),
+                ...(input.source === undefined ? {} : { source: input.source }),
+                ...(input.surface === undefined ? {} : { surface: input.surface }),
+                ...(input.binding === undefined ? {} : { binding: input.binding }),
+            };
+            await this.#write(next, existing?.subject);
+            return next;
+        });
+    }
+    /** Persist one record, rewriting its shard and the index — and the OLD shard if it moved. */
+    async #write(record, previousSubject) {
+        await this.#fs.mkdir(this.#dir());
+        if (previousSubject !== undefined && record.subject !== previousSubject) {
+            const old = await this.#readShard(previousSubject);
+            if (old.intents[record.id] !== undefined) {
+                const { [record.id]: _moved, ...rest } = old.intents;
+                await this.#fs.writeFile(this.#shardPath(previousSubject), serialise({ ...old, intents: rest }));
+            }
+        }
+        const shard = await this.#readShard(record.subject);
+        shard.intents[record.id] = record;
+        await this.#fs.writeFile(this.#shardPath(record.subject), serialise(shard));
+        // The index is DERIVED, never edited in place: two sources of truth would disagree, and the one
+        // that is cheap to read is the one people would trust.
+        const all = await this.#all();
+        await this.#fs.writeFile(this.#indexPath(), serialise(indexFrom(shardsFrom(all))));
+    }
+    /**
+     * Fold the flat file into shards on disk, so the move is visible in a diff rather than implicit.
+     *
+     * Idempotent, and it does not delete `.reticle/intent.json`: an older build still reads it, and a
+     * migration that removes its own source cannot be checked afterwards.
+     */
+    async migrate() {
+        return withFileLock(this.#indexPath(), async () => {
+            const legacy = await this.#legacy();
+            if (0 === legacy.size)
+                return { migrated: 0, subjects: [] };
+            const stored = await this.#storedRecords();
+            const known = new Set(stored.map((r) => r.id));
+            const incoming = [...legacy.values()].filter((r) => !known.has(r.id));
+            if (0 === incoming.length)
+                return { migrated: 0, subjects: [] };
+            const shards = shardsFrom([...stored, ...incoming]);
+            await this.#fs.mkdir(this.#dir());
+            for (const shard of shards) {
+                await this.#fs.writeFile(this.#shardPath(shard.subject), serialise(shard));
+            }
+            await this.#fs.writeFile(this.#indexPath(), serialise(indexFrom(shards)));
+            return {
+                migrated: incoming.length,
+                subjects: [...new Set(incoming.map((r) => r.subject))].sort(),
+            };
+        });
+    }
+}