@reticlehq/server 3.5.0 → 3.7.0

package/dist/features/harness/goals.js

@@ -0,0 +1,330 @@
+/**
+ * The goals a drive was asked to prove, and whether it did.
+ *
+ * A drive asked to "check that "Ada Lovelace" and "Grace Hopper" are listed" replayed old journeys,
+ * dismissed a reminder and reported `finished` without looking for either name (#1316). The driver's
+ * word for it is not evidence, so after every drive the harness checks each goal itself, with the
+ * same assert any agent would make, and reports one verdict per goal.
+ */
+import { PredicateKind, QueryBy, ReticleTool, Verified, VerifiedReason, asRecord, } from '@reticlehq/core';
+/** More than this and the persona is a document, not a set of goals. */
+const MAX_GOALS = 10;
+const MAX_GOAL_LENGTH = 200;
+/**
+ * A string the persona quoted: "straight" or “curly”. Single quotes are apostrophes too often.
+ *
+ * A straight quote opens only after a space, a bracket or the start, and closes only before one or
+ * before sentence punctuation, so a goal that quotes text with quotes in it — the issue title
+ * `"Check failed: an element matching {"testid":"pay"}"` — stays one goal. Matching any `"` split
+ * that title into `{`, `:` and `}`, and graded the fragments.
+ */
+const QUOTED = /(?<![^\s([])"([^\n]{1,200}?)"(?![^\s.,;!?)\]])|“([^”\n]{1,200})”/g;
+/**
+ * A negation in the same clause just before a quote: "must not show "payments"". The window stops
+ * at the previous quote or clause end, so `doesn't show "X" until you click "Y"` negates X only. A
+ * comma ends the clause too: "place the order without the coupon, ending on "Order confirmed"" read
+ * its end as text that must be absent, and the journey failed with no goal to reach.
+ */
+const NEGATED_BEFORE = /(?:\b(?:not|never|no|nor|without|no longer)|n't)\b[^"“”.,;!?]{0,40}$/i;
+/** Said of a goal the page showed before the drive did anything. */
+const ALREADY_SHOWN = `${VerifiedReason.ALREADY_TRUE}: it was on the page before the drive started, so seeing it proves nothing`;
+/**
+ * Said of a goal the page the drive ended on never showed, even scrolled. Undecided, never `no`: the
+ * drive may not have reached the page that shows it, and a "no" there blamed the app for where the
+ * Harness stopped.
+ */
+const NOT_REACHED = `${VerifiedReason.INCONCLUSIVE}: not on the page the drive ended on, even scrolled; the drive may not have reached the page that shows it`;
+/** Said of a goal the page showed when looked at, which the text check nonetheless did not find. */
+const SHOWN_NOT_HELD = `${VerifiedReason.INCONCLUSIVE}: shown on the page when looked at, but the text check did not find it (a field's value, or text it cannot read)`;
+/** Said of an absence goal beside a goal the drive never reached: absent from the wrong page proves nothing. */
+const ABSENCE_UNREACHED = `${VerifiedReason.INCONCLUSIVE}: the drive did not reach the page it ends on, so its absence there proves nothing`;
+/** How the snapshot and the element reader end a text they cut short. */
+const CUT = '…';
+const collapse = (text) => text.replace(/\s+/g, ' ').trim();
+/**
+ * Whether `shown` — one element's text, maybe cut short with "…" — is text the Harness wrote itself
+ * rather than the app: one of `own` (its journey, the names of the plan and the flows it saved), in
+ * full or cut short, holding `goal`. A Harness asked to rename a plan stopped on the /plans list and
+ * was graded PROVED on every quote: the plan it had just saved was named after the journey, and the
+ * journey quoted them all. An element showing exactly the goal is the app's, whoever else wrote it.
+ */
+export function writtenByHarness(shown, goal, own) {
+    const full = collapse(shown);
+    const cut = full.endsWith(CUT);
+    const text = cut ? full.slice(0, -CUT.length).trimEnd() : full;
+    const wanted = collapse(goal);
+    if (text.length <= wanted.length)
+        return false;
+    return own.some((o) => {
+        const mine = collapse(o);
+        if (0 === mine.length || !mine.includes(wanted))
+            return false;
+        if (cut && mine.startsWith(text))
+            return true;
+        // Its own text first ("<journey> and 5 more"): what follows it is not the Harness's.
+        return text.startsWith(mine) && !text.slice(mine.length).includes(wanted);
+    });
+}
+/**
+ * Whether a snapshot shows `goal` anywhere but inside text the Harness wrote itself. Read as text,
+ * not parsed: the journey quotes its goals, so its quotes cut a parsed name short.
+ */
+function shownByApp(tree, goal, own) {
+    const wanted = collapse(goal);
+    const mine = own.map(collapse).filter((o) => wanted.length < o.length && o.includes(wanted));
+    if (0 === mine.length)
+        return tree.includes(goal);
+    return tree.split('\n').some((line) => {
+        let rest = collapse(line);
+        for (const o of mine) {
+            rest = rest.split(o).join('');
+            rest = withoutCutStart(rest, o, wanted.length);
+        }
+        return rest.includes(wanted);
+    });
+}
+/** `line` without each start of `own` the snapshot cut short with "…" (longer than `min`). */
+function withoutCutStart(line, own, min) {
+    let rest = line;
+    for (let at = rest.indexOf(CUT); -1 !== at; at = rest.indexOf(CUT, at + 1)) {
+        const before = rest.slice(0, at);
+        for (let n = Math.min(own.length, before.length); min < n; n -= 1) {
+            if (!before.endsWith(own.slice(0, n)))
+                continue;
+            rest = `${rest.slice(0, at - n)}${rest.slice(at + CUT.length)}`;
+            at -= n;
+            break;
+        }
+    }
+    return rest;
+}
+/** The end state a goal quotes: its last quoted text that it does not negate. */
+export function endOf(goal) {
+    return parseGoals(goal)
+        .filter((g) => true !== g.absent)
+        .at(-1)?.text;
+}
+/** The observable goals in a persona: whatever it quoted, each marked absent when it was negated. */
+export function parseGoals(persona) {
+    if (persona === undefined)
+        return [];
+    const out = new Map();
+    for (const m of persona.matchAll(QUOTED)) {
+        const text = (m[1] ?? m[2] ?? '').trim();
+        if (0 === text.length || out.has(text))
+            continue;
+        const absent = NEGATED_BEFORE.test(persona.slice(0, m.index));
+        out.set(text, absent ? { text, absent } : { text });
+    }
+    return [...out.values()].slice(0, MAX_GOALS);
+}
+/** The texts a persona says must be on the page: the quoted goals it did not negate. */
+export function goalsIn(persona) {
+    return parseGoals(persona).flatMap((g) => (true === g.absent ? [] : [g.text]));
+}
+/**
+ * Check each goal on the page the drive ended on, recording ONE assert for the drive.
+ *
+ * Each quote used to be its own assert, and every assert is a check in the drive's run, so a goal
+ * that quotes where it starts ("from "Count is 0" to "Count is 1"") recorded a "no" for the start
+ * text a working app had rightly replaced, and the run synced as a failing flow. The page is let
+ * settle, read once, and each goal is scrolled to by its text — one below the fold read `unknown`
+ * on a drive that never scrolled. The one recorded assert covers the goals found. A goal not found
+ * is undecided, not failed: nothing here knows the drive reached the page that would show it. A
+ * check that cannot run is `unknown`, never skipped.
+ */
+export async function checkGoals(invoke, goals, 
+/** The page as it read before the drive: a goal it already showed proves nothing at the end. */
+opening, 
+/** Text the Harness wrote itself (its journey, its plan's and flows' names): never evidence. */
+own = []) {
+    const byText = new Map();
+    for (const goal of goals.slice(0, MAX_GOALS)) {
+        const g = 'string' === typeof goal ? { text: goal } : goal;
+        const text = g.text.trim().slice(0, MAX_GOAL_LENGTH);
+        if (0 < text.length && !byText.has(text))
+            byText.set(text, { ...g, text });
+    }
+    const all = [...byText.values()];
+    const before = new Set(all.filter((g) => true !== g.absent && true === opening?.includes(g.text)).map((g) => g.text));
+    const unchecked = (g, reason) => ({
+        text: g.text,
+        ...(true === g.absent ? { absent: true } : {}),
+        verified: Verified.UNKNOWN,
+        reason,
+    });
+    const todo = all.filter((g) => !before.has(g.text));
+    if (0 === todo.length)
+        return all.map((g) => unchecked(g, ALREADY_SHOWN));
+    await settle(invoke);
+    const page = await readOpening(invoke);
+    const found = new Set();
+    for (const g of todo) {
+        if (true === g.absent)
+            continue;
+        if ((await scrolledTo(invoke, g.text, own)) || shownByApp(page ?? '', g.text, own))
+            found.add(g.text);
+    }
+    const unreached = todo.some((g) => true !== g.absent && !found.has(g.text));
+    const absences = unreached ? [] : todo.filter((g) => true === g.absent);
+    const recorded = [...todo.filter((g) => found.has(g.text)), ...absences];
+    const verified = 0 < recorded.length ? await assertGoals(invoke, recorded) : Verified.UNKNOWN;
+    // One joint check that failed says only that SOMETHING did not hold: each goal is asked on its own,
+    // so a goal that held is not graded "no" for a neighbour that did not.
+    const each = new Map();
+    if (Verified.NO === verified && 1 < recorded.length)
+        for (const g of recorded)
+            each.set(g.text, await assertGoals(invoke, [g]));
+    const verdictOf = (g) => each.get(g.text) ?? verified;
+    return all.map((g) => {
+        if (before.has(g.text))
+            return unchecked(g, ALREADY_SHOWN);
+        if (true === g.absent) {
+            return unreached
+                ? unchecked(g, ABSENCE_UNREACHED)
+                : { text: g.text, absent: true, verified: verdictOf(g) };
+        }
+        if (!found.has(g.text))
+            return unchecked(g, NOT_REACHED);
+        // Seen on the page yet the text check says no: a field's value, or text the check cannot read.
+        return Verified.NO === verdictOf(g)
+            ? unchecked(g, SHOWN_NOT_HELD)
+            : { text: g.text, verified: verdictOf(g) };
+    });
+}
+/** The verdict of one assert over `goals`, or unknown when it could not be asked. */
+async function assertGoals(invoke, goals) {
+    try {
+        const result = asRecord(await invoke(ReticleTool.ASSERT, { predicate: allOf(goals) }));
+        return 'string' === typeof result['verified'] ? result['verified'] : Verified.UNKNOWN;
+    }
+    catch {
+        return Verified.UNKNOWN;
+    }
+}
+/**
+ * Whether the page shows `text` once it has settled, scrolled to if need be — how a person looks
+ * for something before saying it is not there. Records no check.
+ */
+export async function lookFor(invoke, text, 
+/** Text the Harness wrote itself: a match inside it is not the page showing the goal. */
+own = []) {
+    await settle(invoke);
+    if (shownByApp((await readOpening(invoke)) ?? '', text, own))
+        return true;
+    return scrolledTo(invoke, text, own);
+}
+/** Long enough for a lazy route to render; short enough that a page that never quiets costs little. */
+const SETTLE_MS = 3000;
+/** Scrolls spent looking for one goal: a long page, not an endless feed. */
+const GOAL_SCROLLS = 10;
+/** Let a page that is still changing finish, so a view still swapping in is not read half-drawn. */
+async function settle(invoke) {
+    try {
+        await invoke(ReticleTool.WAIT_FOR, {
+            predicate: { kind: PredicateKind.SETTLED },
+            timeout_ms: SETTLE_MS,
+            // From the session's start, not from this call: a request the drive's last click opened is
+            // in flight BEFORE this wait begins, and a window that starts now cannot see it. Driven over
+            // MCP: the drive ended on "Generating…", the page read as settled, and a journey whose end
+            // appeared a second later was failed. SETTLE_MS still bounds a request that never answers.
+            since: 0,
+        });
+    }
+    catch {
+        /* a page that never settles is still read as it stands */
+    }
+}
+/**
+ * Bring a goal's text into view, the way a person scrolls to it. True when it was found, and not
+ * inside text the Harness wrote itself.
+ */
+async function scrolledTo(invoke, text, own) {
+    try {
+        const result = asRecord(await invoke(ReticleTool.SCROLL_TO, {
+            by: QueryBy.TEXT,
+            value: text,
+            maxScrolls: GOAL_SCROLLS,
+        }));
+        const element = asRecord(result['element']);
+        const shown = element['text'] ?? element['name'];
+        return (true === result['found'] && !('string' === typeof shown && writtenByHarness(shown, text, own)));
+    }
+    catch {
+        return false;
+    }
+}
+const SNAPSHOT_FULL = 'full';
+/** The page as it reads now, or undefined when it cannot be read. Taken before a drive starts. */
+export async function readOpening(invoke) {
+    try {
+        const tree = asRecord(await invoke(ReticleTool.SNAPSHOT, { mode: SNAPSHOT_FULL }))['tree'];
+        return 'string' === typeof tree ? tree : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+function allOf(goals) {
+    const each = goals.map((g) => {
+        const shown = { kind: PredicateKind.TEXT, contains: g.text };
+        return true === g.absent ? { kind: PredicateKind.NOT, predicate: shown } : shown;
+    });
+    return 1 === each.length ? (each[0] ?? {}) : { kind: PredicateKind.ALL_OF, predicates: each };
+}
+/**
+ * The end state the caller declared, checked like any assert.
+ *
+ * Quoted text only proves some words are on the last page, which is how a drive that never used the
+ * app could pass: from the field, "it only checks that I have certain titles on my page". An `expect`
+ * names an OUTCOME — a route reached, a request made, state changed — in the same predicate grammar
+ * as reticle_assert, and is graded the same way. A check that cannot run is `unknown`.
+ */
+export async function checkExpect(invoke, predicate) {
+    let verified = Verified.UNKNOWN;
+    try {
+        const result = asRecord(await invoke(ReticleTool.ASSERT, { predicate }));
+        if ('string' === typeof result['verified'])
+            verified = result['verified'];
+    }
+    catch {
+        /* stays unknown: the outcome was not proved */
+    }
+    return { text: `expect ${JSON.stringify(predicate).slice(0, MAX_GOAL_LENGTH)}`, verified };
+}
+/** Whether the drive met the goals it was given: undefined when it was given none. */
+export function goalsMet(checks) {
+    return 0 === checks.length ? undefined : checks.every((c) => Verified.YES === c.verified);
+}
+/**
+ * Whether the drive met its goal, as the answer and the run report it. The driver's own `goalMet`
+ * is its word, never evidence: a run answered `goalMet: true` after zero actions on a tab that
+ * already showed the end state, and another with both its goals `unknown`. With goals, met exactly
+ * when every one was proved; with none, the driver's word stands, and only after an action ran.
+ * Undefined when nothing judged it.
+ */
+export function goalOfDrive(claimed, checks, acted) {
+    // Proved goals are the evidence: a `yes` was not on the page before the drive (that reads
+    // already_true), so the run reached it, by the drive or by the hunt after it.
+    if (0 < checks.length)
+        return true === goalsMet(checks);
+    return claimed === undefined ? undefined : acted && claimed;
+}
+/** One line naming the goals the drive did not prove, or undefined when it proved them all. */
+export function unprovedGoals(checks) {
+    const missed = checks.filter((c) => Verified.YES !== c.verified);
+    if (0 === missed.length)
+        return undefined;
+    return (`Not proved: ${missed.map((c) => `${true === c.absent ? 'no ' : ''}"${c.text}" (${c.reason ?? c.verified})`).join(', ')}. ` +
+        'The drive did not finish the job it was given, whatever its own account says.');
+}
+/**
+ * The goals a drive is graded on: every quote, less a value the drive typed — an input, never
+ * evidence — except the journey's end (its last text to show), which stays even when typed: saving
+ * "Order 7" and then seeing it listed is the journey. Absence goals always stay.
+ */
+export function endGoalsOf(goals, typed) {
+    const end = goals.filter((g) => true !== g.absent).at(-1);
+    const inputs = new Set(typed.map((t) => t.trim()));
+    return goals.filter((g) => true === g.absent || g === end || !inputs.has(g.text.trim()));
+}

package/dist/features/harness/harness.d.ts

@@ -107,7 +107,12 @@ export declare const StopReason: {
     readonly STALLED: "stalled";
     /** The toolset or the model itself failed in a way the loop cannot continue through. */
     readonly BROKEN: "broken";
+    /** A person switched autonomous driving off while it was driving. Not a failure. */
+    readonly STOPPED: "stopped";
 };
+/** Thrown by a driver when a person, not a fault, ended the drive. */
+export declare class DriveStoppedError extends Error {
+}
 export type StopReason = (typeof StopReason)[keyof typeof StopReason];
 export interface HarnessResult {
     stopReason: StopReason;
@@ -118,7 +123,36 @@ export interface HarnessResult {
     usage: TokenUsage;
     /** Set when `stopReason` is BROKEN. */
     error?: string;
+    /**
+     * Whether the drive ran at least one check: an `act_and_wait` with an `until`, or an `assert`,
+     * that did not error. A drive without one clicked things and proved nothing.
+     */
+    proved: boolean;
+    /**
+     * Whether the drive reached its whole goal, as the platform judged when it finished. Checks that
+     * held are not the goal reached: "open each section" passed on two presses of Sign in.
+     */
+    goalMet?: boolean;
+    /** The credits the platform says the drive spent; absent for a local model or an older platform. */
+    creditsSpent?: number;
 }
+/**
+ * The answer to a `finish` that came before anything was proved. Sent once per drive: a real drive
+ * asked to "click twice and check it reads 2" clicked once, finished, and saved a flow whose replay
+ * verifies nothing, which is the paid feature producing an empty pass.
+ */
+export declare const MSG_FINISH_UNPROVED: string;
+/**
+ * The answer to a click repeated after it changed nothing. A persona drive clicked the logo four
+ * times on the page the logo leads to, each a wasted action and a wrong route guess.
+ */
+export declare const MSG_REPEAT_CHANGED_NOTHING: string;
+/**
+ * Said on a click whose route guess missed while it opened something on the page. A persona drive
+ * clicked the account button expecting to leave /overview; it opened a menu, the miss read as a
+ * failure, and the drive abandoned the route its plan named instead of choosing "Settings" in it.
+ */
+export declare const MSG_OPENED_IN_PLACE: string;
 /**
  * The one tool the harness adds to Reticle's surface.
  *
@@ -134,6 +168,22 @@ export interface HarnessOptions {
     focus?: string;
     /** How long one model turn may take. Injected so a test can prove the bound without waiting for it. */
     turnTimeoutMs?: number;
+    /** Asked before every turn: true ends the drive as STOPPED. Whoever started it may stop it. */
+    stopped?: () => boolean;
+    /** Asked before every turn: what somebody said to the drive since, handed over as user turns. */
+    inbox?: () => readonly string[];
+    /** The pages the app's map knows: named up front, and offered after a route guess misses. */
+    routes?: readonly string[];
+    /**
+     * The end the goal quotes, and the text the Harness wrote itself (never the app showing it): the
+     * drive stops once a check has held and the page shows it. Driven over MCP: a model that reached
+     * "Generated script for" kept going, navigated to a start page that signs the app out, and ended
+     * on a page without the result — a working journey failed by its own wandering.
+     */
+    endsAt?: {
+        readonly text: string;
+        readonly own: readonly string[];
+    };
 }
 export declare const DEFAULT_MAX_STEPS = 40;
 /**
@@ -149,7 +199,7 @@ export declare const DEFAULT_MAX_STEPS = 40;
  * deterministically for a few hundred tokens, forever. That is the whole economics of this feature,
  * so it is an instruction and not a hope.
  */
-export declare function systemPrompt(focus?: string): string;
+export declare function systemPrompt(focus?: string, routes?: readonly string[]): string;
 /**
  * Run the harness to completion.
  *