@reticlehq/server 3.5.0 → 3.7.0

package/dist/features/harness/harness.js

@@ -17,7 +17,10 @@
  * 3. **Everything is injected** — the model, the tools, the clock. The loop below has no network in
  *    it, which is what lets the whole thing be tested against a scripted model with no API key.
  */
+import { ReticleTool, Verified, WRITING_RULE, asRecord } from '@reticlehq/core';
+import { claimsOnlyRoute, knownPagesLine, routeHint } from './route-guess.js';
 import { withTimeout } from './with-timeout.js';
+import { lookFor } from './goals.js';
 /** Why a run stopped. Every one of these is a real outcome; none of them is an error. */
 export const StopReason = {
     /** The model called `finish` — it believes it is done. */
@@ -28,7 +31,92 @@ export const StopReason = {
     STALLED: 'stalled',
     /** The toolset or the model itself failed in a way the loop cannot continue through. */
     BROKEN: 'broken',
+    /** A person switched autonomous driving off while it was driving. Not a failure. */
+    STOPPED: 'stopped',
 };
+/** Thrown by a driver when a person, not a fault, ended the drive. */
+export class DriveStoppedError extends Error {
+}
+/** The finish's judgement of the goal, and what the drive cost, when it carried them. */
+function goalMetOf(args) {
+    const met = args['goalMet'];
+    const spent = args['creditsSpent'];
+    return {
+        ...('boolean' === typeof met ? { goalMet: met } : {}),
+        ...('number' === typeof spent ? { creditsSpent: spent } : {}),
+    };
+}
+/**
+ * The answer to a `finish` that came before anything was proved. Sent once per drive: a real drive
+ * asked to "click twice and check it reads 2" clicked once, finished, and saved a flow whose replay
+ * verifies nothing, which is the paid feature producing an empty pass.
+ */
+export const MSG_FINISH_UNPROVED = `Not finished: nothing has been proved yet. Before finishing, end the journey with ` +
+    `${ReticleTool.ACT_AND_WAIT} and an \`until\` naming the end state it must reach, or ` +
+    `${ReticleTool.ASSERT} on it. A drive that proves nothing replays as "verified nothing".`;
+/**
+ * The answer to a click repeated after it changed nothing. A persona drive clicked the logo four
+ * times on the page the logo leads to, each a wasted action and a wrong route guess.
+ */
+export const MSG_REPEAT_CHANGED_NOTHING = 'Not done again: the same action on the same control just changed nothing on the page — no ' +
+    'content, request, route or signal. Doing it again will not either. Look at the page and take ' +
+    'the next step the journey names.';
+/**
+ * Said on a click whose route guess missed while it opened something on the page. A persona drive
+ * clicked the account button expecting to leave /overview; it opened a menu, the miss read as a
+ * failure, and the drive abandoned the route its plan named instead of choosing "Settings" in it.
+ */
+export const MSG_OPENED_IN_PLACE = 'This click opened something on this page (a menu, a panel, a dialog) instead of going where you ' +
+    'guessed. That is progress, not a dead end: look at what it opened and act on the item the ' +
+    'journey names next.';
+/** A route-only guess that missed, on a click that added content: it opened something in place. */
+function openedInPlace(outcome) {
+    if (outcome.isError || ReticleTool.ACT_AND_WAIT !== outcome.name)
+        return false;
+    if (!claimsOnlyRoute(asRecord(outcome.args)['until']))
+        return false;
+    const result = asRecord(outcome.result);
+    if (Verified.YES === result['verified'])
+        return false;
+    const summary = asRecord(asRecord(result['trace'])['summary']);
+    return 0 < (Number(summary['domAdded']) || 0);
+}
+/** A route-only guess that missed: the known page to go to instead, when the map knows any. */
+function missedRouteHint(outcome, routes) {
+    if (outcome.isError || ReticleTool.ACT_AND_WAIT !== outcome.name)
+        return undefined;
+    const until = asRecord(outcome.args)['until'];
+    if (!claimsOnlyRoute(until) || Verified.YES === asRecord(outcome.result)['verified'])
+        return undefined;
+    return routeHint(until, routes);
+}
+/** The tools that act on one control, keyed by what they did to which control. */
+const ACTING_TOOLS = new Set([ReticleTool.ACT, ReticleTool.ACT_AND_WAIT]);
+/** What an action did to which control, or undefined for a call that does not act on one. */
+function actKey(name, args) {
+    if (!ACTING_TOOLS.has(name))
+        return undefined;
+    const target = args['ref'] ?? args['target'];
+    if (target === undefined)
+        return undefined;
+    return JSON.stringify([target, args['action'] ?? null, args['value'] ?? null]);
+}
+/** Whether a result's reaction digest says the page did nothing at all; undefined when it has none. */
+function changedNothing(result) {
+    const summary = asRecord(asRecord(asRecord(result)['trace'])['summary']);
+    if ('number' !== typeof summary['total'])
+        return undefined;
+    const moved = ['network', 'domAdded', 'domRemoved', 'domChanged', 'routeChanges', 'signals'].some((k) => 0 < (Number(summary[k]) || 0));
+    return !moved;
+}
+/** A check that ran: the only thing that turns a drive into evidence. */
+function isProof(outcome) {
+    if (outcome.isError)
+        return false;
+    if (ReticleTool.ASSERT === outcome.name)
+        return true;
+    return ReticleTool.ACT_AND_WAIT === outcome.name && asRecord(outcome.args)['until'] !== undefined;
+}
 /**
  * The one tool the harness adds to Reticle's surface.
  *
@@ -51,6 +139,10 @@ export const FINISH_TOOL = {
         additionalProperties: false,
     },
 };
+/** Why a drive ended before the model said so. */
+const msgReachedEnd = (end) => `Reached the end the goal names ("${end}") after a check held, so the drive stopped there.`;
+/** A check in this turn that came back yes: the moment worth looking for the end. */
+const heldCheck = (outcome) => isProof(outcome) && Verified.YES === asRecord(outcome.result)['verified'];
 export const DEFAULT_MAX_STEPS = 40;
 /**
  * How long a single model turn may take before the run gives up on it.
@@ -79,7 +171,7 @@ const OPENING = 'The app is loaded and connected. Begin by looking at the page,
  * deterministically for a few hundred tokens, forever. That is the whole economics of this feature,
  * so it is an instruction and not a hope.
  */
-export function systemPrompt(focus) {
+export function systemPrompt(focus, routes = []) {
     return [
         'You are driving a real web application through a verification tool, to find out whether it actually works.',
         '',
@@ -94,7 +186,9 @@ export function systemPrompt(focus) {
         'the app is broken underneath.',
         '',
         'Drive the app the way a user would: complete whole journeys, not single clicks. Submit the forms.',
-        'Follow the navigation. If something looks interactive, try it.',
+        'Follow the navigation. If something looks interactive, try it — unless the focus below says',
+        'not to: "without opening a run", "changes nothing" and the like bind the whole drive. Read the',
+        'list a journey says to read; do not open what it says to leave closed.',
         '',
         'RECORD WHAT YOU DRIVE. Before each journey call reticle_record { action: "start" }, drive it, then',
         'reticle_flow_save with a short descriptive name. A saved flow replays deterministically on every',
@@ -110,6 +204,14 @@ export function systemPrompt(focus) {
         'expectations as you go. Do not report bugs; drive the app.',
         '',
         'Call `finish` when you have covered the app or cannot get further.',
+        '',
+        WRITING_RULE,
+        ...(0 === routes.length
+            ? []
+            : [
+                '',
+                `Pages this app has: ${knownPagesLine(routes)}. To reach one, press its link or reticle_navigate to it; do not guess which control leads there.`,
+            ]),
         ...(focus === undefined ? [] : ['', `Focus for this run: ${focus}`]),
     ].join('\n');
 }
@@ -122,22 +224,44 @@ export function systemPrompt(focus) {
 export async function runHarness(driver, toolset, options = {}) {
     const maxSteps = options.maxSteps ?? DEFAULT_MAX_STEPS;
     const tools = [...toolset.tools, FINISH_TOOL];
-    const system = systemPrompt(options.focus);
+    const routes = options.routes ?? [];
+    const system = systemPrompt(options.focus, routes);
     const history = [{ role: 'user', text: OPENING }];
     const toolCalls = [];
     const usage = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 };
+    const proved = () => toolCalls.some(isProof);
+    let askedToProve = false;
+    // Actions that just changed nothing; forgotten as soon as any action moves the page.
+    const inert = new Set();
     for (let step = 0; step < maxSteps; step += 1) {
+        if (true === options.stopped?.()) {
+            return {
+                stopReason: StopReason.STOPPED,
+                summary: '',
+                steps: step,
+                toolCalls,
+                usage,
+                proved: proved(),
+                goalMet: false,
+            };
+        }
+        for (const text of options.inbox?.() ?? [])
+            history.push({ role: 'user', text });
         let turn;
         try {
             turn = await withTimeout(driver.turn({ system, tools, history }), options.turnTimeoutMs ?? DEFAULT_TURN_TIMEOUT_MS, 'the model did not answer');
         }
         catch (error) {
             return {
-                stopReason: StopReason.BROKEN,
+                stopReason: error instanceof DriveStoppedError ? StopReason.STOPPED : StopReason.BROKEN,
                 summary: '',
                 steps: step,
                 toolCalls,
                 usage,
+                proved: proved(),
+                // Cut off is not done: the checks that held before a refusal (credits gone, a 5xx) prove
+                // the steps before it, never the journey, and every verdict downstream reads this.
+                goalMet: false,
                 error: error instanceof Error ? error.message : String(error),
             };
         }
@@ -156,10 +280,38 @@ export async function runHarness(driver, toolset, options = {}) {
                 steps: step + 1,
                 toolCalls,
                 usage,
+                proved: proved(),
             };
         }
         history.push({ role: 'assistant', text: turn.text, calls: turn.calls });
         const finish = turn.calls.find((call) => FINISH_TOOL.name === call.name);
+        if (finish !== undefined && !proved() && !askedToProve) {
+            // Whatever else the turn asked for runs first: a check sent alongside `finish` is proof.
+            const others = await Promise.all(turn.calls.filter((c) => c !== finish).map((c) => invokeOne(toolset, c)));
+            toolCalls.push(...others);
+            if (proved()) {
+                const summary = finish.args['summary'];
+                return {
+                    stopReason: StopReason.FINISHED,
+                    summary: 'string' === typeof summary ? summary : turn.text,
+                    steps: step + 1,
+                    toolCalls,
+                    usage,
+                    proved: true,
+                    ...goalMetOf(finish.args),
+                };
+            }
+            askedToProve = true;
+            const refused = {
+                id: finish.id,
+                name: finish.name,
+                args: finish.args,
+                result: { error: MSG_FINISH_UNPROVED },
+                isError: true,
+            };
+            history.push({ role: 'tool', outcomes: [...others, refused] });
+            continue;
+        }
         if (finish !== undefined) {
             const summary = finish.args['summary'];
             return {
@@ -168,16 +320,65 @@ export async function runHarness(driver, toolset, options = {}) {
                 steps: step + 1,
                 toolCalls,
                 usage,
+                proved: proved(),
+                ...goalMetOf(finish.args),
             };
         }
         // Every call the model asked for is answered, including ones that fail. A dropped result leaves
         // the model waiting on an answer that never comes, and it will usually re-ask — burning the
         // budget on a call that already ran, against an app it already changed.
-        const outcomes = await Promise.all(turn.calls.map((call) => invokeOne(toolset, call)));
+        const outcomes = await Promise.all(turn.calls.map((call) => {
+            const key = actKey(call.name, call.args);
+            return key !== undefined && inert.has(key)
+                ? Promise.resolve({
+                    id: call.id,
+                    name: call.name,
+                    args: call.args,
+                    result: { error: MSG_REPEAT_CHANGED_NOTHING },
+                    isError: true,
+                })
+                : invokeOne(toolset, call);
+        }));
+        for (const [i, outcome] of outcomes.entries()) {
+            const hint = openedInPlace(outcome) ? MSG_OPENED_IN_PLACE : missedRouteHint(outcome, routes);
+            if (hint !== undefined)
+                outcomes[i] = { ...outcome, result: { ...asRecord(outcome.result), next: hint } };
+        }
+        for (const outcome of outcomes) {
+            const key = actKey(outcome.name, outcome.args);
+            if (key === undefined || outcome.isError)
+                continue;
+            const nothing = changedNothing(outcome.result);
+            if (true === nothing)
+                inert.add(key);
+            else if (false === nothing)
+                inert.clear();
+        }
         toolCalls.push(...outcomes);
         history.push({ role: 'tool', outcomes });
+        const end = options.endsAt;
+        if (end !== undefined &&
+            outcomes.some(heldCheck) &&
+            (await lookFor((name, args) => toolset.invoke(name, args), end.text, end.own))) {
+            return {
+                stopReason: StopReason.FINISHED,
+                summary: msgReachedEnd(end.text),
+                steps: step + 1,
+                toolCalls,
+                usage,
+                proved: proved(),
+                goalMet: true,
+            };
+        }
     }
-    return { stopReason: StopReason.BUDGET, summary: '', steps: maxSteps, toolCalls, usage };
+    return {
+        stopReason: StopReason.BUDGET,
+        summary: '',
+        steps: maxSteps,
+        toolCalls,
+        usage,
+        proved: proved(),
+    };
 }
 /**
  * One tool call, with its failure captured rather than thrown.

package/dist/features/harness/platform/drive-target.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Make ready the tab a platform drive runs in, as its spec asks, and say what was and was not done.
+ *
+ * The person's own tab, or one opened for the drive: headless, or in a window they can watch. Then
+ * the HUD as asked. Every step that cannot be done is reported, never thrown: a drive asked for a
+ * window on a machine with no display still runs headless if that is all there is, and says so.
+ */
+import { HudVisibility, type DriveApplied, type DriveSpec, type SpecIgnored } from '@reticlehq/core';
+/** How many times the HUD is asked for before giving up: a page's HUD mounts after it connects. */
+export declare const HUD_ATTEMPTS = 4;
+export declare const HUD_RETRY_MS = 500;
+/** What a refused drive is told when no address was given to open. */
+export declare const NO_ADDRESS_TO_OPEN = "Nothing says where this app runs, so no browser could be opened for it. Open the app once with Reticle running, or give the drive an address.";
+export interface DriveTargetPorts {
+    /** The person's own tab for the drive, as the chat already picks it. Null: none may be driven. */
+    pickTab: () => Promise<string | null | undefined>;
+    /** Open `url` in a browser of the daemon's own; answers the tab's session id. Throws why not. */
+    open: (url: string, headed: boolean, hud: HudVisibility | undefined) => Promise<string>;
+    /** Ask the page for this HUD; throws when the page did not take it (no HUD yet, an older SDK). */
+    tuneHud: (sessionId: string, hud: HudVisibility) => Promise<void>;
+    sleep: (ms: number) => Promise<void>;
+}
+export interface PreparedDrive {
+    sessionId: string | null | undefined;
+    applied: DriveApplied;
+    ignored: SpecIgnored[];
+    /** Why the drive cannot run, in a person's words, when it cannot. */
+    refusal?: string;
+}
+export declare function prepareDrive(spec: DriveSpec, fallbackUrl: string | undefined, ports: DriveTargetPorts): Promise<PreparedDrive>;

package/dist/features/harness/platform/drive-target.js

@@ -0,0 +1,81 @@
+/**
+ * Make ready the tab a platform drive runs in, as its spec asks, and say what was and was not done.
+ *
+ * The person's own tab, or one opened for the drive: headless, or in a window they can watch. Then
+ * the HUD as asked. Every step that cannot be done is reported, never thrown: a drive asked for a
+ * window on a machine with no display still runs headless if that is all there is, and says so.
+ */
+import { DriveMode, DriveTarget, HudVisibility, SpecIgnoredReason, } from '@reticlehq/core';
+/** How many times the HUD is asked for before giving up: a page's HUD mounts after it connects. */
+export const HUD_ATTEMPTS = 4;
+export const HUD_RETRY_MS = 500;
+/** What a refused drive is told when no address was given to open. */
+export const NO_ADDRESS_TO_OPEN = 'Nothing says where this app runs, so no browser could be opened for it. Open the app once with Reticle running, or give the drive an address.';
+export async function prepareDrive(spec, fallbackUrl, ports) {
+    const mode = spec.mode ?? DriveMode.LOCAL;
+    const ignored = [];
+    let target = spec.target ?? DriveTarget.TAB;
+    let sessionId;
+    let url;
+    if (DriveTarget.TAB !== target) {
+        url = spec.url ?? fallbackUrl;
+        if (url === undefined)
+            return {
+                sessionId: null,
+                applied: { target, mode },
+                ignored,
+                refusal: NO_ADDRESS_TO_OPEN,
+            };
+        try {
+            sessionId = await ports.open(url, DriveTarget.HEADED === target, spec.hud);
+        }
+        catch (error) {
+            const detail = error instanceof Error ? error.message : String(error);
+            // A window that cannot open falls back to a browser with none: the drive still runs.
+            if (DriveTarget.HEADED !== target)
+                return { sessionId: null, applied: { target, mode, url }, ignored, refusal: detail };
+            ignored.push({ field: 'target', reason: SpecIgnoredReason.UNAVAILABLE, detail });
+            target = DriveTarget.HEADLESS;
+            try {
+                sessionId = await ports.open(url, false, spec.hud);
+            }
+            catch (again) {
+                const why = again instanceof Error ? again.message : String(again);
+                return { sessionId: null, applied: { target, mode, url }, ignored, refusal: why };
+            }
+        }
+    }
+    else {
+        sessionId = await ports.pickTab();
+    }
+    const applied = {
+        target,
+        mode,
+        ...(url === undefined ? {} : { url }),
+        ...('string' === typeof sessionId ? { sessionId } : {}),
+    };
+    if (spec.hud !== undefined && 'string' === typeof sessionId) {
+        const taken = await tune(ports, sessionId, spec.hud);
+        if (taken === undefined)
+            applied.hud = spec.hud;
+        else
+            ignored.push({ field: 'hud', reason: SpecIgnoredReason.UNAVAILABLE, detail: taken });
+    }
+    return { sessionId, applied, ignored };
+}
+/** Ask for the HUD until the page takes it; answers why not when it never does. */
+async function tune(ports, sessionId, hud) {
+    let why = 'the page did not take it';
+    for (let attempt = 1; attempt <= HUD_ATTEMPTS; attempt += 1) {
+        try {
+            await ports.tuneHud(sessionId, hud);
+            return undefined;
+        }
+        catch (error) {
+            why = error instanceof Error ? error.message : String(error);
+            if (attempt < HUD_ATTEMPTS)
+                await ports.sleep(HUD_RETRY_MS);
+        }
+    }
+    return why;
+}

package/dist/features/harness/platform/local-apps.d.ts

@@ -0,0 +1,127 @@
+import { type CoverageCapability, type LinkAppGap, type LinkDevServer, type LinkHelloFailure, type LinkUnsyncedRoot, type LinkCapability, type LinkNotice } from '@reticlehq/core';
+import { type DriveCandidate } from './remote-drive.js';
+/** How often apps are reported. The platform counts an app live for two and a half of these. */
+export declare const LOCAL_APPS_REPORT_MS = 10000;
+/** How long a drive waits for an app it opened to connect before it is refused. */
+export declare const APP_OPEN_WAIT_MS = 20000;
+export interface Platform {
+    url: string;
+    apiKey: string;
+}
+export interface Machine {
+    id: string;
+    name: string;
+}
+/** One connected tab, as far as reporting and picking go. */
+export interface AppTab extends DriveCandidate {
+    title?: string;
+    projectId?: string;
+    adapters: string[];
+    runtime?: string;
+    /** The SDK the page announced. The dashboard compares it with `reticleVersion` per app. */
+    sdkVersion?: string;
+    /** The page's HELLO channels, plus the coverage markers (`source`, `testid`) it earned. */
+    channels?: string[];
+    /** Instrumentation gaps this tab's verdicts recorded, newest first. */
+    gaps?: LinkAppGap[];
+    /** Coverage capabilities that do not apply to this app. */
+    notApplicable?: CoverageCapability[];
+}
+export interface ReportedApp {
+    key: string;
+    name: string;
+    url: string;
+    title?: string;
+    stack: string[];
+    sdkVersion?: string;
+    channels?: string[];
+    gaps?: LinkAppGap[];
+    adapters?: string[];
+    notApplicable?: CoverageCapability[];
+}
+/** What a report carries beside its apps. Every field additive; see `LinkReportKey`. */
+export interface ReportExtras {
+    devServers?: LinkDevServer[];
+    helloFailure?: LinkHelloFailure;
+    unsynced?: LinkUnsyncedRoot[];
+}
+/** `path` under `home` as `~/...`, so no home directory leaves the machine in a report. */
+export declare function homeRelative(path: string, home: string): string;
+/** This machine: a stable id that is not its home path, and the name a person knows it by. */
+export declare const machineOf: (hostname: string, home: string) => Machine;
+/**
+ * One project checkout on one machine. Its port is not part of it. The tab's projectId is: a daemon
+ * that cannot map a project to its checkout files every such project under one shared directory.
+ */
+export declare const appKeyOf: (machineId: string, projectDir: string, projectId: string | undefined) => string;
+/** What a refused open is told. */
+export declare const NOT_AN_APP_ADDRESS = "Reticle only opens a web address on this machine or one of the apps already open here.";
+/**
+ * Whether the daemon may open an address the platform named: a web page on this machine, or one on
+ * an origin a connected tab already has. Never a file, and never anywhere else on the network.
+ */
+export declare function mayOpen(url: string, openUrls: readonly string[]): boolean;
+/** A project id as a name: the SDK stamps `<package>-<8 hex>`, and the hash means nothing to a person. */
+export declare const nameFromProjectId: (projectId: string) => string;
+/** The address as the app knows it: the params Reticle put on it to adopt a tab are not the app's. */
+export declare const withoutMarks: (raw: string) => string;
+/**
+ * The apps to report, grouped by the platform credential each one's project is linked with. The
+ * linked credential is reported even with no app open: it is how the chat knows the machine is there
+ * to open one. A tab whose project is not linked is left out: no chat can drive it.
+ */
+export declare function appsByPlatform(tabs: readonly AppTab[], linked: Platform | undefined, machineId: string, describe: (tab: AppTab) => Promise<{
+    platform?: Platform;
+    dir: string;
+    name: string;
+}>): Promise<{
+    platform: Platform;
+    apps: ReportedApp[];
+}[]>;
+type FetchLike = (url: string, init: RequestInit) => Promise<Response>;
+export interface AppReportDeps {
+    machine: Machine;
+    version: string;
+    /** What this daemon can do for the platform: it offers a person only what is here. */
+    capabilities: readonly LinkCapability[];
+    apps: () => Promise<{
+        platform: Platform;
+        apps: ReportedApp[];
+    }[]>;
+    /**
+     * Open an app so it connects: one somebody asked to bring back, named by its app key so the tab
+     * writes into that app's own `.reticle`. A throw is logged, never fatal.
+     */
+    open: (url: string, appKey?: string) => Promise<void>;
+    /** A drive is waiting for the project this credential is for. */
+    drivePending: (platform: Platform) => void;
+    /** Dev servers, the last refused hello and unsent runs: read once per report. */
+    extras?: () => Promise<ReportExtras>;
+    /**
+     * Something the platform wants the person told ("update Reticle"), for the project this credential
+     * is for. Called once per notice for the daemon's life, however often the platform repeats it.
+     */
+    notify?: (notice: LinkNotice, platform: Platform) => void;
+    fetch?: FetchLike;
+    intervalMs?: number;
+    log?: (line: string) => void;
+}
+export interface AppReports {
+    tick: () => Promise<void>;
+    stop: () => void;
+}
+export declare function startAppReports(deps: AppReportDeps): AppReports;
+/**
+ * Report now rather than on the next tick: the Harness gate just learned what an app's pages carry,
+ * and the platform, which locks a drive on its last report, would otherwise refuse on the old one.
+ */
+export declare const reportAppsNow: () => Promise<void>;
+/**
+ * The tab a drive the chat attached to one app uses: a tab of that app and no other, even on the
+ * same port. None open: open the app, and wait for it to connect. Null when it never does.
+ */
+export declare function pickAppTab(app: {
+    key: string;
+    url: string | null;
+}, goal: string, tabs: () => readonly AppTab[], keyOf: (sessionId: string) => string | undefined, open: (url: string) => Promise<void>, waitMs?: number): Promise<string | null>;
+export {};