@reticlehq/server 2.14.0 → 3.2.0

package/dist/features/harness/fill-values.js

@@ -0,0 +1,153 @@
+/**
+ * What to type into a field, and the one place in a drive worth paying a generating model for.
+ *
+ * A System One model answers typed questions against a state; it cannot write a string. That is not
+ * a shortcoming to route around everywhere — it is why the drive is two orders of magnitude cheaper,
+ * and every decision it makes (which tool, which element, what should follow) is a decision, not a
+ * composition. But a text field is a composition. "A card number", "a search term that returns
+ * results", "a description a human would write" cannot be enumerated from the page, and a label-keyed
+ * table of guesses is what stands in for them today.
+ *
+ * ## Why this is escalation rather than partnership
+ *
+ * Put a generating model in the per-turn loop and you pay its per-turn tax on every step, which is
+ * the cost the cheap driver exists to avoid. Typing is a small minority of what a drive does —
+ * clicks and navigations are the overwhelming majority — so the model is called for the few steps
+ * that compose text, never for the many that select, and a drive with no text fields never calls it
+ * at all.
+ *
+ * ## Why it is cached, and cached where it is
+ *
+ * A generated value is paid for ONCE and then belongs to the project: the same field on the same
+ * app wants the same value tomorrow, and a replay must send exactly what the recording sent or it
+ * is not a replay. Writing them to `.reticle` makes the second drive free, makes the values
+ * reviewable by a human who can correct a bad one by hand, and keeps them out of the model's way
+ * entirely. It is the same economics as the flows themselves: pay once, replay forever.
+ *
+ * ## What it must never do
+ *
+ * Fill a slot, never propose a call. The driver decides what to click and what to claim; this
+ * answers one question — "what text goes in this box" — and a wrong answer costs one act, not a
+ * wrong verdict. Nothing here touches what is judged.
+ */
+import { DEFAULT_HARNESS_BASE_URL, } from './driver.js';
+/** Deliberately small. A field value is a phrase; anything longer is the model misunderstanding. */
+const MAX_TOKENS = 64;
+const ANTHROPIC_VERSION = '2023-06-01';
+const MESSAGES_PATH = '/v1/messages';
+/**
+ * How many values one drive will generate.
+ *
+ * The cache makes the steady state nearly free, but the FIRST drive on a form-heavy app is the one
+ * that pays, and an uncapped loop on a page with forty inputs is a bill nobody agreed to. Past the
+ * cap the heuristic answers, which is what every drive used before this existed.
+ */
+const MAX_PER_DRIVE = 12;
+/**
+ * The fallback, and still the answer for most fields.
+ *
+ * An exploration drive needs PLAUSIBLE input, not creative input: the engine is judging what the app
+ * did with the value, not the value. A field that rejects everything here is itself worth finding,
+ * and shows up as an act that did not settle.
+ */
+export function heuristicFillValue(label) {
+    const l = label.toLowerCase();
+    if (l.includes('email'))
+        return 'harness@reticle.dev';
+    if (l.includes('password'))
+        return 'password';
+    if (l.includes('url') || l.includes('link'))
+        return 'https://example.com';
+    if (l.includes('phone') || l.includes('tel'))
+        return '5550100';
+    if (l.includes('date'))
+        return '2026-01-01';
+    if (l.includes('amount') || l.includes('price') || l.includes('qty') || l.includes('quantity'))
+        return '2';
+    if (l.includes('search') || l.includes('filter'))
+        return 'a';
+    if (l.includes('number') || l.includes('count'))
+        return '42';
+    return 'reticle harness';
+}
+/**
+ * Whether a label is worth spending a model call on.
+ *
+ * The heuristic is RIGHT about an email box and an amount box, and a generated "john@example.com"
+ * is not an improvement on `harness@reticle.dev`. What it cannot do is a field whose label it does
+ * not recognise at all, where it falls back to typing the words "reticle harness" into whatever the
+ * app asked for. That fallback is the whole of what generation is for.
+ */
+export function needsGeneration(label) {
+    return heuristicFillValue(label) === heuristicFillValue('');
+}
+const PROMPT = (label, context) => `A test harness is filling in a form field on a web application so it can check what the app does with the input.\n\n` +
+    `The field is labelled: "${label}"\n` +
+    `${context}\n\n` +
+    `Reply with ONLY the value to type. No quotes, no explanation, no markdown. It must be something a ` +
+    `real user would plausibly enter, and short. If the field wants an identifier or a code, invent a ` +
+    `well-formed one.`;
+/** One string from a generating model, or nothing. Never throws: a drive must not fail over a field. */
+async function generateOne(options, label, context) {
+    const call = options.fetch ?? ((url, init) => fetch(url, init));
+    const baseUrl = options.baseUrl ?? DEFAULT_HARNESS_BASE_URL;
+    try {
+        const res = await call(`${baseUrl}${MESSAGES_PATH}`, {
+            method: 'POST',
+            headers: {
+                'content-type': 'application/json',
+                'x-api-key': options.apiKey,
+                'anthropic-version': ANTHROPIC_VERSION,
+            },
+            body: JSON.stringify({
+                model: options.model ?? 'claude-sonnet-5',
+                max_tokens: MAX_TOKENS,
+                messages: [{ role: 'user', content: PROMPT(label, context) }],
+            }),
+        });
+        if (!res.ok)
+            return undefined;
+        const parsed = JSON.parse(await res.text());
+        const content = parsed.content ?? [];
+        const text = content
+            .filter((block) => 'text' === block.type)
+            .map((block) => block.text ?? '')
+            .join('')
+            .trim()
+            .split('\n')[0];
+        if (text === undefined || 0 === text.length)
+            return undefined;
+        // A model asked for one value occasionally answers with a sentence about the value. Anything
+        // long enough to be prose is not a field value, and typing prose proves nothing about the app.
+        return 80 < text.length ? undefined : text.replace(/^["']|["']$/g, '');
+    }
+    catch {
+        // Offline, refused, rate-limited, malformed. All the same answer: use the heuristic.
+        return undefined;
+    }
+}
+/**
+ * The seam the driver calls: a label in, a value out, always.
+ *
+ * Three layers, cheapest first, and the expensive one is skipped entirely unless the two above it
+ * have nothing better than "type the words reticle harness into this box".
+ */
+export function fillValues(deps) {
+    let spent = 0;
+    return async (label) => {
+        const cached = deps.cache?.get(label);
+        if (cached !== undefined)
+            return cached;
+        const fallback = heuristicFillValue(label);
+        if (deps.generator === undefined || !needsGeneration(label) || MAX_PER_DRIVE <= spent)
+            return fallback;
+        spent += 1;
+        const generated = await generateOne(deps.generator, label, deps.context?.() ?? '');
+        if (generated === undefined)
+            return fallback;
+        // Cached under the LABEL, not the ref: refs expire with the page, labels are what the next drive
+        // and every replay will meet again.
+        deps.cache?.set(label, generated);
+        return generated;
+    };
+}

package/dist/features/harness/harness.d.ts

@@ -0,0 +1,159 @@
+/**
+ * The harness: a model driving Reticle's own tool surface, so a verification needs no coding agent.
+ *
+ * The premise that the AGENT drives is not abandoned here — an agent pointed at the daemon still gets
+ * exactly that. This is for the case where there is no agent in the loop, or where having one is the
+ * expensive part: a preview URL from CI, a persona journey nobody wants to pay 50k context tokens to
+ * re-drive, a first run on an app with no saved flows at all.
+ *
+ * Three properties are load-bearing:
+ *
+ * 1. **It drives the SAME surface an agent would.** Not a private API, not a shortcut into the
+ *    session. If the harness ever needs something the tool surface does not expose, the tool surface
+ *    is missing a tool and that is the bug to fix.
+ * 2. **It decides nothing.** The model chooses what to TRY; the engine decides what HAPPENED. A model
+ *    that graded its own driving would be scoring its own homework, and the verdict would be
+ *    unfalsifiable. The verdict is built from the flows it saved and the journal it left behind.
+ * 3. **Everything is injected** — the model, the tools, the clock. The loop below has no network in
+ *    it, which is what lets the whole thing be tested against a scripted model with no API key.
+ */
+/** One tool as the model sees it. JSON Schema, because that is what the wire wants. */
+export interface HarnessTool {
+    name: string;
+    description: string;
+    inputSchema: Record<string, unknown>;
+}
+/** What the harness may do to the app. Supplied by the caller; the loop never imports a transport. */
+export interface HarnessToolset {
+    tools: readonly HarnessTool[];
+    invoke(name: string, args: Record<string, unknown>): Promise<unknown>;
+}
+/** One call the model asked for. */
+export interface ToolRequest {
+    id: string;
+    name: string;
+    args: Record<string, unknown>;
+}
+/** What a turn cost, including how much of it was served from cache.
+ *
+ * The cache fields are the only way to CONFIRM the prompt is being cached rather than merely marked
+ * for caching. A breakpoint that silently misses — a byte that moved in the prefix, a turn that added
+ * more blocks than the lookback window — costs full price and looks identical from here otherwise.
+ * Caching is most of what makes a drive affordable, so a missed one has to be visible. */
+export interface TokenUsage {
+    /** Processed at full price: the part of this request that was not cached. */
+    input: number;
+    output: number;
+    /** Served from cache at roughly a tenth of the input price. */
+    cacheRead: number;
+    /** Written to cache at roughly 1.25x. Paid once per new prefix. */
+    cacheWrite: number;
+}
+/** What a model turn produced: some prose, and zero or more calls. */
+export interface ModelTurn {
+    text: string;
+    calls: readonly ToolRequest[];
+    /** Tokens this turn cost, when the driver can report them. */
+    usage?: TokenUsage;
+}
+/** The result of one tool call, as it goes back to the model. */
+export interface ToolOutcome {
+    id: string;
+    name: string;
+    /**
+     * What the model asked for, kept so the run can be read back as actions rather than results.
+     *
+     * The result alone says what happened; only the arguments say what was ATTEMPTED. Two consumers
+     * need that and neither can recover it: a report that lists what was driven, and promoting a drive
+     * to a saved flow, which records the anchor plus the action. Without this, two clicks on different
+     * buttons are indistinguishable after the fact.
+     */
+    args: Record<string, unknown>;
+    result: unknown;
+    isError: boolean;
+}
+/** The conversation, in the loop's own shape. The driver translates it to whatever the wire wants. */
+export type HistoryEntry = {
+    role: 'user';
+    text: string;
+} | {
+    role: 'assistant';
+    text: string;
+    calls: readonly ToolRequest[];
+} | {
+    role: 'tool';
+    outcomes: readonly ToolOutcome[];
+};
+/**
+ * The model, reduced to the only thing the loop needs from it.
+ *
+ * A driver is handed the full history each turn rather than being asked to hold it, because the loop
+ * owns what gets remembered — that is where budget enforcement and truncation have to live.
+ */
+export interface ModelDriver {
+    turn(input: {
+        system: string;
+        tools: readonly HarnessTool[];
+        history: readonly HistoryEntry[];
+    }): Promise<ModelTurn>;
+}
+/** Why a run stopped. Every one of these is a real outcome; none of them is an error. */
+export declare const StopReason: {
+    /** The model called `finish` — it believes it is done. */
+    readonly FINISHED: "finished";
+    /** The step budget ran out. The run is still gradeable; it is just incomplete. */
+    readonly BUDGET: "budget";
+    /** The model stopped asking for tools without finishing. Rare, and worth counting. */
+    readonly STALLED: "stalled";
+    /** The toolset or the model itself failed in a way the loop cannot continue through. */
+    readonly BROKEN: "broken";
+};
+export type StopReason = (typeof StopReason)[keyof typeof StopReason];
+export interface HarnessResult {
+    stopReason: StopReason;
+    /** The model's own account of what it drove. NOT a verdict — nothing downstream grades from it. */
+    summary: string;
+    steps: number;
+    toolCalls: readonly ToolOutcome[];
+    usage: TokenUsage;
+    /** Set when `stopReason` is BROKEN. */
+    error?: string;
+}
+/**
+ * The one tool the harness adds to Reticle's surface.
+ *
+ * It exists so "I am done" is a decision the model states explicitly rather than something inferred
+ * from it going quiet — a run that ends because a model had nothing more to say is indistinguishable
+ * from one that ended because the model lost track, and those need different handling.
+ */
+export declare const FINISH_TOOL: HarnessTool;
+export interface HarnessOptions {
+    /** Hard ceiling on model turns. The run is graded whatever happens, so this bounds cost, not value. */
+    maxSteps?: number;
+    /** Extra standing instruction appended to the system prompt (a persona, an intent, a named flow). */
+    focus?: string;
+    /** How long one model turn may take. Injected so a test can prove the bound without waiting for it. */
+    turnTimeoutMs?: number;
+}
+export declare const DEFAULT_MAX_STEPS = 40;
+/**
+ * The standing instruction.
+ *
+ * Written for a SMALL model on purpose. It states the loop, the vocabulary, and the one rule that
+ * makes a drive worth anything — name the consequence BEFORE acting — and then gets out of the way.
+ * It deliberately does not teach bug-finding: the oracles do that, deterministically, and a model
+ * told to hunt for bugs will report its guesses as findings.
+ *
+ * The recording instruction is what makes the drive worth paying for twice. A drive that is not
+ * saved as a flow has to be re-driven by a model on every future run; a drive that IS saved replays
+ * deterministically for a few hundred tokens, forever. That is the whole economics of this feature,
+ * so it is an instruction and not a hope.
+ */
+export declare function systemPrompt(focus?: string): string;
+/**
+ * Run the harness to completion.
+ *
+ * Pure orchestration: no network, no clock, no filesystem. Everything that touches the world arrives
+ * through `driver` and `toolset`, which is what makes this testable against a scripted model.
+ */
+export declare function runHarness(driver: ModelDriver, toolset: HarnessToolset, options?: HarnessOptions): Promise<HarnessResult>;

package/dist/features/harness/harness.js

@@ -0,0 +1,203 @@
+/**
+ * The harness: a model driving Reticle's own tool surface, so a verification needs no coding agent.
+ *
+ * The premise that the AGENT drives is not abandoned here — an agent pointed at the daemon still gets
+ * exactly that. This is for the case where there is no agent in the loop, or where having one is the
+ * expensive part: a preview URL from CI, a persona journey nobody wants to pay 50k context tokens to
+ * re-drive, a first run on an app with no saved flows at all.
+ *
+ * Three properties are load-bearing:
+ *
+ * 1. **It drives the SAME surface an agent would.** Not a private API, not a shortcut into the
+ *    session. If the harness ever needs something the tool surface does not expose, the tool surface
+ *    is missing a tool and that is the bug to fix.
+ * 2. **It decides nothing.** The model chooses what to TRY; the engine decides what HAPPENED. A model
+ *    that graded its own driving would be scoring its own homework, and the verdict would be
+ *    unfalsifiable. The verdict is built from the flows it saved and the journal it left behind.
+ * 3. **Everything is injected** — the model, the tools, the clock. The loop below has no network in
+ *    it, which is what lets the whole thing be tested against a scripted model with no API key.
+ */
+import { withTimeout } from './with-timeout.js';
+/** Why a run stopped. Every one of these is a real outcome; none of them is an error. */
+export const StopReason = {
+    /** The model called `finish` — it believes it is done. */
+    FINISHED: 'finished',
+    /** The step budget ran out. The run is still gradeable; it is just incomplete. */
+    BUDGET: 'budget',
+    /** The model stopped asking for tools without finishing. Rare, and worth counting. */
+    STALLED: 'stalled',
+    /** The toolset or the model itself failed in a way the loop cannot continue through. */
+    BROKEN: 'broken',
+};
+/**
+ * The one tool the harness adds to Reticle's surface.
+ *
+ * It exists so "I am done" is a decision the model states explicitly rather than something inferred
+ * from it going quiet — a run that ends because a model had nothing more to say is indistinguishable
+ * from one that ended because the model lost track, and those need different handling.
+ */
+export const FINISH_TOOL = {
+    name: 'finish',
+    description: 'Call this when you have finished driving the app, or when you cannot make further progress. Summarise what you drove and what you observed. This does not decide whether the app is correct — the engine does that from what it recorded.',
+    inputSchema: {
+        type: 'object',
+        properties: {
+            summary: {
+                type: 'string',
+                description: 'What you drove, in a few sentences: which flows you exercised, what you asserted, and anything you could not reach.',
+            },
+        },
+        required: ['summary'],
+        additionalProperties: false,
+    },
+};
+export const DEFAULT_MAX_STEPS = 40;
+/**
+ * How long a single model turn may take before the run gives up on it.
+ *
+ * Without a bound, `await driver.turn(...)` can hang forever, and a hung turn does not fail quietly —
+ * it holds a leased browser context for as long as it lasts, with nothing anywhere saying why. A
+ * provider stall, a dropped connection or a proxy that accepts and never answers all arrive this way.
+ *
+ * Two minutes is far past any real turn (measured turns are seconds) and far short of an outage. A
+ * turn that times out is a BROKEN drive, and a broken drive is never reported as clean — so this
+ * cannot turn a stall into a false green.
+ */
+const DEFAULT_TURN_TIMEOUT_MS = 120_000;
+/** The opening message. The app is already connected by the time the loop starts. */
+const OPENING = 'The app is loaded and connected. Begin by looking at the page, then drive it.';
+/**
+ * The standing instruction.
+ *
+ * Written for a SMALL model on purpose. It states the loop, the vocabulary, and the one rule that
+ * makes a drive worth anything — name the consequence BEFORE acting — and then gets out of the way.
+ * It deliberately does not teach bug-finding: the oracles do that, deterministically, and a model
+ * told to hunt for bugs will report its guesses as findings.
+ *
+ * The recording instruction is what makes the drive worth paying for twice. A drive that is not
+ * saved as a flow has to be re-driven by a model on every future run; a drive that IS saved replays
+ * deterministically for a few hundred tokens, forever. That is the whole economics of this feature,
+ * so it is an instruction and not a hope.
+ */
+export function systemPrompt(focus) {
+    return [
+        'You are driving a real web application through a verification tool, to find out whether it actually works.',
+        '',
+        'The loop is: look at the page, act on it, and state what should happen BEFORE you act. Naming the',
+        'expected consequence in advance is the difference between a check and a rationalisation.',
+        '',
+        'Use reticle_snapshot to see the page and get element refs, and reticle_query to find one control.',
+        'Use reticle_act_and_wait with `until` to act and get a verdict in one call — that is the tool',
+        'that proves something. reticle_act just performs an action and proves nothing. Use',
+        'reticle_observe and reticle_state to see what the app did. Prefer asserting a signal, a network',
+        'request, or a store value over asserting that some text is on screen — text can be right while',
+        'the app is broken underneath.',
+        '',
+        'Drive the app the way a user would: complete whole journeys, not single clicks. Submit the forms.',
+        'Follow the navigation. If something looks interactive, try it.',
+        '',
+        'RECORD WHAT YOU DRIVE. Before each journey call reticle_record { action: "start" }, drive it, then',
+        'reticle_flow_save with a short descriptive name. A saved flow replays deterministically on every',
+        'future run with no model in the loop, so an unrecorded drive is work that has to be paid for again.',
+        '',
+        'Real users are impatient, and impatience is where a whole class of defect lives. When several',
+        'controls do the same kind of thing (filters, tabs, symbols, tags), drive two of them back to back',
+        'in ONE reticle_act_sequence with `steps`, rather than waiting for the first to finish. An app that',
+        'handles them one at a time and an app that races them look identical until someone hurries.',
+        '',
+        'You are NOT judging whether the app is correct — the engine records everything you do and decides',
+        'that afterwards. Your job is coverage: exercise as much real behaviour as you can, and state clear',
+        'expectations as you go. Do not report bugs; drive the app.',
+        '',
+        'Call `finish` when you have covered the app or cannot get further.',
+        ...(focus === undefined ? [] : ['', `Focus for this run: ${focus}`]),
+    ].join('\n');
+}
+/**
+ * Run the harness to completion.
+ *
+ * Pure orchestration: no network, no clock, no filesystem. Everything that touches the world arrives
+ * through `driver` and `toolset`, which is what makes this testable against a scripted model.
+ */
+export async function runHarness(driver, toolset, options = {}) {
+    const maxSteps = options.maxSteps ?? DEFAULT_MAX_STEPS;
+    const tools = [...toolset.tools, FINISH_TOOL];
+    const system = systemPrompt(options.focus);
+    const history = [{ role: 'user', text: OPENING }];
+    const toolCalls = [];
+    const usage = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 };
+    for (let step = 0; step < maxSteps; step += 1) {
+        let turn;
+        try {
+            turn = await withTimeout(driver.turn({ system, tools, history }), options.turnTimeoutMs ?? DEFAULT_TURN_TIMEOUT_MS, 'the model did not answer');
+        }
+        catch (error) {
+            return {
+                stopReason: StopReason.BROKEN,
+                summary: '',
+                steps: step,
+                toolCalls,
+                usage,
+                error: error instanceof Error ? error.message : String(error),
+            };
+        }
+        if (turn.usage !== undefined) {
+            usage.input += turn.usage.input;
+            usage.output += turn.usage.output;
+            usage.cacheRead += turn.usage.cacheRead;
+            usage.cacheWrite += turn.usage.cacheWrite;
+        }
+        if (0 === turn.calls.length) {
+            // Nothing asked for and no `finish`. Not an error, and not success either — naming it keeps
+            // "the model gave up" out of the same bucket as "the model completed the work".
+            return {
+                stopReason: StopReason.STALLED,
+                summary: turn.text,
+                steps: step + 1,
+                toolCalls,
+                usage,
+            };
+        }
+        history.push({ role: 'assistant', text: turn.text, calls: turn.calls });
+        const finish = turn.calls.find((call) => FINISH_TOOL.name === call.name);
+        if (finish !== undefined) {
+            const summary = finish.args['summary'];
+            return {
+                stopReason: StopReason.FINISHED,
+                summary: 'string' === typeof summary ? summary : turn.text,
+                steps: step + 1,
+                toolCalls,
+                usage,
+            };
+        }
+        // Every call the model asked for is answered, including ones that fail. A dropped result leaves
+        // the model waiting on an answer that never comes, and it will usually re-ask — burning the
+        // budget on a call that already ran, against an app it already changed.
+        const outcomes = await Promise.all(turn.calls.map((call) => invokeOne(toolset, call)));
+        toolCalls.push(...outcomes);
+        history.push({ role: 'tool', outcomes });
+    }
+    return { stopReason: StopReason.BUDGET, summary: '', steps: maxSteps, toolCalls, usage };
+}
+/**
+ * One tool call, with its failure captured rather than thrown.
+ *
+ * A tool that throws is information for the model — a wrong ref, a control that is not there yet —
+ * and it recovers from that routinely. Letting the throw escape would end a run over something the
+ * model was about to handle.
+ */
+async function invokeOne(toolset, call) {
+    try {
+        const result = await toolset.invoke(call.name, call.args);
+        return { id: call.id, name: call.name, args: call.args, result, isError: false };
+    }
+    catch (error) {
+        return {
+            id: call.id,
+            name: call.name,
+            args: call.args,
+            result: { error: error instanceof Error ? error.message : String(error) },
+            isError: true,
+        };
+    }
+}