@reticlehq/server 3.1.0 → 3.3.0

package/dist/features/harness/drive-report.js

@@ -0,0 +1,196 @@
+/**
+ * What a drive did, written from the evidence rather than narrated by the model that drove.
+ *
+ * `HarnessResult.summary` is the driver's own account, and it has always carried a warning that
+ * nothing downstream grades it. That was a caveat when the driver was a language model. It became a
+ * hole the moment a System One model could drive: Jev answers typed questions and cannot produce a
+ * sentence, so `summary` came back EMPTY, and an agent calling `explore` got a result with no
+ * account of what had happened in it at all.
+ *
+ * Restoring prose would have been the obvious fix and the wrong one. The drive already records
+ * every call it made, with the arguments it used and the verdict the engine returned — so the
+ * account can be DERIVED. That is strictly better than a narration: a model summarising its own
+ * drive is the one witness with a reason to round "unknown" up to "worked", and this cannot, because
+ * it is not writing about the drive, it is reading it.
+ *
+ * Written for the agent on the other end of the MCP call. It leads with what was proved, because
+ * that is what a coding agent has to gate on, and it says "not proved" in those words rather than
+ * leaving a verdict of `unknown` to be read as a pass.
+ */
+import { asRecord, asString, ReticleTool, Verified } from '@reticlehq/core';
+/** How many steps are listed before the account starts counting instead of naming. */
+const MAX_LISTED = 12;
+/** Actions that change the app. Reads are not part of the story of what was driven. */
+const DRIVING_TOOLS = new Set([
+    ReticleTool.ACT,
+    ReticleTool.ACT_AND_WAIT,
+    ReticleTool.ACT_SEQUENCE,
+    ReticleTool.NAVIGATE,
+]);
+/**
+ * The human-readable name of what was acted on.
+ *
+ * The ref (`e42`) is meaningless to the agent reading this — it is a handle that expired when the
+ * page changed. A description that says `button "Create"` survives being read tomorrow, so the ref
+ * is only the fallback.
+ */
+function targetOf(args, result) {
+    const described = asString(result['element']) ?? asString(result['name']);
+    if (described !== undefined && 0 < described.length)
+        return described;
+    const target = asRecord(args['target']);
+    for (const key of ['testid', 'text', 'label', 'name']) {
+        const value = asString(target[key]);
+        if (value !== undefined && 0 < value.length)
+            return `${key}=${value}`;
+    }
+    const url = asString(args['url']);
+    if (url !== undefined && 0 < url.length)
+        return url;
+    return asString(args['ref']) ?? 'the page';
+}
+/**
+ * Replays this drive ran, and what each came back with.
+ *
+ * The verdict is `status`, not `passed`. Reading the wrong key reported ten replays as "undecided"
+ * on a run where every one of them had answered — which is the same false-nothing this report was
+ * written to stop, made by the report itself.
+ *
+ * Three outcomes, kept apart because they mean different things to whoever reads this. `ok` is the
+ * journey still holding. `error` is it failing. `drift` is the flow's own anchors no longer
+ * resolving — the app moved under a recording, which is a finding about the RECORDING rather than
+ * proof the feature broke, and collapsing it into "failed" would send somebody to fix working code.
+ */
+export const ReplayOutcome = { OK: 'ok', DRIFT: 'drift', ERROR: 'error' };
+export function replayedFlows(toolCalls) {
+    const out = [];
+    for (const call of toolCalls) {
+        if (ReticleTool.FLOW_REPLAY !== call.name)
+            continue;
+        const name = asString(asRecord(call.args)['flowName']) ?? 'a flow';
+        out.push({
+            name,
+            status: call.isError ? ReplayOutcome.ERROR : asString(asRecord(call.result)['status']),
+        });
+    }
+    return out;
+}
+/**
+ * The declared consequence, as a short phrase rather than a JSON dump.
+ *
+ * One line per kind, because a reader triaging a red needs "it claimed a POST" and not the
+ * predicate's wire shape. Unknown kinds fall back to the kind name, which is still more than the
+ * nothing this used to say.
+ */
+function describeClaim(until) {
+    const p = asRecord(until);
+    const kind = asString(p['kind']);
+    if (kind === undefined)
+        return undefined;
+    if ('signal' === kind) {
+        const name = asString(p['name']);
+        return name === undefined ? 'a signal' : `signal ${name}`;
+    }
+    if ('net' === kind) {
+        const method = asString(p['method']);
+        const url = asString(p['urlContains']);
+        return `a ${method ?? ''} request${url === undefined ? '' : ` to ${url}`}`.replace('  ', ' ');
+    }
+    if ('route' === kind)
+        return `a route containing ${asString(p['contains']) ?? '?'}`;
+    if ('not' === kind) {
+        const inner = describeClaim(p['predicate']);
+        return inner === undefined ? 'not something' : `to leave ${inner}`;
+    }
+    if ('anyOf' === kind)
+        return 'a request or a signal';
+    return kind;
+}
+/** Reduce the raw call log to the actions that actually drove the app. */
+export function drivenSteps(toolCalls) {
+    const steps = [];
+    for (const call of toolCalls) {
+        if (!DRIVING_TOOLS.has(call.name))
+            continue;
+        const args = asRecord(call.args);
+        const result = asRecord(call.result);
+        const action = ReticleTool.NAVIGATE === call.name ? 'navigate' : (asString(args['action']) ?? 'act');
+        steps.push({
+            action,
+            target: targetOf(args, result),
+            verified: call.isError ? 'error' : asString(result['verified']),
+            because: call.isError ? asString(result['error']) : asString(result['because']),
+            claimed: describeClaim(args['until']),
+        });
+    }
+    return steps;
+}
+/**
+ * The account, as lines.
+ *
+ * Proved and not-proved are counted separately and both are stated, because the failure mode this
+ * whole product exists to prevent is a reader taking "we did twelve things" for "twelve things
+ * work". `unknown` is deliberately grouped under NOT proved and labelled as undecided evidence
+ * rather than as a failure — it calls for a better check, not a code change, and collapsing the two
+ * would send an agent to rewrite working code.
+ */
+export function describeDrive(toolCalls, savedFlows) {
+    const steps = drivenSteps(toolCalls);
+    const replays = replayedFlows(toolCalls);
+    /*
+     * A run that only REPLAYED is not a run that did nothing.
+     *
+     * "Nothing was driven" was true of the actions and wrong about the run: sixteen recorded journeys
+     * replayed deterministically, for zero model tokens, and the report called it empty. That is the
+     * cheap half of the plan working exactly as intended, and it has to read as a result.
+     */
+    const replayLine = 0 === replays.length
+        ? undefined
+        : (() => {
+            const held = replays.filter((r) => ReplayOutcome.OK === r.status);
+            const failed = replays.filter((r) => ReplayOutcome.ERROR === r.status);
+            const drifted = replays.filter((r) => ReplayOutcome.DRIFT === r.status);
+            const lines = [
+                `Replayed ${String(replays.length)} recorded journey(s) with NO model in the loop: ` +
+                    `${String(held.length)} still hold, ${String(failed.length)} failed, ${String(drifted.length)} drifted.`,
+            ];
+            if (0 < failed.length)
+                lines.push(`  FAILED: ${failed.map((r) => r.name).join(', ')} — regressions in journeys that used to pass.`);
+            if (0 < drifted.length)
+                lines.push(`  DRIFTED: ${drifted.map((r) => r.name).join(', ')} — the app moved under these recordings; the flow needs re-anchoring, not the app fixing.`);
+            return lines.join('\n');
+        })();
+    if (0 === steps.length)
+        return (replayLine ??
+            'Nothing was driven and nothing was replayed: the run made no action against the app.');
+    const proved = steps.filter((step) => Verified.YES === step.verified);
+    const failed = steps.filter((step) => Verified.NO === step.verified);
+    const undecided = steps.filter((step) => Verified.YES !== step.verified && Verified.NO !== step.verified);
+    const lines = [
+        ...(replayLine === undefined ? [] : [replayLine]),
+        `Drove ${String(steps.length)} action(s): ${String(proved.length)} proved, ` +
+            `${String(failed.length)} failed, ${String(undecided.length)} not decided.`,
+    ];
+    for (const step of steps.slice(0, MAX_LISTED)) {
+        const verdict = step.verified ?? 'nothing declared';
+        const because = step.because === undefined ? '' : ` — ${step.because}`;
+        // The claim, on failures only: it is what separates "the app is broken" from "the drive guessed
+        // wrong", and those two readings need opposite responses from whoever reads this.
+        // Shown on anything that did not pass, not just on a red. `no-fault` means "the consequence was
+        // already true, so this proved nothing" -- which is unreadable without knowing WHAT was claimed,
+        // and is the line that exposed a GET offer being satisfied by a dashboard's own polling.
+        const claimed = Verified.YES !== step.verified && step.claimed !== undefined
+            ? ` (claimed ${step.claimed})`
+            : '';
+        lines.push(`  ${step.action} ${step.target}: ${verdict}${claimed}${because}`);
+    }
+    if (MAX_LISTED < steps.length)
+        lines.push(`  … and ${String(steps.length - MAX_LISTED)} more.`);
+    if (0 < failed.length)
+        lines.push(`${String(failed.length)} action(s) FAILED — the app did not do what the drive declared it would. These are findings.`);
+    if (0 < undecided.length)
+        lines.push(`${String(undecided.length)} action(s) were NOT PROVED. That is undecided evidence, not a failure: it calls for a better check, not a code change.`);
+    if (0 < savedFlows.length)
+        lines.push(`Replay any of this without a model: reticle_verify { action: "flows" } — saved ${savedFlows.join(', ')}.`);
+    return lines.join('\n');
+}

package/dist/features/harness/driver.js

@@ -11,7 +11,7 @@
  * purpose.
  */
 import { z } from 'zod';
-import { ReticleEnv } from '@reticlehq/core';
+import { ReticleEnv, apiKeyFrom } from '@reticlehq/core';
 /**
  * The default driver model.
  *
@@ -67,16 +67,37 @@ const MessageResponse = z.object({
  * routine answer a caller offers a way out of, not a failure.
  */
 export function harnessOptionsFromEnv(env) {
-    const apiKey = env[ReticleEnv.HARNESS_KEY];
-    if (apiKey === undefined || 0 === apiKey.length)
-        return undefined;
     const model = env[ReticleEnv.HARNESS_MODEL];
     const baseUrl = env[ReticleEnv.HARNESS_BASE_URL];
-    return {
-        apiKey,
-        ...(model === undefined || 0 === model.length ? {} : { model }),
-        ...(baseUrl === undefined || 0 === baseUrl.length ? {} : { baseUrl }),
+    const withOptions = (apiKey, fallbackUrl) => {
+        const url = baseUrl === undefined || 0 === baseUrl.length ? fallbackUrl : baseUrl;
+        return {
+            apiKey,
+            ...(model === undefined || 0 === model.length ? {} : { model }),
+            ...(url === undefined || 0 === url.length ? {} : { baseUrl: url }),
+        };
     };
+    const apiKey = env[ReticleEnv.HARNESS_KEY];
+    if (apiKey !== undefined && 0 < apiKey.length)
+        return withOptions(apiKey);
+    /**
+     * The platform key, against the platform.
+     *
+     * Reticle's own service answers the Messages API at `/v1/messages` and forwards with ITS provider
+     * key, so a user who has run `reticle link` can drive with a frontier model having never held an
+     * Anthropic key at all. Without this they would be told to set `ANTHROPIC_API_KEY` to a value
+     * beginning `rk_live_` — which works, and reads like a mistake, and is the sort of instruction
+     * people correct on the way past and then cannot explain why nothing runs.
+     *
+     * An explicit `ANTHROPIC_API_KEY` still wins: someone who exported one meant it.
+     */
+    const cloudKey = apiKeyFrom(env);
+    const cloudUrl = env[ReticleEnv.CLOUD_URL];
+    if (cloudKey === undefined || 0 === cloudKey.length)
+        return undefined;
+    if (cloudUrl === undefined || 0 === cloudUrl.length)
+        return undefined;
+    return withOptions(cloudKey, cloudUrl);
 }
 /** Build a driver backed by the Messages API. */
 export function harnessDriver(options) {

package/dist/features/harness/drivers.d.ts

@@ -0,0 +1,19 @@
+/**
+ * The names of the drivers the harness can be asked for.
+ *
+ * A leaf module on purpose. These used to be re-exported from `harness-explore.ts`, which sits in
+ * the tool surface and is reached, through the toolset and the tool table, from the very tool that
+ * needs to declare them as an enum — a cycle that left the list `[undefined, undefined]` at the
+ * moment zod read it, and failed inside a JSON-schema parser rather than anywhere near the cause.
+ * Nothing here imports anything, so nothing can be half-initialised when it is read.
+ */
+/** Generates its tool calls as text, and names the journeys it records. The default. */
+export declare const ANTHROPIC_DRIVER_NAME = "anthropic";
+/** Answers typed questions; picks from the elements on the page and cannot invent one. */
+export declare const JEV_DRIVER_NAME = "jev";
+/** Generates its tool calls, over Chat Completions. Here so an A/B has a third arm. */
+export declare const OPENAI_DRIVER_NAME = "openai";
+/** What a caller may name. Order is the order a reader sees them in the tool description. */
+export declare const DRIVER_NAMES: readonly ["anthropic", "jev", "openai"];
+/** An injected driver: not selectable, because only a caller in-process can supply one. */
+export declare const CUSTOM_DRIVER_NAME = "custom";

package/dist/features/harness/drivers.js

@@ -0,0 +1,19 @@
+/**
+ * The names of the drivers the harness can be asked for.
+ *
+ * A leaf module on purpose. These used to be re-exported from `harness-explore.ts`, which sits in
+ * the tool surface and is reached, through the toolset and the tool table, from the very tool that
+ * needs to declare them as an enum — a cycle that left the list `[undefined, undefined]` at the
+ * moment zod read it, and failed inside a JSON-schema parser rather than anywhere near the cause.
+ * Nothing here imports anything, so nothing can be half-initialised when it is read.
+ */
+/** Generates its tool calls as text, and names the journeys it records. The default. */
+export const ANTHROPIC_DRIVER_NAME = 'anthropic';
+/** Answers typed questions; picks from the elements on the page and cannot invent one. */
+export const JEV_DRIVER_NAME = 'jev';
+/** Generates its tool calls, over Chat Completions. Here so an A/B has a third arm. */
+export const OPENAI_DRIVER_NAME = 'openai';
+/** What a caller may name. Order is the order a reader sees them in the tool description. */
+export const DRIVER_NAMES = [ANTHROPIC_DRIVER_NAME, JEV_DRIVER_NAME, OPENAI_DRIVER_NAME];
+/** An injected driver: not selectable, because only a caller in-process can supply one. */
+export const CUSTOM_DRIVER_NAME = 'custom';

package/dist/features/harness/fill-values.d.ts

@@ -0,0 +1,67 @@
+/**
+ * What to type into a field, and the one place in a drive worth paying a generating model for.
+ *
+ * A System One model answers typed questions against a state; it cannot write a string. That is not
+ * a shortcoming to route around everywhere — it is why the drive is two orders of magnitude cheaper,
+ * and every decision it makes (which tool, which element, what should follow) is a decision, not a
+ * composition. But a text field is a composition. "A card number", "a search term that returns
+ * results", "a description a human would write" cannot be enumerated from the page, and a label-keyed
+ * table of guesses is what stands in for them today.
+ *
+ * ## Why this is escalation rather than partnership
+ *
+ * Put a generating model in the per-turn loop and you pay its per-turn tax on every step, which is
+ * the cost the cheap driver exists to avoid. Typing is a small minority of what a drive does —
+ * clicks and navigations are the overwhelming majority — so the model is called for the few steps
+ * that compose text, never for the many that select, and a drive with no text fields never calls it
+ * at all.
+ *
+ * ## Why it is cached, and cached where it is
+ *
+ * A generated value is paid for ONCE and then belongs to the project: the same field on the same
+ * app wants the same value tomorrow, and a replay must send exactly what the recording sent or it
+ * is not a replay. Writing them to `.reticle` makes the second drive free, makes the values
+ * reviewable by a human who can correct a bad one by hand, and keeps them out of the model's way
+ * entirely. It is the same economics as the flows themselves: pay once, replay forever.
+ *
+ * ## What it must never do
+ *
+ * Fill a slot, never propose a call. The driver decides what to click and what to claim; this
+ * answers one question — "what text goes in this box" — and a wrong answer costs one act, not a
+ * wrong verdict. Nothing here touches what is judged.
+ */
+import { type HarnessDriverOptions } from './driver.js';
+/** What the caller must be able to do for values to survive past this drive. */
+export interface FillCache {
+    get(label: string): string | undefined;
+    set(label: string, value: string): void;
+}
+/**
+ * The fallback, and still the answer for most fields.
+ *
+ * An exploration drive needs PLAUSIBLE input, not creative input: the engine is judging what the app
+ * did with the value, not the value. A field that rejects everything here is itself worth finding,
+ * and shows up as an act that did not settle.
+ */
+export declare function heuristicFillValue(label: string): string;
+/**
+ * Whether a label is worth spending a model call on.
+ *
+ * The heuristic is RIGHT about an email box and an amount box, and a generated "john@example.com"
+ * is not an improvement on `harness@reticle.dev`. What it cannot do is a field whose label it does
+ * not recognise at all, where it falls back to typing the words "reticle harness" into whatever the
+ * app asked for. That fallback is the whole of what generation is for.
+ */
+export declare function needsGeneration(label: string): boolean;
+/**
+ * The seam the driver calls: a label in, a value out, always.
+ *
+ * Three layers, cheapest first, and the expensive one is skipped entirely unless the two above it
+ * have nothing better than "type the words reticle harness into this box".
+ */
+export declare function fillValues(deps: {
+    cache?: FillCache;
+    generator?: HarnessDriverOptions;
+    /** One line about where the drive is, so a value suits the page it is typed on. */
+    context?: () => string;
+}): (label: string) => Promise<string>;

package/dist/features/harness/fill-values.js

@@ -0,0 +1,153 @@
+/**
+ * What to type into a field, and the one place in a drive worth paying a generating model for.
+ *
+ * A System One model answers typed questions against a state; it cannot write a string. That is not
+ * a shortcoming to route around everywhere — it is why the drive is two orders of magnitude cheaper,
+ * and every decision it makes (which tool, which element, what should follow) is a decision, not a
+ * composition. But a text field is a composition. "A card number", "a search term that returns
+ * results", "a description a human would write" cannot be enumerated from the page, and a label-keyed
+ * table of guesses is what stands in for them today.
+ *
+ * ## Why this is escalation rather than partnership
+ *
+ * Put a generating model in the per-turn loop and you pay its per-turn tax on every step, which is
+ * the cost the cheap driver exists to avoid. Typing is a small minority of what a drive does —
+ * clicks and navigations are the overwhelming majority — so the model is called for the few steps
+ * that compose text, never for the many that select, and a drive with no text fields never calls it
+ * at all.
+ *
+ * ## Why it is cached, and cached where it is
+ *
+ * A generated value is paid for ONCE and then belongs to the project: the same field on the same
+ * app wants the same value tomorrow, and a replay must send exactly what the recording sent or it
+ * is not a replay. Writing them to `.reticle` makes the second drive free, makes the values
+ * reviewable by a human who can correct a bad one by hand, and keeps them out of the model's way
+ * entirely. It is the same economics as the flows themselves: pay once, replay forever.
+ *
+ * ## What it must never do
+ *
+ * Fill a slot, never propose a call. The driver decides what to click and what to claim; this
+ * answers one question — "what text goes in this box" — and a wrong answer costs one act, not a
+ * wrong verdict. Nothing here touches what is judged.
+ */
+import { DEFAULT_HARNESS_BASE_URL, } from './driver.js';
+/** Deliberately small. A field value is a phrase; anything longer is the model misunderstanding. */
+const MAX_TOKENS = 64;
+const ANTHROPIC_VERSION = '2023-06-01';
+const MESSAGES_PATH = '/v1/messages';
+/**
+ * How many values one drive will generate.
+ *
+ * The cache makes the steady state nearly free, but the FIRST drive on a form-heavy app is the one
+ * that pays, and an uncapped loop on a page with forty inputs is a bill nobody agreed to. Past the
+ * cap the heuristic answers, which is what every drive used before this existed.
+ */
+const MAX_PER_DRIVE = 12;
+/**
+ * The fallback, and still the answer for most fields.
+ *
+ * An exploration drive needs PLAUSIBLE input, not creative input: the engine is judging what the app
+ * did with the value, not the value. A field that rejects everything here is itself worth finding,
+ * and shows up as an act that did not settle.
+ */
+export function heuristicFillValue(label) {
+    const l = label.toLowerCase();
+    if (l.includes('email'))
+        return 'harness@reticle.dev';
+    if (l.includes('password'))
+        return 'password';
+    if (l.includes('url') || l.includes('link'))
+        return 'https://example.com';
+    if (l.includes('phone') || l.includes('tel'))
+        return '5550100';
+    if (l.includes('date'))
+        return '2026-01-01';
+    if (l.includes('amount') || l.includes('price') || l.includes('qty') || l.includes('quantity'))
+        return '2';
+    if (l.includes('search') || l.includes('filter'))
+        return 'a';
+    if (l.includes('number') || l.includes('count'))
+        return '42';
+    return 'reticle harness';
+}
+/**
+ * Whether a label is worth spending a model call on.
+ *
+ * The heuristic is RIGHT about an email box and an amount box, and a generated "john@example.com"
+ * is not an improvement on `harness@reticle.dev`. What it cannot do is a field whose label it does
+ * not recognise at all, where it falls back to typing the words "reticle harness" into whatever the
+ * app asked for. That fallback is the whole of what generation is for.
+ */
+export function needsGeneration(label) {
+    return heuristicFillValue(label) === heuristicFillValue('');
+}
+const PROMPT = (label, context) => `A test harness is filling in a form field on a web application so it can check what the app does with the input.\n\n` +
+    `The field is labelled: "${label}"\n` +
+    `${context}\n\n` +
+    `Reply with ONLY the value to type. No quotes, no explanation, no markdown. It must be something a ` +
+    `real user would plausibly enter, and short. If the field wants an identifier or a code, invent a ` +
+    `well-formed one.`;
+/** One string from a generating model, or nothing. Never throws: a drive must not fail over a field. */
+async function generateOne(options, label, context) {
+    const call = options.fetch ?? ((url, init) => fetch(url, init));
+    const baseUrl = options.baseUrl ?? DEFAULT_HARNESS_BASE_URL;
+    try {
+        const res = await call(`${baseUrl}${MESSAGES_PATH}`, {
+            method: 'POST',
+            headers: {
+                'content-type': 'application/json',
+                'x-api-key': options.apiKey,
+                'anthropic-version': ANTHROPIC_VERSION,
+            },
+            body: JSON.stringify({
+                model: options.model ?? 'claude-sonnet-5',
+                max_tokens: MAX_TOKENS,
+                messages: [{ role: 'user', content: PROMPT(label, context) }],
+            }),
+        });
+        if (!res.ok)
+            return undefined;
+        const parsed = JSON.parse(await res.text());
+        const content = parsed.content ?? [];
+        const text = content
+            .filter((block) => 'text' === block.type)
+            .map((block) => block.text ?? '')
+            .join('')
+            .trim()
+            .split('\n')[0];
+        if (text === undefined || 0 === text.length)
+            return undefined;
+        // A model asked for one value occasionally answers with a sentence about the value. Anything
+        // long enough to be prose is not a field value, and typing prose proves nothing about the app.
+        return 80 < text.length ? undefined : text.replace(/^["']|["']$/g, '');
+    }
+    catch {
+        // Offline, refused, rate-limited, malformed. All the same answer: use the heuristic.
+        return undefined;
+    }
+}
+/**
+ * The seam the driver calls: a label in, a value out, always.
+ *
+ * Three layers, cheapest first, and the expensive one is skipped entirely unless the two above it
+ * have nothing better than "type the words reticle harness into this box".
+ */
+export function fillValues(deps) {
+    let spent = 0;
+    return async (label) => {
+        const cached = deps.cache?.get(label);
+        if (cached !== undefined)
+            return cached;
+        const fallback = heuristicFillValue(label);
+        if (deps.generator === undefined || !needsGeneration(label) || MAX_PER_DRIVE <= spent)
+            return fallback;
+        spent += 1;
+        const generated = await generateOne(deps.generator, label, deps.context?.() ?? '');
+        if (generated === undefined)
+            return fallback;
+        // Cached under the LABEL, not the ref: refs expire with the page, labels are what the next drive
+        // and every replay will meet again.
+        deps.cache?.set(label, generated);
+        return generated;
+    };
+}