explorbot 0.2.3 → 0.2.5

package/dist/boat/prima/src/cli.js

@@ -2,81 +2,73 @@ import { Command } from 'commander';
 import dedent from 'dedent';
 import { keepServerRunning } from "../../../src/browser-server.js";
 import { browserErrorMessage } from "../../../src/utils/browser-errors.js";
-import { setPreserveConsoleLogs } from "../../../src/utils/logger.js";
+import { isVerboseMode, setQuietMode } from "../../../src/utils/logger.js";
+import { clearActivityLine, trackActivityLine } from "./activity-line.js";
 import { renderEnvelope } from "./envelope.js";
 import { Prima } from "./prima.js";
 const helpContract = dedent `
-  Prima drives a browser that is already open. One command per process; every command
-  prints a plain-text envelope on stdout and exits 0 when ok, 1 when not.
+  Prima is a high-level AI extension to playwright-cli, driving the browser it has open.
 
-  TIERS - choose by what you hold, not by how hard the step looks
-    pw <fn>        Precise. A Playwright function expression built from a locator you
-                   already verified. No AI on the happy path.
-                     prima pw "({ page }) => page.click('[data-test=submit]')"
-    click / fill   One action described in words; AI resolves it on the current page.
-                     prima click "the primary action button in the header"
-                     prima fill "the search box" "a search term"
-    do <steps...>  Several described steps, run tester-style in one process.
-                     prima do "open the account menu" "choose the settings entry"
-    Never pass a locator or a function expression to click/fill/do - describe the target.
-    Never pass a description to pw - it takes executable code only.
+    playwright-cli open <url>    starts the session
+    prima <command> ...          drives it
+    playwright-cli close         ends it
 
-  LOOP
-    prima go <url|path|words>  reach the page you want to work on
-    prima research             once per new page; returns verified locators
-    prima pw "..."             drive the page with those locators
-    prima verify "..."         assert the outcome (prima ask "..." to inspect instead)
-    Fall back to click/fill/do whenever research left you no locator to hold.
+  One call takes a whole job:
 
-  ENVELOPE
-    ### Result     ok, command, healed, used
-    ### Page       url, title, state hash, visit count
-    ### Changes    what the accessibility tree gained or lost
-    ### Answer | ### Research | ### Verdict   output of ask, research, verify
-    ### Failure    error, reasoning, healing attempts, compact ARIA of the page
-    ### Instance   the browser you are on and the other instances running
-    ### Artifacts  paths to the full aria.yml, page.html and network.jsonl
-    used: is code that already executed - CodeceptJS steps to copy as they are, except
-          for pw, whose Playwright expression a test needs inside I.usePlaywrightTo(...).
-    Log lines can precede the envelope; start parsing at the first ### line.
-
-  HEALING AND FAILURE
-    A failed action is retried by AI along a different route; healed: true means the
-    outcome was reached another way and used: holds the code that worked.
-    --no-heal skips that and fails fast.
-    Failures print compact ARIA inline, so retarget from the envelope itself and open
-    the artifact files only when the inline snapshot is not enough.
-
-  SESSIONS
-    By default prima attaches to the playwright-cli browser of this workspace and works
-    on the tabs it already has open; driving the same session from both tools is the
-    intended usage.
-    playwright-cli open <url>  the session prima attaches to
-    --pw-session <title>       which playwright-cli session, when several are open
-    --endpoint <ep>            attach to a browser server endpoint directly
-    prima browser start        a prima-owned browser instead, when no session is open
-    --instance <name>          which prima-owned browser you talk to; parallel work
-                               needs one each
-    --session [file]           cookies and storage persisted across processes; ignored
-                               while attached, the attached session keeps its own
-    Prima never launches a browser implicitly and never closes an attached one - it
-    disconnects. browser list shows both kinds; ### Instance names the one you are on.
-    Every browser is reached over a Playwright browser-server endpoint, which needs the
-    Node build - run prima as "npx explorbot prima ..." or through the published prima
-    bin; from source under Bun the connection does not open.
-    When no AI model is usable pw still works; for everything else drive
-    playwright-cli directly.
-    Parsed but not active yet: --framework, so reported code is CodeceptJS whatever
-    you pass.
+    prima check "a workflow can be created and appears in the list" --expected "the new workflow is listed"
+    prima do "open the account menu" "choose the settings entry" "switch the theme to dark" "check it took effect"
+    prima pw "({ page }) => page.click('[data-test=submit]')"
+`;
+const checkHelp = dedent `
+  check takes an outcome rather than a click path, and works out how to reach it. It runs
+  on the page you are already on and never reloads it, so an open dialog survives the check.
+  --expected  one outcome the run must reach, repeatable for several. Without it the
+              scenario text is the single expected outcome. Each comes back under
+              ### Expected outcomes as PASSED, FAILED, CONTRADICTION or not verified.
+              "not verified" means the run never checked it, which is not the same
+              as false.
+  Outcomes are settled against a screenshot of the whole page: what a user can see is
+  the proof, and the run log only says what was done. CONTRADICTION means the two
+  disagree - reported with both sides rather than settled one way, and ### Artifacts
+  then names the html, aria and screenshot on disk so you can judge it yourself. Not
+  finding something in the picture is not enough on its own; that is "not verified".
+  ok: follows those outcomes - false when one FAILED or CONTRADICTED, or when the run
+  could not complete, which is reported as such rather than as an app failure.
+  Page problems seen on the way appear under ### Answer, not as step failures.
+`;
+const doHelp = dedent `
+  Each instruction is numbered and accounted for: ### Steps reports each as ok, FAIL or ??.
+  ?? means the action ran but the run ended without confirming that instruction - read the
+  steps above it. Only FAIL and an instruction the page could not carry out fail the command.
+  Nothing runs past the last instruction given. A whole remaining sequence in one call is
+  what makes this tier cheap.
 `;
-function buildOptions(options) {
+const verifyHelp = dedent `
+  Reports each assertion it could express as PASSED or FAILED with its playwright form,
+  and gives no overall verdict - read the lines and decide. "none ran" means the claim
+  could not be expressed, which is not the same as false.
+`;
+const reportHelp = dedent `
+  Commands are logged as they run, so the report needs no browser and outlives the session.
+  The most recent session is reported unless --pw-session names another.
+`;
+const sessionHelp = dedent `
+  --endpoint <ep>    attach to a browser server endpoint directly, skipping discovery
+  --instance <name>  which prima-owned browser you talk to; parallel work needs one each
+  --session [file]   cookies and storage persisted across processes; ignored while
+                     attached, since the attached session keeps its own
+  --framework        parsed but not active yet; reported code is CodeceptJS either way
+  DEBUG='explorbot:*' in front of a command prints the log of everything it does.
+  When no AI model is usable pw still works; for everything else drive playwright-cli.
+`;
+let rootOptions = () => ({});
+function buildOptions(subcommand) {
+    const options = { ...rootOptions(), ...stripEmpty(subcommand) };
     return {
-        verbose: options.verbose || options.debug,
         config: options.config,
         path: options.path,
         instance: options.instance,
         session: options.session,
-        heal: options.heal,
         ephemeral: options.ephemeral,
         framework: options.framework,
         noVision: options.vision === false,
@@ -88,29 +80,38 @@ function buildOptions(options) {
         pwSession: options.pwSession,
     };
 }
+function stripEmpty(options) {
+    const present = {};
+    for (const [key, value] of Object.entries(options || {})) {
+        if (value === undefined)
+            continue;
+        present[key] = value;
+    }
+    return present;
+}
 function addCommonOptions(cmd) {
     return cmd
-        .option('-v, --verbose', 'Enable verbose logging')
-        .option('--debug', 'Enable debug logging (same as --verbose)')
         .option('-c, --config <path>', 'Path to explorbot configuration file')
         .option('-p, --path <path>', 'Working directory path')
         .option('-i, --instance <name>', 'Browser instance to drive')
         .option('--session [file]', 'Persist cookies and storage to a session file')
-        .option('--no-heal', 'Fail immediately instead of letting AI retry a failed action')
         .option('--ephemeral', 'Keep no state between runs; applies to config-free runs, where output goes to a temp directory')
         .option('--framework <name>', 'Not active yet: framework the reported code targets, codeceptjs or playwright')
         .option('--url <url>', 'Page to open when the session has no page yet')
         .option('--endpoint <ep>', 'Websocket endpoint of a browser server to attach to, skipping discovery')
-        .option('--pw-session <title>', 'Title of the playwright-cli session to attach to');
+        .option('--pw-session <title>', 'Title of the playwright-cli session to attach to')
+        .addHelpText('after', `\n${sessionHelp}`);
 }
 function primaFor(options) {
-    setPreserveConsoleLogs(true);
     if (options.ephemeral)
         process.env.EXPLORBOT_EPHEMERAL = '1';
     return new Prima(buildOptions(options));
 }
-async function runPrima(options, command, run) {
+async function runPrima(options, command, run, record = true) {
+    setQuietMode(!isVerboseMode());
+    trackActivityLine();
     const prima = primaFor(options);
+    const startedAt = Date.now();
     let envelope;
     try {
         await prima.start();
@@ -119,6 +120,9 @@ async function runPrima(options, command, run) {
     catch (error) {
         envelope = await prima.toolFailureEnvelope(command, error);
     }
+    if (record)
+        prima.record(envelope, Date.now() - startedAt);
+    clearActivityLine();
     console.log(renderEnvelope(envelope));
     await prima.stop().catch(() => { });
     process.exit(envelope.ok ? 0 : 1);
@@ -136,24 +140,31 @@ async function runBrowser(options, run) {
 }
 export function createPrimaCommands(name = 'prima') {
     const cmd = new Command(name);
-    cmd.description('Drive an already-open browser one command at a time and report back in a plain-text envelope');
+    cmd.description('Tests and drives a web app through described behaviour instead of locators: one command carries a whole scenario, verifies it, and reports the proof');
+    cmd.option('--pw-session <title>', 'Title of the playwright-cli session to attach to');
+    cmd.option('--url <url>', 'Page to open when the session has no page yet');
     cmd.addHelpText('after', `\n${helpContract}`);
+    rootOptions = () => cmd.opts();
     addCommonOptions(cmd.command('pw <fn>').description('Run a Playwright function expression against the open page')).action(async (fn, options) => {
         await runPrima(options, `pw ${fn}`, (prima) => prima.pw(fn));
     });
-    addCommonOptions(cmd.command('do <instructions...>').description('Run high-level instructions tester-style, one argument per instruction')).action(async (instructions, options) => {
+    addCommonOptions(cmd.command('do <instructions...>').description('Run high-level instructions tester-style, one argument per instruction'))
+        .addHelpText('after', `\n${doHelp}`)
+        .action(async (instructions, options) => {
         await runPrima(options, `do ${instructions.join(' ')}`, (prima) => prima.do(instructions));
     });
-    addCommonOptions(cmd.command('click <target>').description('Click an element described in plain words')).action(async (target, options) => {
-        await runPrima(options, `click ${target}`, (prima) => prima.click(target));
-    });
-    addCommonOptions(cmd.command('fill <field> <value>').description('Fill a field described in plain words')).action(async (field, value, options) => {
-        await runPrima(options, `fill ${field} ${value}`, (prima) => prima.fill(field, value));
+    addCommonOptions(cmd.command('check <scenario>').description('Run a scenario end to end as a test, with its own verification, and report the steps it took'))
+        .option('--expected <outcome>', 'An outcome the run must reach; repeat the flag for several', (value, all) => [...all, value], [])
+        .addHelpText('after', `\n${checkHelp}`)
+        .action(async (scenario, options) => {
+        await runPrima(options, `check ${scenario}`, (prima) => prima.check(scenario, options.expected));
     });
     addCommonOptions(cmd.command('ask <question>').description('Answer a question about the current page').option('--no-vision', 'Answer from page structure only, without a screenshot')).action(async (question, options) => {
         await runPrima(options, `ask ${question}`, (prima) => prima.ask(question));
     });
-    addCommonOptions(cmd.command('verify <assertion>').alias('assert').description('Assert a statement about the current page')).action(async (assertion, options) => {
+    addCommonOptions(cmd.command('verify <assertion>').alias('assert').description('Assert a statement about the current page'))
+        .addHelpText('after', `\n${verifyHelp}`)
+        .action(async (assertion, options) => {
         await runPrima(options, `verify ${assertion}`, (prima) => prima.verify(assertion));
     });
     addCommonOptions(cmd.command('research').description('Map the current page and return verified locators').option('--data', 'Include data extraction in the map').option('--deep', 'Expand hidden elements for a deeper map').option('--fresh', 'Ignore the cached map and research the page again')).action(async (options) => {
@@ -164,6 +175,27 @@ export function createPrimaCommands(name = 'prima') {
             options.baseUrl = target;
         await runPrima(options, `go ${target}`, (prima) => prima.go(target));
     });
+    addCommonOptions(cmd.command('config').description('Show models, config file and paths used by this run'))
+        .option('--json', 'Print the resolved config as JSON')
+        .action(async (options) => {
+        setQuietMode(!isVerboseMode());
+        const prima = primaFor(options);
+        console.log(await prima.config(options.json).catch((error) => browserErrorMessage(error)));
+        await prima.stop().catch(() => { });
+        process.exit(0);
+    });
+    addCommonOptions(cmd.command('status <hash>').description('Show the artifacts and page detail recorded for an earlier command')).action(async (hash, options) => {
+        await runPrima(options, `status ${hash}`, (prima) => prima.status(hash), false);
+    });
+    addCommonOptions(cmd.command('report').description('Turn every command of a session into one html and markdown report'))
+        .addHelpText('after', `\n${reportHelp}`)
+        .action(async (options) => {
+        setQuietMode(!isVerboseMode());
+        console.log(await primaFor(options)
+            .report()
+            .catch((error) => browserErrorMessage(error)));
+        process.exit(0);
+    });
     const browser = cmd.command('browser').description('Manage the browsers prima drives');
     addCommonOptions(browser.command('start').description('Start a prima-owned browser and hold it open until Ctrl+C'))
         .option('-s, --show', 'Launch the browser in a visible window')

package/dist/boat/prima/src/envelope.js

@@ -1,7 +1,13 @@
 import { mkdirSync, writeFileSync } from 'node:fs';
 import path from 'node:path';
+const EXPECTATION_LABELS = {
+    passed: 'PASSED       ',
+    failed: 'FAILED       ',
+    unverified: 'not verified ',
+    contradiction: 'CONTRADICTION',
+};
 export function renderEnvelope(data) {
-    const sections = [renderResult(data), renderPage(data), renderOutcome(data), ...renderFailure(data), renderInstance(data.instance), renderArtifacts(data)];
+    const sections = [renderResult(data), renderPage(data), renderValue(data), renderChanges(data), renderSteps(data), renderExpectations(data), renderWarning(data), renderOutcome(data), ...renderFailure(data), renderInstance(data), renderArtifacts(data)];
     return sections.filter((section) => section).join('\n\n');
 }
 export function writeArtifacts(dir, snapshot) {
@@ -9,29 +15,25 @@ export function writeArtifacts(dir, snapshot) {
     const paths = {
         aria: path.resolve(dir, 'aria.yml'),
         html: path.resolve(dir, 'page.html'),
-        network: path.resolve(dir, 'network.jsonl'),
     };
     writeFileSync(paths.aria, snapshot.aria ?? '', 'utf-8');
     writeFileSync(paths.html, snapshot.html ?? '', 'utf-8');
+    if (snapshot.screenshot) {
+        paths.screenshot = path.resolve(dir, 'page.png');
+        writeFileSync(paths.screenshot, snapshot.screenshot);
+    }
+    if (!snapshot.requests.length)
+        return paths;
+    paths.network = path.resolve(dir, 'network.jsonl');
     writeFileSync(paths.network, snapshot.requests.map((request) => `${JSON.stringify(request)}\n`).join(''), 'utf-8');
     return paths;
 }
 function renderResult(data) {
     const lines = [`ok: ${data.ok}`, `command: ${data.command}`];
-    const healed = renderHealed(data);
-    if (healed)
-        lines.push(healed);
     if (data.used?.length)
         lines.push(`used: ${data.used.join('; ')}`);
     return section('Result', lines.join('\n'));
 }
-function renderHealed(data) {
-    if (data.healed === undefined)
-        return null;
-    if (data.healNote)
-        return `healed: ${data.healed} (${data.healNote})`;
-    return `healed: ${data.healed}`;
-}
 function renderPage(data) {
     const { url, previousUrl, title, state, visits } = data.page;
     const urlLabel = `url: ${url}`;
@@ -43,67 +45,115 @@ function renderPage(data) {
     const lines = [align(urlLabel, changedMarker, width), `title: ${title}`, align(stateLabel, `(visit #${visits})`, width)];
     return section('Page', lines.join('\n'));
 }
+function renderValue(data) {
+    if (data.value === undefined)
+        return null;
+    return section('Value', data.value);
+}
+function renderChanges(data) {
+    if (data.changes === undefined || data.changes === null)
+        return null;
+    return section('Changes', data.changes);
+}
+function renderSteps(data) {
+    if (!data.steps?.length)
+        return null;
+    const lines = [];
+    data.steps.forEach((step, index) => {
+        let mark = 'FAIL';
+        if (step.ok)
+            mark = 'ok  ';
+        if (step.unconfirmed)
+            mark = '??  ';
+        lines.push(`${index + 1}. ${mark} ${step.label}`);
+        for (const line of (step.proof || '').split('\n').filter(Boolean))
+            lines.push(`      ${line}`);
+    });
+    if (data.stepFiles)
+        lines.push('', `page after each step: ${data.stepFiles}`);
+    return section('Steps', lines.join('\n'));
+}
+function renderExpectations(data) {
+    if (!data.expectations?.length)
+        return null;
+    const lines = [];
+    data.expectations.forEach((expectation, index) => {
+        lines.push(`${index + 1}. ${EXPECTATION_LABELS[expectation.status]} ${expectation.text}`);
+        if (expectation.status !== 'contradiction' && expectation.status !== 'failed')
+            return;
+        for (const line of (expectation.evidence || '').split('\n').filter(Boolean))
+            lines.push(`      ${line}`);
+    });
+    return section('Expected outcomes', lines.join('\n'));
+}
+function renderWarning(data) {
+    if (!data.warning)
+        return null;
+    return section('Warning', data.warning);
+}
 function renderOutcome(data) {
-    if (data.changes)
-        return section('Changes', data.changes);
     if (data.answer)
         return section('Answer', data.answer);
     if (data.research)
         return section('Research', data.research);
-    if (!data.verdict)
+    if (!data.assertions)
         return null;
-    const lines = [`passed: ${data.verdict.passed}`, `evidence: ${data.verdict.evidence}`, `code: ${data.verdict.code}`];
-    return section('Verdict', lines.join('\n'));
+    if (!data.assertions.length)
+        return section('Assertions', 'none ran — no assertion could express this claim, so nothing was checked against the page');
+    const lines = data.assertions.map((assertion) => {
+        const code = assertion.code
+            .split('\n')
+            .map((line) => line.trim())
+            .filter((line) => line && !line.startsWith('//'))
+            .join(' ');
+        return `${code}  => ${assertion.passed ? 'PASSED' : 'FAILED'}`;
+    });
+    const proof = data.assertions.flatMap((assertion) => assertion.proof);
+    if (proof.length)
+        lines.push('', 'playwright:', ...proof);
+    return section('Assertions', lines.join('\n'));
 }
 function renderFailure(data) {
     if (!data.failure)
         return [];
-    const lines = [`error: ${data.failure.error}`];
-    if (data.failure.reasoning)
-        lines.push(`reasoning: ${data.failure.reasoning}`);
-    return [section('Failure', lines.join('\n')), renderAttempts(data.failure.attempts), renderCompactAria(data.failure.compactAria)];
-}
-function renderAttempts(attempts) {
-    if (!attempts?.length)
-        return null;
-    const labels = attempts.map((attempt, index) => `${index + 1}. ${attempt.code}`);
-    const width = Math.max(...labels.map((label) => label.length)) + 3;
-    const lines = labels.map((label, index) => align(label, `→ ${attempts[index].outcome}`, width));
-    return section(`Healing attempts (${attempts.length})`, lines.join('\n'));
+    return [section('Failure', `error: ${data.failure.error}`), renderCompactAria(data.failure.compactAria)];
 }
 function renderCompactAria(compactAria) {
     if (!compactAria)
         return null;
     return section('Current page (compact ARIA)', compactAria);
 }
-function renderInstance(instance) {
-    const others = instance.others.map((other) => `${other.name} (${tabsLabel(other.tabs)})`);
-    const lines = [`instance: ${instance.name} (${tabsLabel(instance.tabs)}) | other instances: ${otherInstances(others)}`, browserLine(instance)];
-    return section('Instance', lines.join('\n'));
-}
-function otherInstances(others) {
-    if (!others.length)
-        return 'none';
-    return others.join(', ');
+function renderInstance(data) {
+    const instance = data.instance;
+    const parts = [`${instance.name} (${tabsLabel(instance.tabs)})`, browserLine(instance)];
+    if (instance.others.length)
+        parts.push(`other instances: ${instance.others.map((other) => `${other.name} (${tabsLabel(other.tabs)})`).join(', ')}`);
+    if (data.status)
+        parts.push(`details: prima status ${data.status}`);
+    return section('Instance', parts.join(' | '));
 }
 function browserLine(instance) {
     if (instance.attached)
-        return `browser: attached (${instance.attached})`;
+        return `attached to ${instance.attached}`;
     if (instance.startedAgo)
-        return `browser: running, started ${instance.startedAgo} ago`;
+        return `running, started ${instance.startedAgo} ago`;
     if (instance.tabs > 0)
-        return 'browser: running';
-    return 'browser: not running';
+        return 'running';
+    return 'not running';
 }
 function tabsLabel(tabs) {
     if (tabs === 1)
         return '1 tab';
     return `${tabs} tabs`;
 }
-function renderArtifacts(data) {
+export function renderArtifacts(data) {
     if (!data.artifacts)
         return null;
-    const lines = [`aria: ${data.artifacts.aria}`, `html: ${data.artifacts.html}`, `network: ${data.artifacts.network}`];
+    const lines = [`aria:       ${data.artifacts.aria}`, `html:       ${data.artifacts.html}`];
+    if (data.artifacts.screenshot)
+        lines.push(`screenshot: ${data.artifacts.screenshot}`);
+    if (data.artifacts.network)
+        lines.push(`network:    ${data.artifacts.network}`);
     return section('Artifacts', lines.join('\n'));
 }
 function align(label, marker, width) {