explorbot 0.2.1 → 0.2.3

package/dist/boat/prima/src/prima.js

@@ -0,0 +1,635 @@
+import { createRequire } from 'node:module';
+import path from 'node:path';
+import dedent from 'dedent';
+import * as playwright from 'playwright';
+import { ActionResult } from "../../../src/action-result.js";
+import { actionRule, locatorRule } from "../../../src/ai/rules.js";
+import { createCodeceptJSTools } from "../../../src/ai/tools.js";
+import { getAliveEndpoint, launchServer, listInstances, stopServer } from "../../../src/browser-server.js";
+import { ConfigMissingError, ConfigParser, outputPath } from "../../../src/config.js";
+import { ExplorBot } from "../../../src/explorbot.js";
+import { Task } from "../../../src/test-plan.js";
+import { compactAriaSnapshot } from "../../../src/utils/aria.js";
+import { browserErrorMessage } from "../../../src/utils/browser-errors.js";
+import { pluralize } from "../../../src/utils/logger.js";
+import { writeArtifacts } from "./envelope.js";
+import { isFunctionExpression, toCodeceptWrapper } from "./pw-parser.js";
+import { readDescriptors, selectDescriptor } from "./pw-registry.js";
+const MAX_INSTRUCTION_ITERATIONS = 6;
+const MAX_TOOL_ROUNDTRIPS = 5;
+const AI_AGENT_NAME = 'prima';
+const CONNECT_TIMEOUT = 3000;
+const requireLib = createRequire(import.meta.url);
+export class Prima {
+    options;
+    bot;
+    artifactsDir;
+    server = null;
+    attached = null;
+    constructor(options = {}) {
+        this.options = options;
+        this.bot = new ExplorBot({
+            config: options.config,
+            path: options.path,
+            baseUrl: this.configBaseUrl(),
+            verbose: options.verbose,
+            session: options.session,
+            instance: options.instance,
+            headless: true,
+            optionalAi: true,
+        });
+    }
+    async start() {
+        const config = await this.loadConfig();
+        await this.resolveBrowser(config);
+        await this.bot.start();
+        if (!this.options.url)
+            return;
+        if (this.bot.getCurrentState())
+            return;
+        await this.bot.visit(this.options.url);
+    }
+    async stop() {
+        await this.bot.stop();
+    }
+    async pw(expression) {
+        const command = `pw ${expression}`;
+        const validation = isFunctionExpression(expression);
+        if (!validation.valid)
+            return this.toolFailureEnvelope(command, validation.error);
+        const previousState = this.bot.stateManager().getCurrentState();
+        let result = null;
+        let executionError = null;
+        try {
+            const executed = await this.bot.getExplorer().action().execute(toCodeceptWrapper(expression), { verbatim: true });
+            result = executed.actionResult;
+        }
+        catch (error) {
+            executionError = error;
+        }
+        if (executionError)
+            return this.heal(command, expression, executionError, previousState);
+        result ||= await this.capturedResult(previousState);
+        return this.successEnvelope(command, [expression], result, previousState);
+    }
+    async do(instructions) {
+        const command = `do ${instructions.map((instruction) => `"${instruction}"`).join(' ')}`;
+        const guard = await this.aiGuard(command);
+        if (guard)
+            return guard;
+        const provider = this.bot.getProvider();
+        const previousState = this.bot.stateManager().getCurrentState();
+        const conversation = provider.startConversation(this.instructionSystemPrompt(), AI_AGENT_NAME);
+        const task = new Task(instructions.join('; '), previousState?.url || '');
+        const tools = createCodeceptJSTools({ explorer: this.bot.getExplorer(), stateManager: this.bot.stateManager(), ai: provider }, task);
+        conversation.addUserText(this.instructionPrompt(instructions, await this.capturedResult(previousState)));
+        const used = [];
+        let failure = null;
+        let aiError = null;
+        let narration = '';
+        let contextHash = this.bot.stateManager().getCurrentState()?.hash;
+        for (let iteration = 1; iteration <= Math.min(instructions.length + 2, MAX_INSTRUCTION_ITERATIONS); iteration++) {
+            const state = this.bot.stateManager().getCurrentState();
+            if (iteration > 1 && state && state.hash !== contextHash) {
+                contextHash = state.hash;
+                conversation.addUserText(this.pageContext(ActionResult.fromState(state)));
+            }
+            const invoked = await provider.invokeConversation(conversation, tools, { maxToolRoundtrips: MAX_TOOL_ROUNDTRIPS, agentName: AI_AGENT_NAME }).catch((error) => {
+                aiError = error;
+                return null;
+            });
+            if (!invoked)
+                break;
+            const executions = invoked.toolExecutions || [];
+            if (!executions.length) {
+                narration = invoked.response?.text?.trim() || '';
+                break;
+            }
+            for (const execution of executions) {
+                if (!execution.wasSuccessful) {
+                    failure = { code: execution.output?.code || '', message: execution.output?.message || 'action failed' };
+                    continue;
+                }
+                used.push(...this.executedCodes(execution.output?.code));
+                failure = null;
+            }
+        }
+        if (aiError)
+            return this.failureEnvelope(command, aiError, previousState);
+        if (failure) {
+            const envelope = await this.heal(command, failure.code || instructions.join('; '), failure.message, previousState);
+            envelope.used = [...used, ...(envelope.used || [])];
+            return envelope;
+        }
+        if (!used.length) {
+            const reason = ['No action was performed for these instructions on the current page.', narration].filter(Boolean).join(' ');
+            return this.failureEnvelope(command, reason, previousState);
+        }
+        const result = await this.capturedResult(this.bot.stateManager().getCurrentState());
+        return this.successEnvelope(command, used, result, previousState);
+    }
+    async click(target) {
+        return this.do([`click ${target}`]);
+    }
+    async fill(field, value) {
+        return this.do([`fill ${field} with value: ${value}`]);
+    }
+    async ask(question) {
+        const command = `ask ${question}`;
+        const guard = await this.aiGuard(command);
+        if (guard)
+            return guard;
+        const previousState = this.bot.stateManager().getCurrentState();
+        const result = await this.capturedResult(previousState, { screenshot: this.visionEnabled() });
+        return this.reportEnvelope(command, result, previousState, { answer: await this.answer(question, result) });
+    }
+    async verify(assertion) {
+        const command = `verify ${assertion}`;
+        const guard = await this.aiGuard(command);
+        if (guard)
+            return guard;
+        const previousState = this.bot.stateManager().getCurrentState();
+        const result = await this.capturedResult(previousState);
+        const verification = await this.bot.agentNavigator().verifyState(assertion, result);
+        const codes = verification.successfulCodes || [];
+        const verdict = { passed: verification.verified, evidence: this.verdictEvidence(verification.verified, codes), code: codes.join('\n') };
+        return this.reportEnvelope(command, result, previousState, { ok: verification.verified, verdict });
+    }
+    async research(opts = {}) {
+        const flags = [opts.data && '--data', opts.deep && '--deep', opts.fresh && '--fresh'].filter(Boolean);
+        const command = ['research', ...flags].join(' ');
+        const guard = await this.aiGuard(command);
+        if (guard)
+            return guard;
+        const previousState = this.bot.stateManager().getCurrentState();
+        const result = await this.capturedResult(previousState);
+        const uiMap = await this.bot.agentResearcher().research(result, { screenshot: true, data: opts.data, deep: opts.deep, force: opts.fresh });
+        return this.reportEnvelope(command, result, previousState, { research: uiMap });
+    }
+    async go(target) {
+        const command = `go ${target}`;
+        const code = `I.amOnPage('${target}')`;
+        const isUrl = this.isUrlTarget(target);
+        if (!isUrl) {
+            const guard = await this.aiGuard(command);
+            if (guard)
+                return guard;
+        }
+        const previousState = this.bot.stateManager().getCurrentState();
+        let navigationError = null;
+        try {
+            await this.bot.agentNavigator().visit(target);
+        }
+        catch (error) {
+            navigationError = error;
+        }
+        if (navigationError)
+            return this.heal(command, code, navigationError, previousState);
+        const used = [];
+        if (isUrl)
+            used.push(code);
+        const result = await this.capturedResult(this.bot.stateManager().getCurrentState());
+        return this.successEnvelope(command, used, result, previousState);
+    }
+    async browserStart() {
+        const config = await this.loadConfig();
+        let show = config.playwright.show || false;
+        if (this.options.show)
+            show = true;
+        if (this.options.headless)
+            show = false;
+        this.server = await this.launchOwnServer({ browser: config.playwright.browser, show }, this.instanceName());
+    }
+    async browserStop(all = false) {
+        await this.loadConfig();
+        if (this.attached)
+            return false;
+        if (!all)
+            return this.stopInstance(this.instanceName());
+        let stopped = false;
+        for (const instance of listInstances()) {
+            if (await this.stopInstance(instance.name))
+                stopped = true;
+        }
+        return stopped;
+    }
+    async browserStatus() {
+        await this.loadConfig();
+        const info = await this.instanceInfo();
+        const endpoint = await getAliveEndpoint(info.name);
+        const others = info.others.map((other) => other.name).join(', ') || 'none';
+        const lines = [`instance: ${info.name} (${info.tabs} ${pluralize(info.tabs, 'tab')}) | other instances: ${others}`];
+        if (!endpoint)
+            lines.push('browser: not running');
+        if (endpoint)
+            lines.push(`browser: running at ${endpoint}`);
+        return lines.join('\n');
+    }
+    async browserList() {
+        await this.loadConfig();
+        const lines = [];
+        for (const instance of listInstances()) {
+            const endpoint = await getAliveEndpoint(instance.name);
+            if (!endpoint)
+                continue;
+            lines.push(`prima --instance ${instance.name}  ${endpoint}`);
+        }
+        const discovery = await this.discover();
+        await discovery.browser?.close().catch(() => { });
+        for (const descriptor of discovery.candidates) {
+            lines.push(`playwright-cli --pw-session ${descriptor.title}  ${descriptor.endpoint}`);
+        }
+        if (!lines.length)
+            return 'no browser instances running';
+        return lines.join('\n');
+    }
+    async instanceInfo() {
+        const name = this.instanceName();
+        const others = listInstances()
+            .filter((instance) => instance.name !== name)
+            .map((instance) => ({ name: instance.name, tabs: 0 }));
+        const info = { name, tabs: this.tabCount(), others };
+        if (this.attached)
+            info.attached = this.attached;
+        return info;
+    }
+    async toolFailureEnvelope(command, error) {
+        const state = this.bot.getCurrentState();
+        const instance = await this.instanceInfo().catch(() => ({ name: this.instanceName(), tabs: 0, others: [] }));
+        const failure = { error: `tool: ${browserErrorMessage(error)}`, attempts: [] };
+        if (error instanceof ConfigMissingError)
+            failure.error = browserErrorMessage(error);
+        if (state?.ariaSnapshot)
+            failure.compactAria = compactAriaSnapshot(state.ariaSnapshot, true);
+        return {
+            ok: false,
+            command,
+            page: { url: state?.url || '', title: state?.title || '', state: state?.hash || '', visits: 0 },
+            failure,
+            instance,
+        };
+    }
+    async loadConfig() {
+        return ConfigParser.getInstance().loadConfig({ config: this.options.config, path: this.options.path, baseUrl: this.configBaseUrl() });
+    }
+    configBaseUrl() {
+        const url = this.options.baseUrl || this.options.url;
+        if (!url)
+            return undefined;
+        if (!URL.canParse(url))
+            return undefined;
+        return url;
+    }
+    async resolveBrowser(config) {
+        if (this.options.endpoint) {
+            const endpoint = this.options.endpoint;
+            const browserName = config.playwright.browser || 'chromium';
+            if (await this.attachToEndpoint({ file: '', title: '', endpoint, workspaceDir: '', browserName, playwrightLib: '' }))
+                return;
+            throw new Error(dedent `
+        No browser answered at ${endpoint}.
+        Check the endpoint of the running session, or drop --endpoint to attach to the
+        playwright-cli browser of this workspace.
+      `);
+        }
+        const { match, candidates, browser } = await this.discover();
+        if (match && (await this.attachToEndpoint(match, browser)))
+            return;
+        if (!match && candidates.length) {
+            const titles = candidates.map((candidate) => candidate.title).join(', ');
+            throw new Error(dedent `
+        Several playwright-cli sessions are open for this workspace: ${titles}
+        Pick one with --pw-session <title>.
+      `);
+        }
+        if (await this.connectOwnInstance())
+            return;
+        throw new Error(dedent `
+      No browser to drive for instance "${this.instanceName()}".
+      Open one first:
+        playwright-cli open <url>   prima attaches to this workspace session by default
+        prima browser start         starts a prima-owned browser instead
+      Prima never launches a browser implicitly.
+    `);
+    }
+    async discover(descriptors = readDescriptors()) {
+        const title = this.options.pwSession ?? process.env.PLAYWRIGHT_CLI_SESSION;
+        const opts = { workspaceDir: this.workspaceDir(), title };
+        const alive = new Map();
+        for (const candidate of selectDescriptor(descriptors, opts).candidates) {
+            const browser = await this.connectDescriptor(candidate);
+            if (browser)
+                alive.set(candidate, browser);
+        }
+        const selected = selectDescriptor([...alive.keys()], opts);
+        const discovery = { match: selected.match, candidates: selected.candidates };
+        if (selected.match)
+            discovery.browser = alive.get(selected.match);
+        for (const [descriptor, browser] of alive) {
+            if (descriptor === selected.match)
+                continue;
+            await browser.close().catch(() => { });
+        }
+        return discovery;
+    }
+    async attachToEndpoint(descriptor, probed) {
+        const browser = probed || (await this.connectDescriptor(descriptor));
+        if (!browser)
+            return false;
+        this.bot.attachBrowser(browser);
+        this.attached = this.attachmentLabel(descriptor);
+        return true;
+    }
+    async connectDescriptor(descriptor) {
+        const connected = await this.connectWith(playwright, descriptor);
+        if (connected)
+            return connected;
+        return this.connectWith(this.descriptorLib(descriptor), descriptor);
+    }
+    async connectWith(lib, descriptor) {
+        const launcher = lib?.[descriptor.browserName];
+        if (!launcher?.connect)
+            return null;
+        return launcher.connect(descriptor.endpoint, { timeout: CONNECT_TIMEOUT }).catch(() => null);
+    }
+    descriptorLib(descriptor) {
+        if (!descriptor.playwrightLib)
+            return null;
+        try {
+            return requireLib(descriptor.playwrightLib);
+        }
+        catch {
+            return null;
+        }
+    }
+    attachmentLabel(descriptor) {
+        if (!descriptor.title)
+            return `endpoint ${descriptor.endpoint}`;
+        return `playwright-cli session "${descriptor.title}", workspace ${descriptor.workspaceDir}`;
+    }
+    workspaceDir() {
+        return path.resolve(this.options.path || process.cwd());
+    }
+    async connectOwnInstance() {
+        return !!(await getAliveEndpoint(this.instanceName()));
+    }
+    async launchOwnServer(opts, instance) {
+        return launchServer(opts, instance);
+    }
+    async stopInstance(instance) {
+        if (instance === this.instanceName()) {
+            const server = this.server;
+            this.server = null;
+            await server?.close();
+        }
+        return stopServer(instance);
+    }
+    isUrlTarget(target) {
+        const value = target.trim();
+        if (value.startsWith('/'))
+            return true;
+        return URL.canParse(value);
+    }
+    async heal(command, expression, error, previousState) {
+        if (this.options.heal === false)
+            return this.failureEnvelope(command, error, previousState);
+        const navigator = this.healNavigator();
+        if (!navigator) {
+            const envelope = await this.failureEnvelope(command, error, previousState);
+            envelope.healed = false;
+            envelope.healNote = this.aiUnavailableNote();
+            return envelope;
+        }
+        const message = dedent `
+      I tried to run this command on the page: ${expression}
+      But it failed with: ${browserErrorMessage(error)}
+      Reach the same outcome on the current page in a different way.
+    `;
+        const attempts = [];
+        const failedResult = await this.capturedResult(previousState);
+        const resolved = await navigator.resolveState(message, failedResult, { onAttempt: (attempt) => attempts.push(attempt) }).catch(() => false);
+        if (!resolved) {
+            const healAttempts = attempts.map((attempt) => ({ code: attempt.code, outcome: attempt.error || 'ok' }));
+            return this.failureEnvelope(command, error, previousState, healAttempts);
+        }
+        const used = attempts.filter((attempt) => !attempt.error).map((attempt) => attempt.code);
+        const result = await this.capturedResult(this.bot.stateManager().getCurrentState());
+        const envelope = await this.successEnvelope(command, used, result, previousState);
+        envelope.healed = true;
+        envelope.healNote = `recovered after ${attempts.length} ${pluralize(attempts.length, 'attempt')}`;
+        return envelope;
+    }
+    healNavigator() {
+        if (this.aiUnavailable())
+            return null;
+        try {
+            return this.bot.agentNavigator?.() ?? null;
+        }
+        catch {
+            return null;
+        }
+    }
+    aiUnavailable() {
+        try {
+            if (this.bot.getProvider?.())
+                return null;
+        }
+        catch (error) {
+            return browserErrorMessage(error);
+        }
+        return this.bot.aiFailureReason?.() || 'no AI model is configured';
+    }
+    aiUnavailableNote() {
+        const reason = this.aiUnavailable();
+        if (!reason)
+            return 'ai unavailable';
+        return `ai unavailable: ${reason}`;
+    }
+    async aiGuard(command) {
+        const reason = this.aiUnavailable();
+        if (!reason)
+            return null;
+        const message = `this command needs an AI model and none is usable: ${reason}. Drive the browser directly with playwright-cli or prima pw, or fix the AI config in ~/.explorbot/config.js or EXPLORBOT_AI_PROVIDER.`;
+        return this.toolFailureEnvelope(command, message);
+    }
+    instructionSystemPrompt() {
+        return dedent `
+      <role>
+      You are a web automation engineer performing high-level instructions on the page that is already open in the browser.
+      </role>
+
+      <approach>
+      1. Read the page context and perform the instructions in the order they are listed.
+      2. Interact with the page only through the provided tools.
+      3. Pick the smallest interaction that fulfills an instruction, then move to the next one.
+      4. After the page changes, work from the updated context you are given, not from the earlier one.
+      5. Stop calling tools when every instruction is done, or when an instruction cannot be performed on this page — say what is missing instead.
+      </approach>
+
+      ${locatorRule}
+
+      ${actionRule}
+    `;
+    }
+    instructionPrompt(instructions, result) {
+        const list = instructions.map((instruction, index) => `${index + 1}. ${instruction}`).join('\n');
+        return dedent `
+      <instructions>
+      ${list}
+      </instructions>
+
+      ${this.pageContext(result)}
+    `;
+    }
+    pageContext(result) {
+        const experience = this.bot.experienceTracker?.()?.renderExperienceTocFor?.(result) || '';
+        return dedent `
+      <page url="${result.url}" title="${result.title}">
+      ${compactAriaSnapshot(result.ariaSnapshot, true)}
+      </page>
+
+      ${experience}
+    `;
+    }
+    executedCodes(code) {
+        if (typeof code !== 'string')
+            return [];
+        return code
+            .split('\n')
+            .map((line) => line.trim())
+            .filter((line) => line);
+    }
+    visionEnabled() {
+        if (this.options.noVision)
+            return false;
+        return this.bot.getProvider().hasVision?.() === true;
+    }
+    async answer(question, result) {
+        const seen = await this.visionAnswer(question, result);
+        if (seen)
+            return seen;
+        const described = await this.textAnswer(question, result);
+        if (this.options.noVision)
+            return described;
+        return `${described}\n\n(vision was unavailable, answered from page structure)`;
+    }
+    async visionAnswer(question, result) {
+        if (!this.visionEnabled())
+            return null;
+        return this.bot.agentResearcher().answerQuestionAboutScreenshot(result, question);
+    }
+    async textAnswer(question, result) {
+        const provider = this.bot.getProvider();
+        const summary = await this.bot.agentResearcher().summary(result);
+        const prompt = dedent `
+      Answer the question about the web page described below.
+      Rely only on the page summary and the accessibility tree given here.
+      When the answer is not present in them, say which part of the page would be needed instead of guessing.
+      Keep the answer under five sentences.
+
+      <question>
+      ${question}
+      </question>
+
+      <page_summary>
+      ${summary}
+      </page_summary>
+
+      <page_aria>
+      ${compactAriaSnapshot(result.ariaSnapshot, true)}
+      </page_aria>
+    `;
+        const response = await provider.chat([{ role: 'user', content: prompt }], provider.getModelForAgent?.(AI_AGENT_NAME), { agentName: AI_AGENT_NAME });
+        return response?.text || '';
+    }
+    verdictEvidence(verified, codes) {
+        if (!verified)
+            return 'no assertion held on the current page';
+        if (!codes.length)
+            return 'already verified on this page';
+        return `${codes[0]} passed`;
+    }
+    async successEnvelope(command, used, result, previousState) {
+        return {
+            ok: true,
+            command,
+            used,
+            page: this.pageBlock(result, previousState),
+            changes: await this.pageChanges(result, previousState, used[0]),
+            instance: await this.instanceInfo(),
+            artifacts: await this.writeSnapshot(result),
+        };
+    }
+    async failureEnvelope(command, error, previousState, attempts = []) {
+        const result = await this.capturedResult(previousState);
+        const failure = { error: browserErrorMessage(error), attempts };
+        if (result.ariaSnapshot)
+            failure.compactAria = compactAriaSnapshot(result.ariaSnapshot, true);
+        if (attempts.length)
+            failure.reasoning = [...new Set(attempts.map((attempt) => attempt.outcome))].join('; ');
+        return {
+            ok: false,
+            command,
+            page: this.pageBlock(result, previousState),
+            failure,
+            instance: await this.instanceInfo(),
+            artifacts: await this.writeSnapshot(result),
+        };
+    }
+    async reportEnvelope(command, result, previousState, outcome) {
+        return {
+            ok: true,
+            command,
+            page: this.pageBlock(result, previousState),
+            ...outcome,
+            instance: await this.instanceInfo(),
+            artifacts: await this.writeSnapshot(result),
+        };
+    }
+    async capturedResult(previousState, opts = {}) {
+        const captured = await this.bot
+            .getExplorer()
+            ?.capture(opts)
+            .catch(() => null);
+        if (captured)
+            return captured;
+        if (previousState)
+            return ActionResult.fromState(previousState);
+        return new ActionResult({ url: '' });
+    }
+    pageBlock(result, previousState) {
+        return {
+            url: result.url,
+            previousUrl: previousState?.url,
+            title: result.title,
+            state: result.getStateHash(),
+            visits: this.bot.stateManager().getVisitCount(result.url),
+        };
+    }
+    async pageChanges(result, previousState, code) {
+        if (!previousState)
+            return null;
+        const toolResult = await result.toToolResult(ActionResult.fromState(previousState), code);
+        return toolResult.pageDiff?.ariaChanges ?? null;
+    }
+    async writeSnapshot(result) {
+        return writeArtifacts(this.nextArtifactDir(), {
+            aria: result.ariaSnapshot,
+            html: await result.combinedHtml(),
+            requests: this.bot.requestStore().getRequests(),
+        });
+    }
+    nextArtifactDir() {
+        this.artifactsDir ||= outputPath('prima');
+        return path.join(this.artifactsDir, new Date().toISOString().replace(/[:.]/g, '-'));
+    }
+    tabCount() {
+        const page = this.bot.getExplorer()?.page;
+        if (!page)
+            return 0;
+        return page.context().pages().length;
+    }
+    instanceName() {
+        return this.options.instance || 'default';
+    }
+}

package/dist/boat/prima/src/pw-parser.js

@@ -0,0 +1,18 @@
+const FUNCTION_SHAPE = /^(async\s+)?(function\b|\()/;
+export function isFunctionExpression(expr) {
+    const trimmed = expr.trim();
+    if (!trimmed)
+        return { valid: false, error: 'empty expression; pass a function like ({ page }) => ...' };
+    if (!FUNCTION_SHAPE.test(trimmed))
+        return { valid: false, error: 'expression must be a function like ({ page }) => ... destructuring the playwright objects it needs' };
+    try {
+        new Function(`return (${trimmed})`);
+    }
+    catch (e) {
+        return { valid: false, error: `not a valid function expression: ${e.message}` };
+    }
+    return { valid: true };
+}
+export function toCodeceptWrapper(expr) {
+    return `I.usePlaywrightTo('pw', async (playwright) => (${expr.trim()})(playwright))`;
+}