@minionry/minion 0.7.36

package/dist/server/services/browser/reasoning-agent.js

@@ -0,0 +1,659 @@
+// Copyright (c) 2025-present Minionry, Inc.
+// SPDX-License-Identifier: LicenseRef-Minionry-Software
+import { randomUUID } from 'node:crypto';
+import { formatOperationForReview, mapBouncerDecisionToOutcome, reviewOperation, } from '../../mcp/bouncer-integration.js';
+import { browserLog } from './debug-log.js';
+import { getBrowserDriverState } from './driver-state.js';
+const agentLog = browserLog('agent');
+export const AGENT_HEARTBEAT_INTERVAL_MS = 10_000;
+export const DEFAULT_MAX_STEPS = 24;
+export const MAX_CONSECUTIVE_NONMUTATING_STEPS = 3;
+export const MAX_TURNS_PER_GOAL = 6;
+export const MAX_STEP_RETRIES = 2;
+export const MAX_REPLANS = 2;
+export const ARBITRATION_WAIT_MS = 30_000;
+export const ARBITRATION_POLL_MS = 500;
+export const FAILSAFE_RESULT_PREFIX = 'browser-reasoner:';
+function isFailsafeResult(result) {
+    return result?.startsWith(FAILSAFE_RESULT_PREFIX) ?? false;
+}
+function formatPlan(goals) {
+    return `Plan:\n${goals.map((g, i) => `${i + 1}. ${g}`).join('\n')}`;
+}
+export const EXTRACT_CONTENT_CHAR_BUDGET = 8_000;
+export function serializeExtractContent(root, charBudget = EXTRACT_CONTENT_CHAR_BUDGET) {
+    const lines = [];
+    let chars = 0;
+    let truncated = false;
+    const walk = (node) => {
+        if (truncated)
+            return;
+        const name = node.name?.trim();
+        if (name) {
+            let line = `${node.role} "${name}"`;
+            if (node.value !== undefined)
+                line += ` value="${node.value}"`;
+            if (chars + line.length + 1 > charBudget) {
+                truncated = true;
+                return;
+            }
+            lines.push(line);
+            chars += line.length + 1;
+        }
+        for (const child of node.children) {
+            if (truncated)
+                break;
+            walk(child);
+        }
+    };
+    walk(root);
+    if (truncated)
+        lines.push('... [content truncated]');
+    return lines.join('\n');
+}
+function validateDone(input) {
+    if (!input.isFailsafe)
+        return input.isLastGoal ? 'finish' : 'advance';
+    if (input.failsafeStreak < MAX_STEP_RETRIES)
+        return 'retry';
+    if (input.replans < MAX_REPLANS)
+        return 'replan';
+    return 'finish';
+}
+function validateProgress(input) {
+    if (input.planLength <= 1 || input.turnsOnGoal + 1 < MAX_TURNS_PER_GOAL)
+        return 'continue';
+    if (input.replans < MAX_REPLANS)
+        return 'replan';
+    return input.isLastGoal ? 'finish' : 'advance';
+}
+export const defaultStepValidator = (input) => input.directive.kind === 'done' ? validateDone(input) : validateProgress(input);
+export function parseNavigationIntent(task) {
+    const trimmed = task.trim();
+    if (!trimmed)
+        return null;
+    const verb = /^(?:open|go\s+to|goto|navigate\s+to|nav\s+to|visit|load|browse\s+to)\s+(.+)$/i;
+    const match = trimmed.match(verb);
+    const candidate = (match ? match[1] : trimmed).trim();
+    if (/\s/.test(candidate))
+        return null;
+    return normalizeUrlCandidate(candidate);
+}
+function normalizeUrlCandidate(raw) {
+    let s = raw.replace(/^['"]+|['"]+$/g, '').replace(/[.,!?;:]+$/g, '');
+    if (!s)
+        return null;
+    const scheme = s.match(/^([a-z][a-z0-9+.-]*):\/\//i);
+    if (scheme) {
+        const proto = scheme[1].toLowerCase();
+        if (proto !== 'http' && proto !== 'https')
+            return null;
+    }
+    else {
+        const host = s.split(/[/?#]/, 1)[0];
+        const isLocalhost = host === 'localhost' || host.startsWith('localhost:');
+        if (!isLocalhost && !/^[^\s.]+(?:\.[^\s.]+)+$/.test(host))
+            return null;
+        s = `https://${s}`;
+    }
+    try {
+        const url = new URL(s);
+        if (url.protocol !== 'http:' && url.protocol !== 'https:')
+            return null;
+        return url.toString();
+    }
+    catch {
+        return null;
+    }
+}
+export function shouldCaptureScreenshot(input) {
+    if (input.failsafeStreak > 0)
+        return true;
+    return input.interactiveNodeCount === 0 && input.totalNodeCount > 1;
+}
+export function countInteractiveNodes(node) {
+    let total = INTERACTIVE_SNAPSHOT_ROLES.has(node.role.toLowerCase()) ? 1 : 0;
+    for (const child of node.children)
+        total += countInteractiveNodes(child);
+    return total;
+}
+const INTERACTIVE_SNAPSHOT_ROLES = new Set([
+    'button', 'link', 'textbox', 'searchbox', 'checkbox', 'radio', 'combobox',
+    'listbox', 'option', 'menuitem', 'menuitemcheckbox', 'menuitemradio',
+    'menulistoption', 'tab', 'switch', 'slider', 'spinbutton', 'treeitem',
+    'gridcell', 'columnheader', 'rowheader',
+]);
+export function withNavigationFastPath(inner) {
+    return (input) => {
+        const navUrl = parseNavigationIntent(input.task);
+        if (navUrl) {
+            const navigated = input.history.some((h) => h.phase === 'act');
+            return navigated
+                ? { kind: 'done', result: `Navigated to ${navUrl}` }
+                : { kind: 'navigate', url: navUrl, summary: `Navigate to ${navUrl}` };
+        }
+        return inner(input);
+    };
+}
+export const REASONING_UNAVAILABLE_RESULT = 'Autonomous browser reasoning is unavailable, so this task was not attempted. ' +
+    'Direct navigation still works — try "go to <site>". To enable multi-step browser tasks, ' +
+    'make sure the selected reasoning engine (e.g. the Claude CLI) is installed and authenticated.';
+export function defaultAgentReasoner({ task, history, }) {
+    const navUrl = parseNavigationIntent(task);
+    const navigated = history.some((h) => h.phase === 'act');
+    if (navUrl && !navigated) {
+        agentLog('reasoner.navigate', `url=${navUrl}`);
+        return { kind: 'navigate', url: navUrl, summary: `Navigate to ${navUrl}` };
+    }
+    if (navUrl && navigated) {
+        return { kind: 'done', result: `Navigated to ${navUrl}` };
+    }
+    agentLog('reasoner.unavailable', `non-navigation task with no model-backed reasoner: "${task}"`);
+    return { kind: 'done', result: REASONING_UNAVAILABLE_RESULT };
+}
+const AGENT_ACTION_TOOL_NAMES = {
+    click: 'browser_click',
+    type: 'browser_type',
+    hover: 'browser_hover',
+    pressKey: 'browser_press_key',
+    selectOption: 'browser_select_option',
+    fillForm: 'browser_fill_form',
+};
+function agentActionOperation(action) {
+    const { method, ...args } = action;
+    return formatOperationForReview(AGENT_ACTION_TOOL_NAMES[method], args);
+}
+export const defaultAgentActionReviewer = (action, tabId) => reviewOperation({ operation: agentActionOperation(action), context: { browserTabId: tabId } });
+export class ReasoningAgent {
+    host;
+    sink;
+    reasoner;
+    planner;
+    validator;
+    reviewer;
+    task;
+    agentId;
+    agentLabel;
+    initialUrl;
+    maxSteps;
+    heartbeatIntervalMs;
+    settleDelayMs;
+    setIntervalFn;
+    clearIntervalFn;
+    sleepFn;
+    tabId;
+    leased = false;
+    heartbeatHandle = null;
+    pending = null;
+    finished = false;
+    cancelled = false;
+    history = [];
+    cachedSnapshot = null;
+    nonMutatingStreak = 0;
+    plan = [];
+    stepIndex = 0;
+    turnsOnGoal = 0;
+    failsafeStreak = 0;
+    replans = 0;
+    constructor(opts) {
+        this.host = opts.host;
+        this.sink = opts.sink;
+        this.reasoner = opts.reasoner;
+        this.planner = opts.planner;
+        this.validator = opts.validator ?? defaultStepValidator;
+        this.reviewer = opts.reviewer ?? defaultAgentActionReviewer;
+        this.task = opts.task;
+        this.agentId = opts.agentId ?? `browser-agent-${randomUUID()}`;
+        this.agentLabel = opts.agentLabel ?? 'Browser Agent';
+        this.tabId = opts.tabId;
+        this.initialUrl = opts.initialUrl ?? 'about:blank';
+        this.maxSteps = opts.maxSteps ?? DEFAULT_MAX_STEPS;
+        this.heartbeatIntervalMs = opts.heartbeatIntervalMs ?? AGENT_HEARTBEAT_INTERVAL_MS;
+        this.settleDelayMs = opts.settleDelayMs ?? 200;
+        this.setIntervalFn = opts.setIntervalFn ?? ((handler, ms) => setInterval(handler, ms));
+        this.clearIntervalFn = opts.clearIntervalFn ?? clearInterval;
+        this.sleepFn = opts.sleepFn ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+    }
+    get pendingProposalId() {
+        return this.pending?.proposalId ?? null;
+    }
+    get id() {
+        return this.agentId;
+    }
+    async run() {
+        try {
+            if (!this.tabId) {
+                const tab = await this.host.openTab(this.initialUrl);
+                this.tabId = tab.id;
+            }
+            await this.host.leaseTab({ tabId: this.tabId, agentId: this.agentId });
+            this.leased = true;
+            this.startHeartbeat();
+            await this.sink.badgeTab();
+            this.recordTarget();
+            await this.planTask();
+            for (let i = 0; i < this.maxSteps && !this.cancelled; i++) {
+                const snapshot = await this.observe();
+                const currentGoal = this.currentGoalText();
+                const screenshot = await this.maybeCaptureScreenshot(snapshot);
+                const directive = await this.reasoner({
+                    task: this.task.task,
+                    snapshot,
+                    history: this.history,
+                    ...(currentGoal !== undefined ? { currentGoal } : {}),
+                    ...(screenshot ? { screenshot } : {}),
+                });
+                agentLog('directive', `step=${i} goal=${this.stepIndex + 1}/${this.plan.length} kind=${directive.kind}`);
+                if (this.cancelled)
+                    break;
+                if (await this.advanceStateMachine(directive))
+                    return;
+            }
+            this.finish(this.cancelled ? 'cancelled' : 'success');
+        }
+        catch (err) {
+            this.finish('error', undefined, err);
+        }
+        finally {
+            this.cleanup();
+        }
+    }
+    async planTask() {
+        this.plan = [this.task.task];
+        if (!this.planner)
+            return;
+        if (parseNavigationIntent(this.task.task) !== null)
+            return;
+        let goals = [];
+        try {
+            goals = await this.planner({ task: this.task.task });
+        }
+        catch {
+            goals = [];
+        }
+        if (goals.length > 1) {
+            this.plan = goals;
+            this.recordStep('plan', formatPlan(goals));
+        }
+    }
+    currentGoalText() {
+        if (this.plan.length <= 1)
+            return undefined;
+        return this.plan[Math.min(this.stepIndex, this.plan.length - 1)];
+    }
+    advanceGoal() {
+        this.stepIndex += 1;
+        this.turnsOnGoal = 0;
+        this.failsafeStreak = 0;
+        this.nonMutatingStreak = 0;
+    }
+    async advanceStateMachine(directive) {
+        const isFailsafe = directive.kind === 'done' && isFailsafeResult(directive.result);
+        if (directive.kind !== 'done') {
+            if (await this.executeDirective(directive))
+                return true;
+        }
+        const verdict = this.validator({
+            directive,
+            isFailsafe,
+            isLastGoal: this.stepIndex >= this.plan.length - 1,
+            planLength: this.plan.length,
+            turnsOnGoal: this.turnsOnGoal,
+            failsafeStreak: this.failsafeStreak,
+            replans: this.replans,
+        });
+        return this.applyVerdict(verdict, directive);
+    }
+    async applyVerdict(verdict, directive) {
+        switch (verdict) {
+            case 'finish':
+                this.finish('success', directive.kind === 'done' ? directive.result : undefined);
+                return true;
+            case 'advance':
+                this.advanceGoal();
+                return false;
+            case 'retry':
+                this.failsafeStreak += 1;
+                this.cachedSnapshot = null;
+                return false;
+            case 'replan':
+                return this.replan(directive);
+            case 'continue': {
+                this.failsafeStreak = 0;
+                this.turnsOnGoal += 1;
+                if (directive.kind === 'extract') {
+                    if (++this.nonMutatingStreak >= MAX_CONSECUTIVE_NONMUTATING_STEPS) {
+                        this.finish('success', `Stopped after ${this.nonMutatingStreak} read-only steps without making progress on the task.`);
+                        return true;
+                    }
+                }
+                else {
+                    this.nonMutatingStreak = 0;
+                }
+                return false;
+            }
+        }
+    }
+    async replan(trigger) {
+        if (!this.planner) {
+            this.finish('success', trigger.kind === 'done' ? trigger.result : undefined);
+            return true;
+        }
+        this.replans += 1;
+        let goals = [];
+        try {
+            const tabId = this.requireTabId();
+            const snapshot = this.cachedSnapshot ?? (await this.host.dom.snapshot({ tabId }));
+            goals = await this.planner({ task: this.task.task, snapshot });
+        }
+        catch {
+            goals = [];
+        }
+        this.plan = goals.length >= 1 ? goals : [this.task.task];
+        this.stepIndex = 0;
+        this.turnsOnGoal = 0;
+        this.failsafeStreak = 0;
+        this.nonMutatingStreak = 0;
+        this.recordStep('plan', `Re-planning from the current page.\n${formatPlan(this.plan)}`);
+        return false;
+    }
+    async executeDirective(directive) {
+        switch (directive.kind) {
+            case 'done':
+                return false;
+            case 'navigate':
+                await this.navigate(directive.url, directive.summary);
+                return false;
+            case 'extract':
+                await this.extract(directive.summary);
+                return false;
+            case 'act': {
+                const decision = await this.greenlight(directive.action, directive.summary);
+                if (decision.outcome === 'deny') {
+                    if (decision.reason !== undefined) {
+                        this.recordStep('act', `Skipped — Bouncer denied: ${decision.reason}`);
+                    }
+                    this.finish('cancelled', decision.reason);
+                    return true;
+                }
+                await this.act(decision.action, directive.summary, decision.proposalId);
+                return false;
+            }
+        }
+    }
+    approve(payload) {
+        if (!this.pending || this.pending.proposalId !== payload.proposalId)
+            return false;
+        const { resolve } = this.pending;
+        this.pending = null;
+        resolve(payload);
+        return true;
+    }
+    cancel() {
+        this.cancelled = true;
+        this.stopHeartbeat();
+        if (this.pending) {
+            const { resolve, proposalId } = this.pending;
+            this.pending = null;
+            resolve({ proposalId, verdict: 'deny' });
+        }
+    }
+    async reviewAction(action) {
+        const tabId = this.requireTabId();
+        const gate = await this.waitForPerson(tabId, AGENT_ACTION_TOOL_NAMES[action.method]);
+        if (!gate.allow) {
+            return { outcome: 'autoDeny', reason: `Driver arbitration: ${gate.hint}` };
+        }
+        const decision = await this.reviewer(action, tabId);
+        return { outcome: mapBouncerDecisionToOutcome(decision), reason: decision.reasoning };
+    }
+    async waitForPerson(tabId, tool) {
+        const driverState = getBrowserDriverState();
+        let gate = driverState.gateSync(tabId, tool);
+        if (gate.allow)
+            return gate;
+        agentLog('arbitration.wait', `tab=${tabId.slice(0, 8)} tool=${tool} state=${gate.state}`);
+        this.emitStep('observe', 'Waiting for you to finish on this tab…');
+        for (let waited = 0; waited < ARBITRATION_WAIT_MS && !this.cancelled; waited += ARBITRATION_POLL_MS) {
+            await this.sleepFn(ARBITRATION_POLL_MS);
+            gate = driverState.gateSync(tabId, tool);
+            if (gate.allow)
+                return gate;
+        }
+        return gate;
+    }
+    async greenlight(action, summary) {
+        const review = await this.reviewAction(action);
+        if (this.cancelled)
+            return { outcome: 'deny' };
+        if (review.outcome === 'autoDeny') {
+            return { outcome: 'deny', reason: review.reason };
+        }
+        if (review.outcome === 'autoAllow') {
+            return { outcome: 'proceed', action };
+        }
+        const proposalId = randomUUID();
+        const proposed = {
+            proposalId,
+            summary,
+            ...(this.task.boardId ? { boardId: this.task.boardId } : {}),
+        };
+        this.sink.emit({ type: 'browser_agent_proposed', data: proposed });
+        const verdict = await new Promise((resolve) => {
+            this.pending = { proposalId, resolve };
+        });
+        if (verdict.verdict === 'deny')
+            return { outcome: 'deny' };
+        const effective = verdict.verdict === 'edit'
+            ? this.applyEdit(action, verdict.editedText)
+            : action;
+        return { outcome: 'proceed', action: effective, proposalId };
+    }
+    applyEdit(action, editedText) {
+        if (editedText === undefined)
+            return action;
+        if (action.method === 'type')
+            return { ...action, text: editedText };
+        return action;
+    }
+    async settle(kind) {
+        if (kind === 'act' && this.settleDelayMs > 0) {
+            await this.sleepFn(this.settleDelayMs);
+        }
+        try {
+            await this.host.dom.waitFor({
+                tabId: this.requireTabId(),
+                timeoutMs: kind === 'navigate' ? 10_000 : 5_000,
+                pollIntervalMs: 150,
+                condition: { type: 'navigation' },
+                agentId: this.agentId,
+            });
+        }
+        catch {
+        }
+    }
+    async observe() {
+        const tabId = this.requireTabId();
+        const snapshot = this.cachedSnapshot ?? (await this.host.dom.snapshot({ tabId }));
+        this.cachedSnapshot = snapshot;
+        this.recordStep('observe', `Observed ${countNodes(snapshot.root)} accessibility nodes`);
+        return snapshot;
+    }
+    async maybeCaptureScreenshot(snapshot) {
+        const capture = shouldCaptureScreenshot({
+            failsafeStreak: this.failsafeStreak,
+            interactiveNodeCount: countInteractiveNodes(snapshot.root),
+            totalNodeCount: countNodes(snapshot.root),
+        });
+        if (!capture)
+            return null;
+        try {
+            const shot = await this.host.dom.screenshot({
+                tabId: this.requireTabId(),
+                format: 'jpeg',
+                quality: 60,
+            });
+            if (!shot?.data)
+                return null;
+            this.recordStep('observe', 'Captured a screenshot to visually ground the next step');
+            return { data: shot.data, format: shot.format === 'jpeg' ? 'jpeg' : 'png' };
+        }
+        catch {
+            return null;
+        }
+    }
+    async navigate(url, summary) {
+        const tabId = this.requireTabId();
+        agentLog('navigate', `tab=${tabId.slice(0, 8)} url=${url}`);
+        this.markDriving(tabId);
+        await this.host.navigate(url, tabId);
+        await this.settle('navigate');
+        this.cachedSnapshot = null;
+        this.recordStep('act', summary ?? `Navigated to ${url}`);
+    }
+    async act(action, summary, proposalId) {
+        const tabId = this.requireTabId();
+        const agentId = this.agentId;
+        const { dom } = this.host;
+        this.markDriving(tabId);
+        switch (action.method) {
+            case 'click':
+                await dom.click({ tabId, ref: action.ref, agentId });
+                break;
+            case 'type':
+                await dom.type({
+                    tabId,
+                    ref: action.ref,
+                    text: action.text,
+                    agentId,
+                    ...(action.submit !== undefined ? { submit: action.submit } : {}),
+                });
+                break;
+            case 'hover':
+                await dom.hover({ tabId, ref: action.ref, agentId });
+                break;
+            case 'pressKey':
+                await dom.pressKey({ tabId, key: action.key, agentId });
+                break;
+            case 'selectOption':
+                await dom.selectOption({ tabId, ref: action.ref, values: action.values, agentId });
+                break;
+            case 'fillForm':
+                await dom.fillForm({ tabId, fields: action.fields, agentId });
+                break;
+        }
+        await this.settle('act');
+        this.cachedSnapshot = null;
+        this.recordStep('act', summary, proposalId);
+    }
+    async extract(summary) {
+        const tabId = this.requireTabId();
+        const snapshot = this.cachedSnapshot ?? (await this.host.dom.snapshot({ tabId }));
+        this.cachedSnapshot = snapshot;
+        const [consoleEntries, networkEntries] = await Promise.all([
+            this.host.dom.console({ tabId }),
+            this.host.dom.network({ tabId }),
+        ]);
+        const detail = summary ??
+            `Extracted ${countNodes(snapshot.root)} nodes, ${consoleEntries.length} console, ${networkEntries.length} network entries`;
+        const content = serializeExtractContent(snapshot.root);
+        this.recordStep('extract', detail, undefined, content ? `${detail}\nPage content:\n${content}` : detail);
+    }
+    recordTarget() {
+        const { engine, model } = this.task;
+        if (!engine && !model)
+            return;
+        const target = [engine ?? 'default engine', model && model !== 'default' ? model : null]
+            .filter(Boolean)
+            .join(' · ');
+        this.emitStep('plan', `Running on ${target}`);
+    }
+    recordStep(phase, detail, proposalId, historyDetail) {
+        this.history.push({ phase, detail: historyDetail ?? detail });
+        this.emitStep(phase, detail, proposalId);
+    }
+    emitStep(phase, detail, proposalId) {
+        const payload = {
+            phase,
+            detail,
+            ...(this.task.taskId ? { taskId: this.task.taskId } : {}),
+            ...(proposalId ? { proposalId } : {}),
+            ...(this.task.engine ? { engine: this.task.engine } : {}),
+            ...(this.task.model ? { model: this.task.model } : {}),
+        };
+        agentLog('step', `phase=${phase} task=${this.task.taskId ?? '-'} detail=${detail}`);
+        this.sink.emit({ type: 'browser_agent_step', data: payload });
+    }
+    finish(status, result, err) {
+        if (this.finished)
+            return;
+        this.finished = true;
+        const userResult = toUserFacingResult(result);
+        const payload = {
+            status,
+            ...(this.task.taskId ? { taskId: this.task.taskId } : {}),
+            ...(this.task.boardId ? { boardId: this.task.boardId } : {}),
+            ...(userResult !== undefined ? { result: userResult } : {}),
+            ...(status === 'error'
+                ? { error: err instanceof Error ? err.message : String(err ?? 'unknown error') }
+                : {}),
+        };
+        const rawNote = userResult !== result ? ` rawResult="${result}"` : '';
+        agentLog('done', `status=${status} task=${this.task.taskId ?? '-'}${rawNote}`);
+        this.sink.emit({ type: 'browser_agent_done', data: payload });
+    }
+    startHeartbeat() {
+        if (this.heartbeatHandle)
+            return;
+        this.heartbeatHandle = this.setIntervalFn(() => {
+            void this.heartbeatTick();
+        }, this.heartbeatIntervalMs);
+    }
+    async heartbeatTick() {
+        if (!this.tabId)
+            return;
+        try {
+            await this.host.heartbeat({ agentId: this.agentId, tabIds: [this.tabId] });
+        }
+        catch {
+        }
+    }
+    stopHeartbeat() {
+        if (this.heartbeatHandle) {
+            this.clearIntervalFn(this.heartbeatHandle);
+            this.heartbeatHandle = null;
+        }
+    }
+    cleanup() {
+        this.stopHeartbeat();
+        if (this.tabId && this.leased) {
+            this.leased = false;
+            void this.host
+                .releaseTab({ tabId: this.tabId, agentId: this.agentId })
+                .then(() => this.sink.badgeTab())
+                .catch(() => {
+            });
+        }
+    }
+    requireTabId() {
+        if (!this.tabId)
+            throw new Error('reasoning-agent: no tab acquired');
+        return this.tabId;
+    }
+    markDriving(tabId) {
+        getBrowserDriverState().recordInput(tabId, 'agent');
+    }
+}
+function toUserFacingResult(result) {
+    if (result === undefined)
+        return undefined;
+    if (!isFailsafeResult(result))
+        return result;
+    return "I couldn't complete this browser task — I had trouble understanding the page. Please try rephrasing the task, or take the action manually.";
+}
+function countNodes(node) {
+    let total = 1;
+    for (const child of node.children)
+        total += countNodes(child);
+    return total;
+}