@nuanu-ai/agentbrowse 0.2.7 → 0.2.8

package/dist/commands/poll-intent.js

@@ -0,0 +1,57 @@
+/**
+ * browse poll-intent <intentId> — Poll stored-secret intent state and capture one-time delivery.
+ */
+import { outputFailure, outputJSON } from '../output.js';
+import { saveSecretIntentSnapshot } from '../runtime-state.js';
+import { cacheTransientSecret, deleteCachedTransientSecret, saveSession } from '../session.js';
+import { getSecretBackend } from '../secrets/backend.js';
+import { describeSecretIntentStatus, serializeSecretIntent } from '../secrets/intent-output.js';
+export async function pollIntent(session, intentId) {
+    const backend = getSecretBackend();
+    const result = await backend.getSecretIntent(intentId);
+    if (!result) {
+        return outputFailure({
+            error: 'secret_intent_not_found',
+            message: 'Protected intent lookup failed because the requested intent does not exist.',
+            outcomeType: 'blocked',
+            reason: 'The provided intentId was not found in AgentPay Cabinet.',
+            intentId,
+        });
+    }
+    const snapshot = saveSecretIntentSnapshot(session, result.snapshot);
+    if (result.deliveredValues) {
+        cacheTransientSecret(session, {
+            intentId: snapshot.intentId,
+            fillRef: snapshot.fillRef,
+            storedSecretRef: snapshot.storedSecretRef,
+            cachedAt: new Date().toISOString(),
+            approvedAt: snapshot.approvedAt,
+            expiresAt: snapshot.expiresAt,
+            values: result.deliveredValues,
+        });
+    }
+    else if (snapshot.status === 'denied' ||
+        snapshot.status === 'timed_out' ||
+        snapshot.status === 'completed') {
+        deleteCachedTransientSecret(session, snapshot.intentId);
+    }
+    saveSession(session);
+    if (result.deliveredValues && snapshot.status === 'completed') {
+        return outputJSON({
+            success: true,
+            ...serializeSecretIntent(snapshot),
+            message: 'Protected intent delivered a one-time secret payload and is ready to fill.',
+            reason: 'AgentPay approved this request, returned the protected values, and completed the intent.',
+            nextAction: 'fill-secret',
+        });
+    }
+    const statusContract = describeSecretIntentStatus(snapshot.status);
+    outputJSON({
+        success: true,
+        ...serializeSecretIntent(snapshot),
+        ...(statusContract.outcomeType ? { outcomeType: statusContract.outcomeType } : {}),
+        message: statusContract.message,
+        reason: statusContract.reason,
+        ...(statusContract.nextAction ? { nextAction: statusContract.nextAction } : {}),
+    });
+}

package/dist/commands/screenshot.d.ts

@@ -1,5 +1,6 @@
 /**
  * browse screenshot [--path <file>] — Capture a screenshot of the current page.
  */
-export declare function screenshot(cdpUrl: string, filePath?: string): Promise<void>;
+import type { BrowseSession } from '../session.js';
+export declare function screenshot(session: BrowseSession, filePath?: string): Promise<void>;
 //# sourceMappingURL=screenshot.d.ts.map

package/dist/commands/screenshot.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"screenshot.d.ts","sourceRoot":"","sources":["../../src/commands/screenshot.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,wBAAsB,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAyBjF"}
+{"version":3,"file":"screenshot.d.ts","sourceRoot":"","sources":["../../src/commands/screenshot.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAYnD,wBAAsB,UAAU,CAAC,OAAO,EAAE,aAAa,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAqDzF"}

package/dist/commands/screenshot.js

@@ -1,27 +1,59 @@
 /**
  * browse screenshot [--path <file>] — Capture a screenshot of the current page.
  */
-import { connectStagehand, getPageInfo } from '../stagehand.js';
-import { outputJSON, outputError, info } from '../output.js';
-export async function screenshot(cdpUrl, filePath) {
+import { getProtectedExposure } from '../runtime-state.js';
+import { saveSession } from '../session.js';
+import { outputFailure, outputJSON, outputContractFailure, info } from '../output.js';
+import { connectPlaywright, disconnectPlaywright, listPages, resolvePageByRef, syncSessionPage, } from '../playwright-runtime.js';
+import { buildProtectedScreenshotBlockedResult } from '../secrets/protected-artifact-guard.js';
+export async function screenshot(session, filePath) {
     const outputPath = filePath ?? `/tmp/browse-screenshot-${Date.now()}.png`;
-    let stagehand;
+    const pageRef = session.runtime?.currentPageRef ?? 'p0';
+    const protectedExposure = getProtectedExposure(session, pageRef);
+    if (protectedExposure) {
+        return outputFailure(buildProtectedScreenshotBlockedResult(protectedExposure));
+    }
+    let browser = null;
+    let failureMessage = null;
     try {
-        stagehand = await connectStagehand(cdpUrl);
+        browser = await connectPlaywright(session.cdpUrl);
     }
     catch (err) {
-        outputError(`Failed to connect to browser: ${err instanceof Error ? err.message : String(err)}`);
+        return outputContractFailure({
+            error: 'browser_connection_failed',
+            outcomeType: 'blocked',
+            message: 'Screenshot capture could not start because AgentBrowse failed to connect to the browser.',
+            reason: err instanceof Error ? err.message : String(err),
+        });
     }
     try {
-        const page = stagehand.context.pages()[0];
+        const pages = listPages(browser);
+        const page = session.runtime?.pages[pageRef] || pages.length > 1
+            ? await resolvePageByRef(browser, session, pageRef)
+            : pages[0];
+        if (!page) {
+            throw new Error('no_open_pages');
+        }
         await page.screenshot({ path: outputPath });
         info(`Screenshot saved: ${outputPath}`);
-        const { url, title } = await getPageInfo(stagehand);
-        await stagehand.close();
-        outputJSON({ success: true, path: outputPath, url, title });
+        const { url, title } = await syncSessionPage(session, pageRef, page);
+        saveSession(session);
+        outputJSON({ success: true, pageRef, path: outputPath, url, title });
     }
     catch (err) {
-        await stagehand.close();
-        outputError(`Screenshot failed: ${err instanceof Error ? err.message : String(err)}`);
+        failureMessage = `Screenshot failed: ${err instanceof Error ? err.message : String(err)}`;
+    }
+    finally {
+        if (browser) {
+            disconnectPlaywright(browser);
+        }
+    }
+    if (failureMessage) {
+        outputContractFailure({
+            error: 'screenshot_failed',
+            outcomeType: 'blocked',
+            message: 'Screenshot capture failed.',
+            reason: failureMessage.replace(/^Screenshot failed:\s*/, ''),
+        });
     }
 }

package/dist/commands/select-action-executor.d.ts

@@ -0,0 +1,10 @@
+import type { FrameLocator, Locator, Page } from 'playwright-core';
+import { type ActionExecutionGuards } from './action-execution-guards.js';
+type LocatorRoot = Page | FrameLocator | Locator;
+type SelectActionHelpers = {
+    focusLocator: (page: Page, locator: Locator, attempts: string[]) => Promise<void>;
+    clearLocatorForReplacement: (locator: Locator, attempts: string[]) => Promise<void>;
+};
+export declare function applySelectLikeAction(page: Page, root: LocatorRoot, locator: Locator, value: string, attempts: string[], helpers: SelectActionHelpers, guards?: ActionExecutionGuards): Promise<boolean>;
+export {};
+//# sourceMappingURL=select-action-executor.d.ts.map

package/dist/commands/select-action-executor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"select-action-executor.d.ts","sourceRoot":"","sources":["../../src/commands/select-action-executor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAEL,KAAK,qBAAqB,EAC3B,MAAM,8BAA8B,CAAC;AAEtC,KAAK,WAAW,GAAG,IAAI,GAAG,YAAY,GAAG,OAAO,CAAC;AAMjD,KAAK,mBAAmB,GAAG;IACzB,YAAY,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAClF,0BAA0B,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACrF,CAAC;AAoFF,wBAAsB,qBAAqB,CACzC,IAAI,EAAE,IAAI,EACV,IAAI,EAAE,WAAW,EACjB,OAAO,EAAE,OAAO,EAChB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAAE,EAClB,OAAO,EAAE,mBAAmB,EAC5B,MAAM,CAAC,EAAE,qBAAqB,GAC7B,OAAO,CAAC,OAAO,CAAC,CA0BlB"}

package/dist/commands/select-action-executor.js

@@ -0,0 +1,91 @@
+import { runActionExecutionGuard, } from './action-execution-guards.js';
+const OPTION_RESOLVE_TIMEOUT_MS = 250;
+const ASYNC_OPTION_WAIT_MS = 1_000;
+const OPTION_POLL_INTERVAL_MS = 100;
+async function clickOptionFromRoot(root, value, attempts, options, guards) {
+    const deadline = Date.now() + Math.max(0, options?.waitForMs ?? 0);
+    while (true) {
+        await runActionExecutionGuard(guards, 'select.option-resolve');
+        const roleCandidates = ['option', 'menuitem'];
+        for (const role of roleCandidates) {
+            attempts.push(`option.resolve:role:${role}`);
+            try {
+                const option = root.getByRole(role, { name: value }).first();
+                await option.click({ timeout: OPTION_RESOLVE_TIMEOUT_MS });
+                attempts.push(`option.click:role:${role}`);
+                return true;
+            }
+            catch {
+                // Try the next candidate.
+            }
+        }
+        attempts.push('option.resolve:text');
+        try {
+            const option = root.getByText(value).first();
+            await option.click({ timeout: OPTION_RESOLVE_TIMEOUT_MS });
+            attempts.push('option.click:text');
+            return true;
+        }
+        catch {
+            if (Date.now() >= deadline) {
+                return false;
+            }
+        }
+        await runActionExecutionGuard(guards, 'select.option-await');
+        attempts.push('option.await');
+        await new Promise((resolve) => setTimeout(resolve, OPTION_POLL_INTERVAL_MS));
+    }
+}
+async function tryTypeToFilterSelectOption(root, locator, value, attempts, helpers, guards) {
+    if (typeof locator.pressSequentially !== 'function') {
+        return false;
+    }
+    await helpers.clearLocatorForReplacement(locator, attempts);
+    attempts.push('locator.pressSequentially.select');
+    try {
+        await runActionExecutionGuard(guards, 'select.press-sequentially');
+        await locator.pressSequentially(value);
+    }
+    catch {
+        return false;
+    }
+    return clickOptionFromRoot(root, value, attempts, { waitForMs: ASYNC_OPTION_WAIT_MS }, guards);
+}
+async function pressSelectTriggerKey(locator, key, attempts) {
+    attempts.push(`locator.press:${key}`);
+    try {
+        await locator.press(key);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+export async function applySelectLikeAction(page, root, locator, value, attempts, helpers, guards) {
+    attempts.push('locator.selectOption');
+    try {
+        await locator.selectOption(value);
+        return false;
+    }
+    catch {
+        await runActionExecutionGuard(guards, 'select.after-error');
+        await helpers.focusLocator(page, locator, attempts);
+        if (await clickOptionFromRoot(root, value, attempts, undefined, guards)) {
+            return true;
+        }
+        if (await tryTypeToFilterSelectOption(root, locator, value, attempts, helpers, guards)) {
+            return true;
+        }
+        for (const key of ['Enter', 'Space', 'ArrowDown']) {
+            await runActionExecutionGuard(guards, `select.press:${key.toLowerCase()}`);
+            const pressed = await pressSelectTriggerKey(locator, key, attempts);
+            if (!pressed) {
+                continue;
+            }
+            if (await clickOptionFromRoot(root, value, attempts, undefined, guards)) {
+                return true;
+            }
+        }
+        throw new Error('select_option_not_found');
+    }
+}

package/dist/commands/semantic-observe.d.ts

@@ -0,0 +1,24 @@
+import type { TargetAcceptancePolicy, TargetAllowedAction, TargetCapability, TargetContext, TargetControlFamily, TargetStructure } from '../runtime-state.js';
+type ObserveInventoryTarget = {
+    kind?: string;
+    label?: string;
+    role?: string;
+    interactionHint?: 'click';
+    placeholder?: string;
+    title?: string;
+    states?: Record<string, string | boolean | number>;
+    context?: TargetContext;
+    capability?: TargetCapability;
+    allowedActions?: TargetAllowedAction[];
+    acceptancePolicy?: TargetAcceptancePolicy;
+    controlFamily?: TargetControlFamily;
+    surfaceRef?: string;
+    surfacePriority?: number;
+    structure?: TargetStructure;
+    framePath?: string[];
+    frameUrl?: string;
+    formSelector?: string;
+};
+export declare function rerankDomTargetsForGoal<T extends ObserveInventoryTarget>(instruction: string, targets: ReadonlyArray<T>): Promise<T[]>;
+export {};
+//# sourceMappingURL=semantic-observe.d.ts.map

package/dist/commands/semantic-observe.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic-observe.d.ts","sourceRoot":"","sources":["../../src/commands/semantic-observe.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,sBAAsB,EACtB,mBAAmB,EACnB,gBAAgB,EAChB,aAAa,EACb,mBAAmB,EACnB,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAE7B,KAAK,sBAAsB,GAAG;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC,CAAC;IACnD,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,cAAc,CAAC,EAAE,mBAAmB,EAAE,CAAC;IACvC,gBAAgB,CAAC,EAAE,sBAAsB,CAAC;IAC1C,aAAa,CAAC,EAAE,mBAAmB,CAAC;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,CAAC;AA0UF,wBAAsB,uBAAuB,CAAC,CAAC,SAAS,sBAAsB,EAC5E,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC,GACxB,OAAO,CAAC,CAAC,EAAE,CAAC,CAkDd"}

package/dist/commands/semantic-observe.js

@@ -0,0 +1,344 @@
+import { z } from 'zod';
+import { AgentpayStagehandLlmClient } from '../agentpay-stagehand-llm.js';
+import { resolveAgentpayGatewayConfig } from '../agentpay-gateway.js';
+const rerankSchema = z.object({
+    matches: z
+        .array(z.object({
+        candidateId: z.string(),
+    }))
+        .max(8),
+});
+const RERANK_CANDIDATE_LIMIT = 120;
+const FORM_BUCKET_RESERVE_LIMIT = 48;
+const FORM_BUCKET_RESERVE_PER_BUCKET = 8;
+const SCOPE_BUCKET_RESERVE_LIMIT = 24;
+const SCOPE_BUCKET_RESERVE_PER_BUCKET = 2;
+const IFRAME_BUCKET_RESERVE_LIMIT = 24;
+const IFRAME_BUCKET_RESERVE_PER_BUCKET = 2;
+const HIGH_SIGNAL_SCOPE_KINDS = new Set([
+    'dialog',
+    'listbox',
+    'menu',
+    'grid',
+    'tabpanel',
+    'popover',
+    'dropdown',
+    'datepicker',
+    'card',
+    'form',
+]);
+function isFieldLikeTarget(target) {
+    const kind = (target.kind ?? '').trim().toLowerCase();
+    const role = (target.role ?? '').trim().toLowerCase();
+    return (['input', 'textarea', 'select', 'combobox'].includes(kind) ||
+        ['textbox', 'combobox'].includes(role));
+}
+function semanticFormContextKey(context) {
+    for (const node of [context?.landmark, context?.container, context?.group, context?.item]) {
+        const kind = (node?.kind ?? '').trim().toLowerCase();
+        if (kind !== 'form') {
+            continue;
+        }
+        const label = (node?.label ?? node?.text ?? '').replace(/\s+/g, ' ').trim().toLowerCase();
+        return `${kind}|${label}`;
+    }
+    return undefined;
+}
+function formBucketKey(target) {
+    const formSelector = target.formSelector?.trim();
+    if (formSelector) {
+        return `selector:${formSelector}`;
+    }
+    const contextKey = semanticFormContextKey(target.context);
+    return contextKey ? `context:${contextKey}` : undefined;
+}
+function candidateBucketKey(target, options = {}) {
+    const frameKey = target.framePath?.join('>') ?? 'top';
+    const formKey = formBucketKey(target);
+    const surfaceKey = (options.preferFormBucket ? formKey : undefined) ??
+        target.surfaceRef ??
+        formKey ??
+        'page-root';
+    return `${frameKey}|${surfaceKey}`;
+}
+function isPrimaryFormControlTarget(target) {
+    const kind = (target.kind ?? '').trim().toLowerCase();
+    const role = (target.role ?? '').trim().toLowerCase();
+    const controlFamily = (target.controlFamily ?? '').trim().toLowerCase();
+    const acceptancePolicy = (target.acceptancePolicy ?? '').trim().toLowerCase();
+    if (target.allowedActions?.includes('fill') || target.allowedActions?.includes('type')) {
+        return true;
+    }
+    if (target.allowedActions?.includes('select')) {
+        return true;
+    }
+    if (['text-input', 'select', 'datepicker'].includes(controlFamily)) {
+        return true;
+    }
+    if (acceptancePolicy === 'submit' || acceptancePolicy === 'date-selection') {
+        return true;
+    }
+    return (Boolean(formBucketKey(target)) &&
+        acceptancePolicy === 'disclosure' &&
+        (kind === 'button' || role === 'button'));
+}
+function primaryFormTargetPriority(target) {
+    const acceptancePolicy = (target.acceptancePolicy ?? '').trim().toLowerCase();
+    if (isFieldLikeTarget(target)) {
+        return 0;
+    }
+    if (acceptancePolicy === 'submit') {
+        return 1;
+    }
+    if (acceptancePolicy === 'selection' || acceptancePolicy === 'date-selection') {
+        return 2;
+    }
+    if (acceptancePolicy === 'disclosure') {
+        return 3;
+    }
+    return 4;
+}
+function isHighSignalScopeCandidate(target) {
+    if ((target.capability ?? '').trim().toLowerCase() !== 'scope') {
+        return false;
+    }
+    const kind = (target.kind ?? '').trim().toLowerCase();
+    return HIGH_SIGNAL_SCOPE_KINDS.has(kind);
+}
+function scopeCandidatePriority(target) {
+    const kind = (target.kind ?? '').trim().toLowerCase();
+    if (kind === 'dialog' || kind === 'listbox' || kind === 'menu') {
+        return 0;
+    }
+    if (kind === 'card') {
+        return 1;
+    }
+    if (kind === 'grid' || kind === 'tabpanel' || kind === 'datepicker') {
+        return 2;
+    }
+    if (kind === 'form') {
+        return 3;
+    }
+    return 4;
+}
+function collectBucketedCandidateIndexes(targets, options) {
+    const bucketEntries = new Map();
+    for (const [index, target] of targets.entries()) {
+        if (!options.predicate(target)) {
+            continue;
+        }
+        const bucketKey = options.bucketKeyOf(target);
+        if (!bucketKey) {
+            continue;
+        }
+        const entries = bucketEntries.get(bucketKey) ?? [];
+        entries.push({ index, target });
+        bucketEntries.set(bucketKey, entries);
+    }
+    const bucketQueues = [...bucketEntries.values()]
+        .map((entries) => entries
+        .sort((left, right) => {
+        const leftPriority = options.priorityOf?.(left.target) ?? 0;
+        const rightPriority = options.priorityOf?.(right.target) ?? 0;
+        if (leftPriority !== rightPriority) {
+            return leftPriority - rightPriority;
+        }
+        return left.index - right.index;
+    })
+        .slice(0, options.perBucketLimit))
+        .sort((left, right) => left[0].index - right[0].index);
+    const selectedIndexes = [];
+    while (selectedIndexes.length < options.totalLimit) {
+        let progressed = false;
+        for (const queue of bucketQueues) {
+            const next = queue.shift();
+            if (!next) {
+                continue;
+            }
+            selectedIndexes.push(next.index);
+            progressed = true;
+            if (selectedIndexes.length >= options.totalLimit) {
+                break;
+            }
+        }
+        if (!progressed) {
+            break;
+        }
+    }
+    return selectedIndexes;
+}
+function diversifyCandidates(targets) {
+    if (targets.length <= RERANK_CANDIDATE_LIMIT) {
+        return [...targets];
+    }
+    const orderedIndexes = [];
+    const selectedIndexes = new Set();
+    const pushSelectedIndexes = (indexes) => {
+        for (const index of indexes) {
+            if (selectedIndexes.has(index)) {
+                continue;
+            }
+            selectedIndexes.add(index);
+            orderedIndexes.push(index);
+            if (orderedIndexes.length >= RERANK_CANDIDATE_LIMIT) {
+                break;
+            }
+        }
+    };
+    pushSelectedIndexes(collectBucketedCandidateIndexes(targets, {
+        totalLimit: FORM_BUCKET_RESERVE_LIMIT,
+        perBucketLimit: FORM_BUCKET_RESERVE_PER_BUCKET,
+        bucketKeyOf: (target) => candidateBucketKey(target, { preferFormBucket: true }),
+        predicate: (target) => Boolean(formBucketKey(target)) && isPrimaryFormControlTarget(target),
+        priorityOf: primaryFormTargetPriority,
+    }));
+    pushSelectedIndexes(collectBucketedCandidateIndexes(targets, {
+        totalLimit: SCOPE_BUCKET_RESERVE_LIMIT,
+        perBucketLimit: SCOPE_BUCKET_RESERVE_PER_BUCKET,
+        bucketKeyOf: (target) => candidateBucketKey(target),
+        predicate: isHighSignalScopeCandidate,
+        priorityOf: scopeCandidatePriority,
+    }));
+    pushSelectedIndexes(collectBucketedCandidateIndexes(targets, {
+        totalLimit: IFRAME_BUCKET_RESERVE_LIMIT,
+        perBucketLimit: IFRAME_BUCKET_RESERVE_PER_BUCKET,
+        bucketKeyOf: (target) => candidateBucketKey(target),
+        predicate: (target) => Boolean(target.framePath?.length) && isFieldLikeTarget(target),
+    }));
+    for (let index = 0; index < targets.length && orderedIndexes.length < RERANK_CANDIDATE_LIMIT; index += 1) {
+        if (selectedIndexes.has(index)) {
+            continue;
+        }
+        selectedIndexes.add(index);
+        orderedIndexes.push(index);
+    }
+    return orderedIndexes.map((index) => targets[index]).slice(0, RERANK_CANDIDATE_LIMIT);
+}
+function buildCandidateSummary(target, index) {
+    const parts = [`id=c${index + 1}`];
+    const container = target.context?.container;
+    const item = target.context?.item;
+    const group = target.context?.group;
+    const landmark = target.context?.landmark;
+    const layout = target.context?.layout;
+    const visual = target.context?.visual;
+    if (target.kind)
+        parts.push(`kind=${JSON.stringify(target.kind)}`);
+    if (target.role)
+        parts.push(`role=${JSON.stringify(target.role)}`);
+    if (target.label)
+        parts.push(`label=${JSON.stringify(target.label)}`);
+    if (target.interactionHint) {
+        parts.push(`interactionHint=${JSON.stringify(target.interactionHint)}`);
+    }
+    if (target.capability)
+        parts.push(`capability=${JSON.stringify(target.capability)}`);
+    if (target.allowedActions?.length) {
+        parts.push(`allowedActions=${JSON.stringify(target.allowedActions)}`);
+    }
+    if (target.acceptancePolicy) {
+        parts.push(`acceptancePolicy=${JSON.stringify(target.acceptancePolicy)}`);
+    }
+    if (target.controlFamily) {
+        parts.push(`controlFamily=${JSON.stringify(target.controlFamily)}`);
+    }
+    if (target.surfaceRef)
+        parts.push(`surfaceRef=${JSON.stringify(target.surfaceRef)}`);
+    if (typeof target.surfacePriority === 'number') {
+        parts.push(`surfacePriority=${JSON.stringify(target.surfacePriority)}`);
+    }
+    if (target.framePath?.length)
+        parts.push(`framePath=${JSON.stringify(target.framePath)}`);
+    if (target.frameUrl)
+        parts.push(`frameUrl=${JSON.stringify(target.frameUrl)}`);
+    if (target.formSelector)
+        parts.push(`formSelector=${JSON.stringify(target.formSelector)}`);
+    if (target.structure)
+        parts.push(`structure=${JSON.stringify(target.structure)}`);
+    if (target.placeholder)
+        parts.push(`placeholder=${JSON.stringify(target.placeholder)}`);
+    if (target.title)
+        parts.push(`title=${JSON.stringify(target.title)}`);
+    if (target.states)
+        parts.push(`state=${JSON.stringify(target.states)}`);
+    if (item?.kind)
+        parts.push(`itemKind=${JSON.stringify(item.kind)}`);
+    if (item?.label)
+        parts.push(`itemLabel=${JSON.stringify(item.label)}`);
+    if (item?.text)
+        parts.push(`itemText=${JSON.stringify(item.text)}`);
+    if (group?.kind)
+        parts.push(`groupKind=${JSON.stringify(group.kind)}`);
+    if (group?.label)
+        parts.push(`groupLabel=${JSON.stringify(group.label)}`);
+    if (group?.text)
+        parts.push(`groupText=${JSON.stringify(group.text)}`);
+    if (container?.kind)
+        parts.push(`containerKind=${JSON.stringify(container.kind)}`);
+    if (container?.label)
+        parts.push(`containerLabel=${JSON.stringify(container.label)}`);
+    if (container?.text)
+        parts.push(`containerText=${JSON.stringify(container.text)}`);
+    if (landmark?.kind)
+        parts.push(`landmarkKind=${JSON.stringify(landmark.kind)}`);
+    if (landmark?.label)
+        parts.push(`landmarkLabel=${JSON.stringify(landmark.label)}`);
+    if (landmark?.text)
+        parts.push(`landmarkText=${JSON.stringify(landmark.text)}`);
+    if (layout?.lane)
+        parts.push(`lane=${JSON.stringify(layout.lane)}`);
+    if (layout?.band)
+        parts.push(`band=${JSON.stringify(layout.band)}`);
+    if (visual)
+        parts.push(`visual=${JSON.stringify(visual)}`);
+    if (target.context?.hintText)
+        parts.push(`hintText=${JSON.stringify(target.context.hintText)}`);
+    return parts.join(' | ');
+}
+function normalizeCandidateId(value) {
+    return value.trim().toLowerCase();
+}
+export async function rerankDomTargetsForGoal(instruction, targets) {
+    if (targets.length === 0) {
+        return [];
+    }
+    const gateway = resolveAgentpayGatewayConfig();
+    const client = new AgentpayStagehandLlmClient(gateway);
+    const candidates = diversifyCandidates(targets);
+    const prompt = [
+        'You are choosing from already discovered visible actionable candidates on a webpage.',
+        'Select only the candidate IDs that directly satisfy the user goal.',
+        'Use container/landmark/layout/state information to disambiguate similar labels.',
+        'Prefer candidates whose surrounding container clearly corroborates the goal.',
+        'Prefer actionable candidates over scope or informational candidates unless the goal explicitly asks for a container or region.',
+        'Prefer direct visible targets over indirect controls when the direct visible target is present.',
+        'Prefer candidates that belong to an active local surface (dialog/listbox/popover/card container) over unrelated page-root candidates when both are plausible matches.',
+        'For date cells, seat cells, and other structured-grid targets, use row/column/zone/cell metadata and state to distinguish plausible matches.',
+        'When the goal refers to an input field or typed value, prefer the directly editable input/textarea/select over a non-editable wrapper such as a combobox shell.',
+        'When the goal refers to a specific field or control, an exact disabled/readonly match is still relevant. Prefer the exact field over nearby prerequisite controls and use state/context to indicate that it is gated.',
+        'Prefer candidates whose explicit state matches the goal and avoid candidates whose state contradicts the goal.',
+        'Use visual cues only as supporting evidence when they help distinguish active vs muted targets in the same group.',
+        'When the goal is about setting or choosing a form/search/filter value, prefer the primary form control or open picker inside a form/dialog/combobox over lower informational charts, summaries, or content collections that merely mention similar values.',
+        'When the goal asks for a form or a set of related controls, return the direct controls from the relevant form instead of nearby navigation links or surrounding section headings.',
+        'If a secondary chart/list mirrors the same options as a primary form control, do not select the secondary surface unless the goal explicitly asks for that chart/list.',
+        'Avoid navigation, filter, and summary controls when the goal refers to a concrete content item unless the goal explicitly asks for those controls.',
+        'Do not invent IDs. Return an empty list if nothing clearly matches.',
+        '',
+        `Goal: ${instruction}`,
+        '',
+        'Candidates:',
+        ...candidates.map((target, index) => buildCandidateSummary(target, index)),
+    ].join('\n');
+    const result = await client.createChatCompletion({
+        logger: () => { },
+        options: {
+            messages: [{ role: 'user', content: prompt }],
+            response_model: {
+                name: 'Observation',
+                schema: rerankSchema,
+            },
+        },
+    });
+    const selectedIds = new Set(result.data.matches.map((match) => normalizeCandidateId(match.candidateId)));
+    return candidates.filter((_, index) => selectedIds.has(`c${index + 1}`));
+}

package/dist/commands/status.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"status.d.ts","sourceRoot":"","sources":["../../src/commands/status.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,wBAAsB,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAgD5C"}
+{"version":3,"file":"status.d.ts","sourceRoot":"","sources":["../../src/commands/status.ts"],"names":[],"mappings":"AAAA;;GAEG;AAqDH,wBAAsB,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CA8F5C"}