browxai 0.7.0

package/dist/page/fill-form.js

@@ -0,0 +1,261 @@
+// Multi-field form-fill primitive. Fills N field/value pairs atomically in
+// one action window, with an optional final `submit` click — replaces the
+// fill/fill/fill/click round-trip pattern with a single dispatch.
+//
+// Semantics that callers can rely on:
+//
+//   1. **Atomic pre-resolution.** Every field's target (ref / selector /
+//      coords) is resolved BEFORE any DOM write. If any target fails to
+//      resolve, the call returns `ok:false` with a structured
+//      `fieldResolution` block listing every field's outcome — and *no*
+//      partial fills land. The agent gets a single "this form isn't ready"
+//      signal instead of a half-filled form that's hard to recover from.
+//
+//   2. **Sequential dispatch.** Once resolution succeeds, fields are filled
+//      in array order. Each fill goes through the same Playwright `.fill()`
+//      path the single-field primitive uses; per-field secrets materialisation
+//      and post-probe masking compose unchanged. The first per-field error
+//      stops the loop; later fields are reported as `skipped` so the agent
+//      can see how far the dispatch got.
+//
+//   3. **Per-field probes.** The result carries `elements: ElementProbe[]`
+//      in dispatch order — the multi-target variant of the single-field
+//      `element` probe. When a submit is supplied, `element` (singular) is
+//      the submit's post-click probe so single-target consumers don't have
+//      to feature-detect.
+//
+//   4. **One action window.** Navigation / network / structure / console /
+//      pageErrors / snapshotDelta are captured ACROSS the whole sequence,
+//      not per field. The agent sees "did the form submit succeed?" at the
+//      same envelope level as a one-field fill — one round-trip, one diff.
+import { runInActionWindow, } from "./actionresult.js";
+import { locatorFor } from "./locator.js";
+import { materialiseValue, maskProbe, preProbe, probe } from "./actions.js";
+const DEFAULT_TIMEOUT_MS = 5_000;
+/** Pure helper: render a one-line summary of an ActionTarget so the
+ *  resolution-failure envelope is human-readable. Exported for unit tests. */
+export function summariseTarget(t) {
+    if (t.ref)
+        return `ref=${t.ref}`;
+    if (t.selector) {
+        return t.contextRef ? `selector=${t.selector} (in ${t.contextRef})` : `selector=${t.selector}`;
+    }
+    if (t.coords)
+        return `coords=${t.coords.x},${t.coords.y}`;
+    return "<empty target>";
+}
+/** Pure helper: validate the args shape before touching the page. Throws
+ *  on structural problems (empty fields, coord targets for fill — which
+ *  Playwright can't drive). Exported for unit tests. */
+export function validateFillFormArgs(args) {
+    if (!Array.isArray(args.fields) || args.fields.length === 0) {
+        throw new Error("fill_form: `fields` must be a non-empty array of {target, value}");
+    }
+    for (let i = 0; i < args.fields.length; i++) {
+        const f = args.fields[i];
+        if (!f || typeof f !== "object") {
+            throw new Error(`fill_form: fields[${i}] is not an object`);
+        }
+        if (typeof f.value !== "string") {
+            throw new Error(`fill_form: fields[${i}].value must be a string`);
+        }
+        if (!f.target) {
+            throw new Error(`fill_form: fields[${i}].target is required`);
+        }
+        if (f.target.coords) {
+            throw new Error(`fill_form: fields[${i}] uses a coords target — fill requires a real input/textarea element, ` +
+                `so pass ref/selector/named. coords stays for click/hover.`);
+        }
+    }
+}
+/**
+ * Resolve every field's locator and confirm it actually matches a node in
+ * the DOM. Returns either `{ ok: true, locators, … }` (every field resolved
+ * to ≥1 matching node) or `{ ok: false, resolutions }` (one or more misses;
+ * caller emits a structured failure and runs NO writes).
+ *
+ * Exported for unit tests — the atomic-resolution invariant is the most
+ * important part of the primitive, and we test it directly.
+ */
+/** Resolve one field/submit target into a locator + resolution record. `index`
+ *  is the field index, or -1 for the submit target (which drives the zero-node
+ *  error wording, preserving the prior byte-identical messages). */
+async function resolveOneTarget(page, refs, target, index, targetSummary) {
+    const zeroNodeError = index === -1
+        ? "submit target resolved to zero DOM nodes"
+        : "target resolved to zero DOM nodes — element no longer present";
+    try {
+        const loc = locatorFor(page, refs, target);
+        // count() is the cheap "does this resolve" gate — surfaces "ref exists in
+        // registry but no longer in the DOM" before we start typing.
+        const count = await loc.count().catch(() => 0);
+        if (count === 0) {
+            return { resolution: { index, targetSummary, ok: false, error: zeroNodeError } };
+        }
+        return { locator: loc, resolution: { index, targetSummary, ok: true } };
+    }
+    catch (err) {
+        return {
+            resolution: {
+                index,
+                targetSummary,
+                ok: false,
+                error: err instanceof Error ? err.message : String(err),
+            },
+        };
+    }
+}
+export async function resolveFieldsAtomically(page, refs, fields, submit) {
+    const resolutions = [];
+    const locators = [];
+    for (let i = 0; i < fields.length; i++) {
+        const r = await resolveOneTarget(page, refs, fields[i].target, i, summariseTarget(fields[i].target));
+        resolutions.push(r.resolution);
+        if (r.locator)
+            locators.push(r.locator);
+    }
+    let submitLocator;
+    let submitResolution;
+    if (submit) {
+        const r = await resolveOneTarget(page, refs, submit, -1, `submit ${summariseTarget(submit)}`);
+        submitLocator = r.locator;
+        submitResolution = r.resolution;
+    }
+    const allFieldsOk = resolutions.every((r) => r.ok);
+    const submitOk = !submitResolution || submitResolution.ok;
+    if (!allFieldsOk || !submitOk) {
+        return { ok: false, resolutions, submitResolution };
+    }
+    return { ok: true, locators, submitLocator, resolutions };
+}
+/**
+ * Build the dispatched-action descriptor for the result envelope. The
+ * value field carries a compact "n field(s) [+submit]" tag so transcripts
+ * stay greppable without dumping every field's value in plaintext.
+ */
+function descriptorFor(args) {
+    const n = args.fields.length;
+    const suffix = args.submit ? " +submit" : "";
+    return { type: "fillForm", value: `${n} field${n === 1 ? "" : "s"}${suffix}` };
+}
+/**
+ * Compose-with-existing-fill multi-field form primitive. See file header
+ * for the contract. Failure envelopes mirror the action-window shape so
+ * agents see the same `ok / action / navigation / structure / console …`
+ * surface they get from every other action tool.
+ */
+/** Build the empty-window `ok:false` envelope for an atomic pre-flight rejection
+ *  (no fields typed). Mirrors the action-window shape so callers see the same
+ *  surface every action tool returns. */
+function fillFormFailure(ctx, descriptor, error, fieldResolution) {
+    const url = ctx.page.url();
+    return {
+        ok: false,
+        action: descriptor,
+        navigation: { changed: false, from: url, to: url, kind: null },
+        structure: { appeared: [], removed: [], newTabs: [] },
+        console: { errors: [], warnings: 0 },
+        pageErrors: [],
+        network: { summary: { total: 0, byType: {}, failed: 0 } },
+        tokensEstimate: 0,
+        warnings: [],
+        error,
+        fieldResolution,
+    };
+}
+/** Materialise every field's value (resolving registered-secret aliases) before
+ *  any write. Returns the materialised values or a pre-flight failure (a
+ *  rejection on field 3 mustn't leave 0..2 filled). */
+function materialiseFields(ctx, args, descriptor, fieldResolution) {
+    const materialised = [];
+    for (let i = 0; i < args.fields.length; i++) {
+        const f = args.fields[i];
+        const mat = materialiseValue(ctx, f.value);
+        if (!mat.ok) {
+            return {
+                ok: false,
+                failure: fillFormFailure(ctx, descriptor, `fill_form: secrets materialisation rejected fields[${i}]: ${mat.error}`, fieldResolution),
+            };
+        }
+        materialised.push({
+            value: mat.value,
+            alias: mat.alias,
+            descriptorValue: mat.alias ? `<${mat.alias}>` : f.value,
+        });
+    }
+    return { ok: true, materialised };
+}
+export async function fillForm(ctx, args) {
+    validateFillFormArgs(args);
+    const descriptor = descriptorFor(args);
+    // Atomic pre-resolution happens BEFORE we open the action window — a
+    // resolution failure shouldn't pay for a network tap + a11y pre-tree.
+    const resolution = await resolveFieldsAtomically(ctx.page, ctx.refs, args.fields, args.submit);
+    if (!resolution.ok) {
+        const allResolutions = [
+            ...resolution.resolutions,
+            ...(resolution.submitResolution ? [resolution.submitResolution] : []),
+        ];
+        const missedSummaries = allResolutions
+            .filter((r) => !r.ok)
+            .map((r) => `[${r.index}] ${r.targetSummary}: ${r.error}`)
+            .join("; ");
+        return fillFormFailure(ctx, descriptor, `fill_form: atomic pre-resolution rejected the call — no fields were typed. Misses: ${missedSummaries}`, allResolutions);
+    }
+    // Pre-validate secrets materialisation for every field too (same "fail
+    // atomically before writing" posture).
+    const mat = materialiseFields(ctx, args, descriptor, resolution.resolutions);
+    if (!mat.ok)
+        return mat.failure;
+    const materialised = mat.materialised;
+    // ---- single action window across the whole sequence ----
+    // The body returns the *final* probe (submit's post-click probe if a
+    // submit was supplied; else the last field's post-fill probe). Per-field
+    // probes accumulate on the shared `perField` array and are attached to
+    // the result on the way out so callers see them under `elements`.
+    const perField = [];
+    let fillFailure;
+    const result = await runInActionWindow(ctx, descriptor, args, async () => {
+        let finalProbe;
+        for (let i = 0; i < resolution.locators.length; i++) {
+            const loc = resolution.locators[i];
+            const m = materialised[i];
+            const target = args.fields[i].target;
+            try {
+                const pre = await preProbe(loc);
+                await loc.fill(m.value, { timeout: args.deadlineMs ?? DEFAULT_TIMEOUT_MS });
+                const probed = await probe(loc, target, m.descriptorValue, pre);
+                const masked = maskProbe(probed, ctx);
+                perField.push(masked);
+                finalProbe = masked;
+            }
+            catch (err) {
+                // Mid-loop fill failure. Record the skipped tail so the agent can
+                // see how far the dispatch got, then rethrow — the action-window
+                // catches it and surfaces the structured `ok:false` envelope.
+                const skipped = [];
+                for (let j = i + 1; j < resolution.locators.length; j++)
+                    skipped.push(j);
+                fillFailure = { atIndex: i, skipped };
+                // Push a stub probe so per-field positions align with the input
+                // array — the failing index carries `stillAttached:false` so the
+                // agent doesn't have to count.
+                perField.push({ ref: target.ref, stillAttached: false });
+                throw err;
+            }
+        }
+        if (resolution.submitLocator) {
+            const pre = await preProbe(resolution.submitLocator);
+            await resolution.submitLocator.click({ timeout: args.deadlineMs ?? DEFAULT_TIMEOUT_MS });
+            finalProbe = await probe(resolution.submitLocator, args.submit, undefined, pre);
+        }
+        return finalProbe;
+    });
+    // Attach per-field probes + fillFailure to the result.
+    const out = result;
+    if (perField.length > 0)
+        out.elements = perField;
+    if (fillFailure)
+        out.fillFailure = fillFailure;
+    return out;
+}

package/dist/page/find.d.ts

@@ -0,0 +1,158 @@
+import type { CDPSession, Frame, Page } from "playwright-core";
+import { type A11yNode, type StructuralContext } from "./a11y.js";
+import type { RefRegistry } from "./refs.js";
+import type { SnapshotSubstrate } from "./snapshot-substrate.js";
+import { type VisibleRect } from "./bbox.js";
+import type { FeedbackMemory } from "./learning.js";
+export interface FindCandidate {
+    ref: string;
+    role: string;
+    name?: string;
+    testId?: string;
+    /** "high" = tier-1 testid, "medium" = role+name, "low" = tier 4-5 fallback. */
+    stability: "high" | "medium" | "low";
+    /** Concrete selector string a consumer can transcribe into a flow-file. Disambiguated
+     *  with `:visible` / `:nth-match(..., 1)` when the bare hint matched multiple DOM
+     *  nodes (). */
+    selectorHint: string;
+    /** Which preference-order tier produced the hint (1–5). */
+    selectorTier: 1 | 2 | 3 | 4 | 5;
+    /** Visible-rect bbox; null when fully clipped. */
+    bbox: VisibleRect | null;
+    /** True when the element is fully clipped (bbox is null). */
+    clipped: boolean;
+    /** whether the element can be acted on right now. `true` = visible
+     *  + enabled + on-screen. `"disabled"` / `"off-screen"` / `"covered"` describe *why*
+     *  if not. Lets a calibration agent reject `<input disabled>`-shaped halts at
+     *  write-time instead of at run-time. */
+    actionable: true | "disabled" | "off-screen" | "covered";
+    /** Internal score — higher = better match for the query. */
+    score: number;
+    /** structural neighbourhood when this candidate sits inside a repeated
+     *  container (table row, listitem, repeated card). Lets the caller filter
+     *  by row / column without re-walking the snapshot. Absent when the
+     *  candidate isn't in a recognised repeated structure. */
+    context?: StructuralContext;
+}
+export interface FindOptions {
+    query: string;
+    maxCandidates?: number;
+    /** Configured test-attribute list (sourced from BROWX_TEST_ATTRIBUTES). */
+    testAttributes: string[];
+    /** emit a `warnings: ["no candidate scored confidently…"]` block
+     *  on the result when no top candidate exceeds this score. Default 0 (off). */
+    confidenceFloor?: number;
+    /** limit ranking to descendants of this ref (from a prior
+     *  snapshot/find). "The seconds input *under* the AI Voiceover panel" without
+     *  encoding the relationship in natural language. Ignored if the ref isn't in
+     *  the current snapshot. */
+    contextRef?: string;
+    /**  learned ranking: prior session feedback applied as a per-candidate
+     *  score bonus. Skip / null = no learning bonus. */
+    feedback?: FeedbackMemory;
+    /** which fallback tools to *name* in the "no visible candidate"
+     *  warning. Capability-aware so we never point an agent at a disabled tool
+     *  (`coords` needs `action`; `eval_js` needs `eval`). */
+    fallbackHints?: {
+        coords: boolean;
+        evalJs: boolean;
+    };
+    /** drop non-actionable candidates (off-screen / clipped / covered /
+     *  disabled) entirely instead of ranking them last. A confident hidden
+     *  hit still lures agents into coordinate fallbacks despite the warning;
+     *  `visibleOnly` returns an empty list + the same warning rather than a
+     *  misleading hit. Default false (hidden candidates kept, ranked last). */
+    visibleOnly?: boolean;
+    /** when set, scope ranking + ref-binding to this child Frame.
+     *  Refs minted are bound to the frame on the registry so subsequent
+     *  actions land inside the iframe. The bbox/actionable probes resolve
+     *  via the frame's own locator surface. Omitted = main frame (existing
+     *  behaviour, byte-identical). */
+    frame?: Frame;
+    /** stable frame ID of `frame`, used for ref namespacing in the
+     *  registry and for the snapshot warning. Required when `frame` is set. */
+    frameId?: string;
+    /** shadow DOM piercing.
+     *  - `undefined` (default) — preserves pre-v0.5.0 behaviour. Playwright's
+     *    a11y tree auto-pierces open shadow roots; the DOM-walk fallback does
+     *    not recurse into shadow content.
+     *  - `"open"` — additionally have the DOM-walk fallback recurse through
+     *    every reachable open shadow root.
+     *  - `"closed"` — open-walk + a CDP `pierce:true` pass that surfaces
+     *    elements inside CLOSED shadow roots. Best-effort: when CDP refuses
+     *    the pierce call (older Chromium, attached-mode quirks), falls back
+     *    to open-only and the result carries a warning. Closed-shadow
+     *    candidates carry a warning of their own — they're inspectable
+     *    evidence, not actionable targets (Playwright's locator engine
+     *    cannot reach them). Closed-shadow CDP harvesting only runs on the
+     *    main frame; in a frame-scoped find, `"closed"` degrades to `"open"`.
+     *  - `false` — neither path recurses into shadow content. */
+    pierce?: "open" | "closed" | false;
+}
+export interface FindResult {
+    candidates: FindCandidate[];
+    warnings: string[];
+}
+export declare function find(page: Page | null, substrate: SnapshotSubstrate, refs: RefRegistry, opts: FindOptions, 
+/** Raw CDP handle for the visible-rect bbox fast path — present only on
+ *  chromium (where a11y nodes carry `backendDOMNodeId`). Off Chromium the
+ *  walker mints no `backendDOMNodeId`, so this is unused and the portable
+ *  `locatorBoundingBox` fallback computes the box. Optional so the engine
+ *  type never enters the find tool path. */
+cdp?: CDPSession): Promise<FindResult>;
+/**
+ * Stable-partition candidates: actionable ones first (preserving score order),
+ * non-actionable (off-screen / clipped / covered / disabled) last — so a
+ * slightly-lower-scored *visible* match outranks a high-scored hidden modal.
+ * `visibleOnly` drops the hidden tier entirely: an empty result + the
+ * "no visible candidate" warning is safer than a confident hidden hit the
+ * agent will chase into a coordinate fallback.
+ *
+ * Within the actionable tier, a second stable partition demotes
+ * non-interactive structural/layout containers below interactive controls,
+ * but *only* when at least one actionable interactive candidate exists — an
+ * aliased query ("the X panel in the right rail") otherwise lets the
+ * enclosing wrapper outrank the button/tab the agent actually wants. If no
+ * actionable interactive candidate matched, containers are left in place
+ * (they may be the best available target). Pure; exported for tests.
+ */
+export declare function rankByVisibility(candidates: FindCandidate[], visibleOnly: boolean): {
+    ranked: FindCandidate[];
+    visibleCount: number;
+};
+/**
+ * the "all candidates off-screen → probably the wrong match" warning.
+ * Capability-aware — only names a fallback tool the caller actually has
+ * enabled (`coords` ⇐ `action`, `eval_js` ⇐ `eval`). Pure; exported for tests.
+ */
+export declare function noVisibleCandidateWarning(count: number, fallbackHints?: {
+    coords: boolean;
+    evalJs: boolean;
+}): string;
+export declare function scoreNode(node: A11yNode, q: string, qTokens: string[]): number;
+/**
+ * The five-tier preference order from :
+ *   1. `[<test-attr>="…"]`           → stability "high"  (any configured test-attribute)
+ *   2. role + accessible name        → stability "medium"
+ *   3. stable text on stable role    → covered by tier 2 (the DOM-walk's nameFor()
+ *                                       computes name from aria-label / labelledby /
+ *                                       textContent in that order, so a `<button>Submit</button>`
+ *                                       already gets `role=button[name="Submit"]` via tier 2)
+ *   4. structural (#id, semantic)    → stability "low"  (id present + id-shaped stable)
+ *   5. positional (last resort)      → stability "low"
+ *
+ *  update: tier 4 now fires when the node has an HTML `id` attribute that
+ * looks stable (not a numeric/UUID content-keyed id). The id-stability heuristic:
+ * reject pure-numeric (`123`), short numeric+letter combos that look generated
+ * (e.g. `mui-1234`), or strings matching common content-keyed shapes. Anything
+ * with two or more `-`/`_`-separated word segments is treated as stable.
+ */
+export declare function buildSelectorHint(node: Pick<A11yNode, "role" | "name" | "testId" | "testIdAttr" | "id">): {
+    hint: string;
+    tier: 1 | 2 | 3 | 4 | 5;
+    stability: FindCandidate["stability"];
+};
+/** Heuristic: is this HTML `id` value likely to survive across page reloads?
+ *  Rejects content-keyed shapes (pure-numeric, MUI-generated `mui-N`, UUID-shaped).
+ *  Accepts ids with two or more word segments separated by `-`/`_`/`:`. */
+export declare function isLikelyStableId(id: string): boolean;