browxai 0.7.0

package/dist/page/plan.js

@@ -0,0 +1,308 @@
+// `plan()` / `execute()` — separate intent capture from dispatch.
+//
+// `plan()` resolves a natural-language query (via the same scoring as `find()`)
+// down to a single bound, serialisable `ActionDescriptor` carrying:
+//   - the action `verb` (`click` / `fill` / `hover` / `press` / `select`),
+//   - the verb's structured `args` (the `value` for `fill`, `key` for `press`, etc.),
+//   - the resolved `ref` (the existing stable-key namespace — NOT a parallel id system),
+//   - the `evidence` (selectorHint, stability, score, top-N candidates) the caller
+//     can audit before deciding to execute,
+//   - an `expiresAt` deadline (epoch-ms).
+//
+// `execute(descriptor)` then re-resolves the ref via `refs.ts` and dispatches the
+// verb's action against the live page. Two structured failure modes:
+//
+//   - `error: "descriptor expired"`     → past `expiresAt`. Caller should re-`plan`.
+//   - `error: "ref no longer resolves"` → the ref is no longer in the registry, or
+//                                          the underlying locator no longer addresses
+//                                          a live element. Caller should re-`plan`.
+//
+// Explicit non-goal — mock dispatch. `execute()` actually runs the action; the
+// value here is *capturing and replaying intent*, not *suppressing effects*.
+// (An action that fakes its effect is a footgun — the agent thinks it ran
+// something it didn't.) For a no-effect-but-want-to-see-the-target dry-run,
+// inspect the returned descriptor's `evidence` block before calling `execute`.
+import { randomUUID } from "node:crypto";
+import { find } from "./find.js";
+import * as actions from "./actions.js";
+/** Action verbs `plan()`/`execute()` understand. Single-target verbs only —
+ *  page-level / multi-step verbs (`navigate`, `scroll`, `wait_for`, `choose_option`)
+ *  are out of scope here; they're either trivially planned (`navigate(url)` doesn't
+ *  need a ranked candidate) or expand into multiple action-window dispatches and
+ *  belong in their own primitives. */
+export const PLAN_VERBS = ["click", "fill", "hover", "press", "select"];
+const DEFAULT_TTL_MS = 60_000;
+const MIN_TTL_MS = 1_000;
+const MAX_TTL_MS = 30 * 60_000;
+/** Validate per-verb args at plan time so a bad descriptor can never reach
+ *  `execute()`. Pure; exported for tests. */
+export function validateVerbArgs(verb, args) {
+    const a = args ?? {};
+    if (verb === "fill") {
+        if (typeof a.value !== "string")
+            return 'plan: verb "fill" requires verbArgs.value (string)';
+    }
+    if (verb === "press") {
+        if (typeof a.key !== "string" || a.key.length === 0)
+            return 'plan: verb "press" requires verbArgs.key (non-empty string)';
+    }
+    if (verb === "select") {
+        if (!Array.isArray(a.values) || a.values.length === 0)
+            return 'plan: verb "select" requires verbArgs.values (non-empty string[])';
+    }
+    if (verb === "click") {
+        if (a.button !== undefined && !["left", "right", "middle"].includes(a.button)) {
+            return `plan: verb "click" — button must be left/right/middle (got ${JSON.stringify(a.button)})`;
+        }
+    }
+    return null;
+}
+/** Clamp a caller-supplied ttl into the sane range. Pure; exported for tests. */
+export function clampTtl(ttlMs) {
+    const v = typeof ttlMs === "number" && Number.isFinite(ttlMs) ? ttlMs : DEFAULT_TTL_MS;
+    return Math.min(Math.max(v, MIN_TTL_MS), MAX_TTL_MS);
+}
+/** Project a `find()` candidate into a descriptor's `evidence` block.
+ *  Pure; exported for tests. */
+export function evidenceFromCandidate(query, picked, alternatives, warnings) {
+    const alts = alternatives.slice(0, 4).map((c) => {
+        const a = { ref: c.ref, role: c.role, score: c.score };
+        if (c.name !== undefined)
+            a.name = c.name;
+        if (c.testId !== undefined)
+            a.testId = c.testId;
+        return a;
+    });
+    const ev = {
+        query,
+        selectorHint: picked.selectorHint,
+        selectorTier: picked.selectorTier,
+        stability: picked.stability,
+        role: picked.role,
+        score: picked.score,
+        actionable: picked.actionable,
+        warnings,
+        alternatives: alts,
+    };
+    if (picked.name !== undefined)
+        ev.name = picked.name;
+    if (picked.testId !== undefined)
+        ev.testId = picked.testId;
+    return ev;
+}
+/** Build a descriptor from a resolved candidate + caller args. Pure; exported
+ *  for tests. */
+export function buildDescriptor(args) {
+    const now = args.now ?? Date.now();
+    return {
+        id: randomUUID(),
+        ref: args.picked.ref,
+        verb: args.verb,
+        args: { ...args.verbArgs },
+        evidence: evidenceFromCandidate(args.query, args.picked, args.alternatives, args.warnings),
+        expiresAt: now + args.ttlMs,
+    };
+}
+/** rough JSON-byte/4 token estimate for a descriptor — matches the
+ *  shape `tokensEstimate` uses across other tools. Pure; exported for tests. */
+export function estimateDescriptorTokens(d) {
+    return Math.ceil(JSON.stringify(d).length / 4);
+}
+/**
+ * `plan()`: rank `query` against the live tree, pick the top candidate,
+ * validate the verb's args, return a serialisable descriptor (no dispatch).
+ *
+ * Returns `{ ok: false }` with structured reasons when:
+ *   - the verb's required args are missing/wrong-shape (validated up front),
+ *   - `find()` returned no candidates (or none above `confidenceFloor`).
+ *
+ * Note: a descriptor IS returned even if the top candidate is non-actionable
+ * (off-screen / disabled). The caller can inspect `evidence.actionable` and
+ * decide; we don't silently refuse to plan against a hidden modal, because
+ * sometimes that's exactly what the caller wants to confirm.
+ */
+export async function plan(page, substrate, refs, opts, 
+/** CDP handle for find()'s visible-rect bbox fast path — chromium only. */
+cdp) {
+    const argError = validateVerbArgs(opts.verb, opts.verbArgs);
+    if (argError) {
+        return { ok: false, error: argError, warnings: [], tokensEstimate: 0 };
+    }
+    const ttlMs = clampTtl(opts.ttlMs);
+    const result = await find(page, substrate, refs, {
+        query: opts.query,
+        testAttributes: opts.testAttributes,
+        contextRef: opts.contextRef,
+        confidenceFloor: opts.confidenceFloor,
+        fallbackHints: opts.fallbackHints,
+        // bound the candidate list so evidence.alternatives stays small (we
+        // only ever return the top + up to 4 alts).
+        maxCandidates: 5,
+    }, cdp);
+    if (result.candidates.length === 0) {
+        return {
+            ok: false,
+            error: `plan: no candidate matched query ${JSON.stringify(opts.query)}`,
+            warnings: result.warnings,
+            tokensEstimate: 0,
+        };
+    }
+    const [picked, ...rest] = result.candidates;
+    if (!picked) {
+        return {
+            ok: false,
+            error: `plan: no candidate matched query ${JSON.stringify(opts.query)}`,
+            warnings: result.warnings,
+            tokensEstimate: 0,
+        };
+    }
+    const descriptor = buildDescriptor({
+        picked,
+        alternatives: rest,
+        query: opts.query,
+        verb: opts.verb,
+        verbArgs: opts.verbArgs ?? {},
+        warnings: result.warnings,
+        ttlMs,
+    });
+    return {
+        ok: true,
+        descriptor,
+        warnings: result.warnings,
+        tokensEstimate: estimateDescriptorTokens(descriptor),
+    };
+}
+/** Static descriptor validation (shape only — does not touch the page).
+ *  Pure; exported for tests. */
+export function validateDescriptor(d) {
+    if (!d || typeof d !== "object")
+        return { ok: false, error: "execute: descriptor must be an object" };
+    const obj = d;
+    if (typeof obj.id !== "string")
+        return { ok: false, error: "execute: descriptor.id missing or not a string" };
+    if (typeof obj.ref !== "string")
+        return { ok: false, error: "execute: descriptor.ref missing or not a string" };
+    if (typeof obj.verb !== "string" || !PLAN_VERBS.includes(obj.verb)) {
+        return { ok: false, error: `execute: descriptor.verb must be one of ${PLAN_VERBS.join("/")}` };
+    }
+    if (typeof obj.expiresAt !== "number")
+        return { ok: false, error: "execute: descriptor.expiresAt missing or not a number" };
+    if (typeof obj.args !== "object" || obj.args === null)
+        return { ok: false, error: "execute: descriptor.args must be an object" };
+    // evidence is informational — present-but-malformed should not block
+    // dispatch, but it must be at least an object so callers can rely on
+    // `descriptor.evidence.selectorHint` being readable when present.
+    if (obj.evidence !== undefined && (typeof obj.evidence !== "object" || obj.evidence === null)) {
+        return { ok: false, error: "execute: descriptor.evidence must be an object when present" };
+    }
+    return { ok: true, descriptor: obj };
+}
+/**
+ * `execute(descriptor)`: dispatch a previously-planned descriptor. Re-resolves
+ * the ref via `refs.ts` (the same stable-key scheme `find()`/`snapshot()` use),
+ * runs the verb's action, and returns the dispatched `ActionResult`.
+ *
+ * Refusal modes (no dispatch happens):
+ *   - `reason: "expired"` — past `expiresAt`.
+ *   - `reason: "ref-gone"` — `refs.has(descriptor.ref)` is false. The agent
+ *                            should re-`plan` against the current snapshot.
+ *   - `reason: "invalid"` — descriptor failed shape validation (bad verb,
+ *                           missing fields, malformed args).
+ *
+ * Capability gating: this function does NOT enforce capabilities itself — the
+ * MCP `execute` handler runs `gateCheck(verb)` (e.g. `gateCheck("click")`)
+ * before calling in, so a descriptor with `verb: "click"` is denied when the
+ * `action` capability is disabled, surfacing the *underlying* capability error
+ * (not a generic "execute denied").
+ */
+/** Pre-flight validation for `execute`: schema, expiry, ref-presence, and
+ *  re-validated verb args. Returns the validated descriptor or a failure
+ *  outcome to return verbatim. */
+function validateForExecute(ctx, rawDescriptor, now) {
+    const validated = validateDescriptor(rawDescriptor);
+    if (!validated.ok) {
+        return {
+            ok: false,
+            outcome: { ok: false, error: validated.error, reason: "invalid", tokensEstimate: 0 },
+        };
+    }
+    const d = validated.descriptor;
+    if (now > d.expiresAt) {
+        return {
+            ok: false,
+            outcome: {
+                ok: false,
+                error: `execute: descriptor expired (${now - d.expiresAt}ms past expiresAt). Re-plan against the current snapshot.`,
+                reason: "expired",
+                tokensEstimate: 0,
+            },
+        };
+    }
+    if (!ctx.refs.has(d.ref)) {
+        return {
+            ok: false,
+            outcome: {
+                ok: false,
+                error: `execute: ref "${d.ref}" no longer in the session's registry — the page likely re-snapshotted ` +
+                    `to a tree where the bound element is absent. Re-plan against the current snapshot.`,
+                reason: "ref-gone",
+                tokensEstimate: 0,
+            },
+        };
+    }
+    // Re-validate verb args at execute time too — a hand-edited descriptor could
+    // have dropped a required arg between plan and execute.
+    const argError = validateVerbArgs(d.verb, d.args);
+    if (argError) {
+        return {
+            ok: false,
+            outcome: { ok: false, error: argError, reason: "invalid", tokensEstimate: 0 },
+        };
+    }
+    return { ok: true, descriptor: d };
+}
+export async function execute(ctx, rawDescriptor, opts = {}) {
+    const now = opts.now ?? Date.now();
+    const pre = validateForExecute(ctx, rawDescriptor, now);
+    if (!pre.ok)
+        return pre.outcome;
+    const d = pre.descriptor;
+    const target = { ref: d.ref };
+    // Forwarded window options. We don't override the underlying action's
+    // recordingHint plumbing here — the server-level handler computes one from
+    // the ref (mirroring how `click`/`fill` are registered).
+    const windowOpts = {
+        ...(opts.mode !== undefined ? { mode: opts.mode } : {}),
+        ...(opts.maxResultTokens !== undefined ? { maxResultTokens: opts.maxResultTokens } : {}),
+        ...(opts.deadlineMs !== undefined ? { deadlineMs: opts.deadlineMs } : {}),
+        ...(opts.deadlineWarning !== undefined ? { deadlineWarning: opts.deadlineWarning } : {}),
+        ...(opts.recordingHint !== undefined ? { recordingHint: opts.recordingHint } : {}),
+    };
+    let result;
+    switch (d.verb) {
+        case "click":
+            result = await actions.click(ctx, {
+                target,
+                ...(d.args.button ? { button: d.args.button } : {}),
+                ...windowOpts,
+            });
+            break;
+        case "fill":
+            result = await actions.fill(ctx, { target, value: d.args.value, ...windowOpts });
+            break;
+        case "hover":
+            result = await actions.hover(ctx, { target, ...windowOpts });
+            break;
+        case "press":
+            result = await actions.press(ctx, { target, key: d.args.key, ...windowOpts });
+            break;
+        case "select":
+            result = await actions.select(ctx, { target, values: d.args.values, ...windowOpts });
+            break;
+    }
+    return {
+        ok: true,
+        result,
+        tokensEstimate: result.tokensEstimate,
+    };
+}

package/dist/page/point_probe.d.ts

@@ -0,0 +1,46 @@
+import type { Page } from "playwright-core";
+export interface ProbedElement {
+    tag: string;
+    id?: string;
+    testId?: string;
+    role?: string;
+    name?: string;
+    classes?: string;
+    pointerEvents: string;
+    visibility: string;
+    display: string;
+    zIndex: string;
+    cursor: string;
+    bbox: {
+        x: number;
+        y: number;
+        width: number;
+        height: number;
+    } | null;
+}
+export interface PointProbeResult {
+    ok: boolean;
+    point: {
+        x: number;
+        y: number;
+    };
+    /** `document.elementsFromPoint(x,y)` top-down, capped at 8. The first entry
+     *  is what a real click at this point would hit. */
+    stack: ProbedElement[];
+    /** nearest scrollable ancestor of the top element (overflow auto/scroll
+     *  with overflowing content), or null. */
+    scrollContainer: ProbedElement | null;
+    /** nearest clickable ancestor of the top element (a/button, role
+     *  button/link, [onclick], [tabindex]), or null — what a click here would
+     *  *semantically* activate even if the literal hit is a child glyph. */
+    clickableAncestor: ProbedElement | null;
+    /** present only when `crop:true` — a small PNG (base64) around the point,
+     *  bounded; off by default to keep results token-cheap. */
+    cropBase64?: string;
+}
+export declare function pointProbe(page: Page, point: {
+    x: number;
+    y: number;
+}, opts?: {
+    crop?: boolean;
+}): Promise<PointProbeResult>;

package/dist/page/point_probe.js

@@ -0,0 +1,99 @@
+// `point_probe` — read-only coordinate target inspection.
+//
+// In canvas / virtualised-timeline / painted UIs the real target isn't a
+// clean accessible element, so an agent driving by `coords` is trusting a
+// screenshot estimate of "what is actually under this point". point_probe
+// answers it deterministically: the full `elementsFromPoint` stack with each
+// layer's identity + the computed properties that decide hit-testing
+// (pointer-events / visibility / z-index / cursor), plus the nearest scroll
+// container and clickable ancestor. No agent JS (fixed server script).
+const MAX_STACK = 8;
+// `page.evaluate(string)` treats the string as an *expression* — a
+// `function(arg){…}` string is never called and args are ignored. So this is
+// an arg-less IIFE with the (numeric, zod-validated) coords interpolated in.
+function buildProbeScript(x, y, max) {
+    return `(() => {
+  var x = ${x}, y = ${y}, MAX = ${max};
+  function summ(el) {
+    if (!el || !el.tagName) return null;
+    var cs = (el.ownerDocument && el.ownerDocument.defaultView)
+      ? el.ownerDocument.defaultView.getComputedStyle(el) : null;
+    var r = el.getBoundingClientRect ? el.getBoundingClientRect() : null;
+    var cls = (typeof el.className === 'string' && el.className) ? el.className.slice(0, 120) : undefined;
+    return {
+      tag: el.tagName.toLowerCase(),
+      id: el.id || undefined,
+      testId: el.getAttribute ? (el.getAttribute('data-testid') || undefined) : undefined,
+      role: el.getAttribute ? (el.getAttribute('role') || undefined) : undefined,
+      name: (el.getAttribute && el.getAttribute('aria-label')) ||
+            (el.textContent ? el.textContent.trim().slice(0, 40) : undefined) || undefined,
+      classes: cls,
+      pointerEvents: cs ? cs.pointerEvents : '',
+      visibility: cs ? cs.visibility : '',
+      display: cs ? cs.display : '',
+      zIndex: cs ? cs.zIndex : '',
+      cursor: cs ? cs.cursor : '',
+      bbox: r ? { x: r.x, y: r.y, width: r.width, height: r.height } : null,
+    };
+  }
+  var els = (document.elementsFromPoint ? document.elementsFromPoint(x, y) : []).slice(0, MAX);
+  var top = els[0] || null;
+  function scrollAncestor(el) {
+    var n = el;
+    while (n && n.tagName) {
+      var cs = getComputedStyle(n);
+      var oy = cs.overflowY, ox = cs.overflowX;
+      var scrolly = (oy === 'auto' || oy === 'scroll') && n.scrollHeight > n.clientHeight;
+      var scrollx = (ox === 'auto' || ox === 'scroll') && n.scrollWidth > n.clientWidth;
+      if (scrolly || scrollx) return n;
+      n = n.parentElement;
+    }
+    return null;
+  }
+  function clickableAncestor(el) {
+    var n = el;
+    while (n && n.tagName) {
+      var t = n.tagName.toLowerCase();
+      var role = n.getAttribute ? n.getAttribute('role') : null;
+      if (t === 'a' || t === 'button' || role === 'button' || role === 'link' ||
+          (n.hasAttribute && (n.hasAttribute('onclick') || n.hasAttribute('tabindex')))) return n;
+      n = n.parentElement;
+    }
+    return null;
+  }
+  return {
+    stack: els.map(summ),
+    scrollContainer: top ? summ(scrollAncestor(top)) : null,
+    clickableAncestor: top ? summ(clickableAncestor(top)) : null,
+  };
+})()`;
+}
+export async function pointProbe(page, point, opts = {}) {
+    const probed = await page.evaluate(buildProbeScript(point.x, point.y, MAX_STACK));
+    const result = {
+        ok: true,
+        point,
+        stack: probed.stack ?? [],
+        scrollContainer: probed.scrollContainer ?? null,
+        clickableAncestor: probed.clickableAncestor ?? null,
+    };
+    if (opts.crop) {
+        try {
+            const half = 40;
+            const buf = await page.screenshot({
+                clip: {
+                    x: Math.max(0, point.x - half),
+                    y: Math.max(0, point.y - half),
+                    width: half * 2,
+                    height: half * 2,
+                },
+                type: "png",
+            });
+            result.cropBase64 = Buffer.from(buf).toString("base64");
+        }
+        catch {
+            /* crop is best-effort — the probe stack is the primary signal */
+        }
+    }
+    return result;
+}

package/dist/page/recording.d.ts

@@ -0,0 +1,67 @@
+import type { DispatchedAction } from "./actionresult.js";
+import type { FindCandidate } from "./find.js";
+export interface RecordedStep {
+    id: string;
+    action: DispatchedAction;
+    url: string;
+    selectorHint?: string;
+    /** Stability of the locator at calibration time.  friend. */
+    stability?: FindCandidate["stability"];
+    /** Optional agent-supplied annotation for the doc emission. */
+    annotation?: {
+        copy: string;
+        arrow?: string;
+        target?: string;
+    };
+    ts: number;
+}
+export declare class Recorder {
+    private steps;
+    private name;
+    private autoCounter;
+    start(flowName: string): {
+        ok: true;
+        name: string;
+    };
+    active(): boolean;
+    /** Read-only access to the recorded trace + the flow name without ending
+     *  the recording. Used by trace-export tools (e.g. the Playwright-script
+     *  exporter) that need to lower the steps to a runnable artefact while the
+     *  recording is still in progress. Returns null when no recording is
+     *  active. */
+    inspect(): {
+        name: string;
+        steps: ReadonlyArray<RecordedStep>;
+    } | null;
+    /** Record an action that just happened. The caller (server.ts action handlers)
+     *  passes the descriptor + the URL it ended at + whatever selectorHint was
+     *  used to resolve the target. Best-effort: if no recording is active, this
+     *  is a no-op. */
+    record(descriptor: DispatchedAction, url: string, hint?: {
+        selectorHint: string;
+        stability?: FindCandidate["stability"];
+    }): void;
+    /** Attach an annotation to the most-recent step (or by id). */
+    annotate(args: {
+        stepId?: string;
+        copy: string;
+        arrow?: string;
+        target?: string;
+    }): {
+        ok: boolean;
+        error?: string;
+    };
+    /** End the recording. Returns the YAML draft + the step count. */
+    end(): {
+        name: string;
+        yaml: string;
+        stepCount: number;
+    };
+    private suggestId;
+    /** Render the recording as YAML. Minimal site-docs-flavoured shape; consumers
+     *  can post-process. We don't pull in a YAML library to keep deps lean — the
+     *  output is small + predictable. */
+    private toYaml;
+    private collectLocators;
+    private locatorNameFor;
+}