browxai 0.7.0

package/dist/tools/read-observe-buffer-tools.js

@@ -0,0 +1,385 @@
+import { inspectElement } from "../page/inspect.js";
+import { generateLocator } from "../page/generate-locator.js";
+import { watchWindow } from "../page/watch.js";
+import { pointProbe } from "../page/point_probe.js";
+import { sampleMetric, ELEMENT_METRICS } from "../page/sample.js";
+import { withDeadline } from "../util/deadline.js";
+import { estimateTokens } from "../util/tokens.js";
+import { REF_OR_SELECTOR, SESSION_ARG, TIMEOUT_ARG } from "./schemas.js";
+/**
+ * Read / observe — buffer reads + element diagnostics. The session ring-buffer
+ * reads (console_read / network_read / ws_read / network_body), the DOM-metric
+ * sampler + window watcher, element inspection / locator generation / point
+ * probing, and the gated `eval_js` escape hatch. Registered through the shared
+ * `ToolHost` seam.
+ */
+export function registerReadObserveBufferTools(host) {
+    const { z, register, gateCheck, entryFor, asTarget, scriptFor, ctxFor, cfgActionTimeout, actionTimeout, egressFor, caps, } = host;
+    register("console_read", {
+        capability: "read",
+        batchable: true,
+        description: "Recent console messages from the page (ring buffer).",
+        inputSchema: { limit: z.number().int().positive().max(500).optional(), ...SESSION_ARG },
+    }, async ({ limit, session }) => {
+        const g = gateCheck("console_read");
+        if (g)
+            return g;
+        const e = await entryFor(session);
+        const rows = e.console.recent(limit ?? 50);
+        return { content: [{ type: "text", text: JSON.stringify(rows, null, 2) }] };
+    });
+    register("network_read", {
+        capability: "read",
+        batchable: true,
+        description: "Session-wide ring buffer of recent network requests (500 most recent; oldest evicted on overflow). For per-action attribution use `ActionResult.network` from any action tool — that's the primary surface. This is the 'what happened across the session' view; useful when an XHR isn't tied to a specific action you just ran. Noise types (Image/Font/Stylesheet/Media/beacons) folded into `summary.byType.other`.",
+        inputSchema: { limit: z.number().int().positive().max(500).optional(), ...SESSION_ARG },
+    }, async ({ limit, session }) => {
+        const g = gateCheck("network_read");
+        if (g)
+            return g;
+        const e = await entryFor(session);
+        const result = e.network.recent(limit ?? 50);
+        return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
+    });
+    register("sample", {
+        capability: "read",
+        batchable: true,
+        description: "sample a DOM metric over a window and return the time series — jank / CLS / scroll-drift QA. `metric` is a **fixed enum** (no agent-supplied JS — that's `eval_js`, gated). With a `ref`/`selector`/`named` target: `scrollTop`/`scrollLeft`/`scrollHeight`/`scrollWidth`/`clientWidth`/`clientHeight`/`bboxX`/`bboxY`/`bboxWidth`/`bboxHeight`. Without a target: the document scroller (`bbox*` is rejected — needs an element). `everyFrame:true` uses requestAnimationFrame; else `intervalMs` (default 100, min 16). Returns `{ metric, scope, durationMs, mode, count, series:[{tMs,value}], truncated? }`. Caps: 30 s, 2000 points. Read-only (`read`).",
+        inputSchema: {
+            ...REF_OR_SELECTOR,
+            metric: z.enum(ELEMENT_METRICS).describe("Fixed metric to sample."),
+            durationMs: z.number().int().positive().max(30_000).describe("Window length (ms, ≤30000)."),
+            everyFrame: z
+                .boolean()
+                .optional()
+                .describe("Sample every animation frame (rAF). Default false → fixed interval."),
+            intervalMs: z
+                .number()
+                .int()
+                .positive()
+                .max(5000)
+                .optional()
+                .describe("Sampling interval (ms, default 100, min 16). Ignored when everyFrame:true."),
+            summary: z
+                .boolean()
+                .optional()
+                .describe("Series-omission control; the reduced summary ({count,min,max,first,last,distinctCount,firstChangeTMs}) is ALWAYS returned. true=omit the full series; false=always include it; omit this arg=auto (the series is dropped for large windows >300 points, with `autoSummarised:true` on the result — re-request with summary:false for the raw set)."),
+            ...SESSION_ARG,
+        },
+    }, async (args) => {
+        const g = gateCheck("sample");
+        if (g)
+            return g;
+        const e = await entryFor(args.session);
+        const hasTarget = !!(args.ref || args.selector || args.named || args.coords);
+        const target = hasTarget ? asTarget(args, "sample", e.refs) : undefined;
+        if (target && "coords" in target) {
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: JSON.stringify({
+                            error: "sample: coords targets unsupported — use a ref/selector/named element, or omit target for the window",
+                        }, null, 2),
+                    },
+                ],
+            };
+        }
+        try {
+            const result = await sampleMetric(e.session.page(), e.refs, {
+                target,
+                metric: args.metric,
+                durationMs: args.durationMs,
+                everyFrame: args.everyFrame,
+                intervalMs: args.intervalMs,
+                summary: args.summary,
+            });
+            return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
+        }
+        catch (err) {
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: JSON.stringify({ error: err instanceof Error ? err.message : String(err) }, null, 2),
+                    },
+                ],
+            };
+        }
+    });
+    register("watch", {
+        capability: "read",
+        batchable: true,
+        description: "observe a fixed time window with NO driving action. Samples top-level transient surfaces (dialog/alert/status/toast/tooltip/log) across the window so a region that appears AND disappears inside it is caught (endpoint-only diffs miss it) — double-fire toasts, flash-of-content, 'notification never broadcast'. Returns `{ durationMs, samples, regions:[{ role, name, ref, appearedAtMs, disappearedAtMs }], console, network, wsFrames }`. Read-only (`read`). Caps at 60s.",
+        inputSchema: {
+            durationMs: z.number().int().positive().max(60_000).describe("Window length (ms, ≤60000)."),
+            sampleMs: z
+                .number()
+                .int()
+                .positive()
+                .max(5000)
+                .optional()
+                .describe("Sampling interval (ms, default 250, min 50)."),
+            ...SESSION_ARG,
+        },
+    }, async ({ durationMs, sampleMs, session }) => {
+        const g = gateCheck("watch");
+        if (g)
+            return g;
+        const e = await entryFor(session);
+        const result = await watchWindow(ctxFor(e), { durationMs, sampleMs });
+        // Egress sink — the NetworkTap inside `watchWindow` already saw the
+        // secrets registry (via `ctx.secrets`) and sanitised URLs / mutation
+        // responseShape keys. The remaining channel that can echo a literal
+        // value is `regions[].name` (a11y node names — e.g. a status-region
+        // whose visible text reads back the just-filled token). Deep-mask
+        // the whole result so any future string leaf is also covered. Routed
+        // through the injected egress chokepoint (RFC 0004 P3 / D4) — byte-identical
+        // to the prior `caps.enabled.has("secrets") ? e.secrets.applyMaskDeep(...) : ...`.
+        const masked = egressFor(e).maskDeep(result);
+        return { content: [{ type: "text", text: JSON.stringify(masked, null, 2) }] };
+    });
+    register("inspect", {
+        capability: "read",
+        batchable: true,
+        description: "read an element's whitelisted computed styles + box + overflow/clip state. The layout-break / control-state verification primitive — confirm `cursor: not-allowed` vs `wait`, a flex row's `childCount`, a label that overflows (`overflowing.y`), `display:none`/`visibility:hidden`. Returns `{ found, box, styles, overflowing:{x,y}, visible, childCount }`. Read-only (capability `read`); distinct from `find()` (ranking) and `text_search` (presence). Coords targets aren't supported (no element to resolve).",
+        inputSchema: {
+            ...REF_OR_SELECTOR,
+            styles: z
+                .array(z.string())
+                .optional()
+                .describe('Extra computed-style property names to include beyond the default set (camelCase, e.g. "borderBottomWidth").'),
+            ...SESSION_ARG,
+        },
+    }, async (args) => {
+        const g = gateCheck("inspect");
+        if (g)
+            return g;
+        const e = await entryFor(args.session);
+        const target = asTarget(args, "inspect", e.refs);
+        if ("coords" in target) {
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: JSON.stringify({ found: false, error: "inspect requires ref/selector/named, not coords" }, null, 2),
+                    },
+                ],
+            };
+        }
+        const { locatorFor } = await import("../page/locator.js");
+        const loc = locatorFor(e.session.page(), e.refs, target);
+        let result;
+        try {
+            result = await withDeadline(inspectElement(loc, args.styles ?? []), cfgActionTimeout(), "inspect");
+        }
+        catch (err) {
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: JSON.stringify({ found: false, error: err instanceof Error ? err.message : String(err) }, null, 2),
+                    },
+                ],
+            };
+        }
+        // Egress sink — `styles.content` / `background-image: url(...)` can echo
+        // a registered real-value rendered into the computed-style stream.
+        // Low-risk channel (the reviewer flagged as NIT) but the masking layer
+        // is cheap; pin the invariant per-sink.
+        const maskedInspect = egressFor(e).maskDeep(result);
+        return { content: [{ type: "text", text: JSON.stringify(maskedInspect, null, 2) }] };
+    });
+    register("generate_locator", {
+        capability: "read",
+        batchable: true,
+        description: "Convert a session-internal `ref` (from snapshot()/find()) into a Playwright-string locator expression an adopter can paste into a `.spec.ts` — the bridge between agent-driven exploration and a deterministic regression suite. Returns `{ ok, playwright, stability, components }` (or `{ ok:false, failure:{kind:\"ref-not-found\"} }` when the ref isn't in this session's registry — no throw). `playwright` is a real Playwright expression rooted on `page` (e.g. `page.getByRole('button', { name: 'Save' })`, `page.getByTestId('save-btn')`, `page.locator('main > table > tbody > tr:nth-child(4)')`). `stability` is the same per-tier label `find()` emits (high = testid OR role+name; medium = stable structural / text on stable role; low = positional / role-only). `components` is the structured breakdown of the parts the string is built from — adopters who want to compose their own locator (chain `.filter()`, combine two kinds) can read this without re-parsing the string. Read-only; no new capability — reuses `read`.",
+        inputSchema: {
+            ref: z.string().describe("Stable `eN` ref from a prior snapshot()/find()/plan() result."),
+            ...SESSION_ARG,
+        },
+    }, async ({ ref, session }) => {
+        const g = gateCheck("generate_locator");
+        if (g)
+            return g;
+        const e = await entryFor(session);
+        const result = generateLocator(ref, (r) => e.refs.locatorOf(r));
+        // Secrets masking: the emitted `playwright` string + `components`
+        // values can echo a real `name` / `testId` that was registered via the
+        // secrets registry. Same exposure class as `find()`'s `selectorHint`
+        // and `inspect`'s stringly outputs — mask through the per-session
+        // registry on egress (the injected chokepoint — RFC 0004 P3 / D4).
+        const masked = egressFor(e).maskDeep(result);
+        const tokensEstimate = estimateTokens(JSON.stringify(masked));
+        return {
+            content: [
+                { type: "text", text: JSON.stringify({ ...masked, tokensEstimate }, null, 2) },
+            ],
+        };
+    });
+    register("point_probe", {
+        capability: "read",
+        description: "Read-only: what is actually under a viewport coordinate. Returns the full `document.elementsFromPoint` stack (top-down, first = what a real click hits), each layer's tag/id/testId/role/name/classes + computed pointer-events/visibility/display/z-index/cursor + bbox, plus the nearest scroll container and nearest clickable ancestor of the top element. The coordinate-target verifier for canvas / virtualised-timeline / painted UIs where the target isn't a clean accessible element — prove a coordinate hits the intended layer before driving `click({coords})` instead of trusting a screenshot estimate. `crop:true` adds a small bounded PNG around the point (off by default — token-cheap). No agent JS.",
+        inputSchema: {
+            coords: z.object({ x: z.number(), y: z.number() }).describe("Viewport CSS px."),
+            crop: z
+                .boolean()
+                .optional()
+                .describe("Default false. Include a small (80×80) PNG crop (base64) around the point."),
+            ...SESSION_ARG,
+        },
+    }, async ({ coords, crop, session }) => {
+        const g = gateCheck("point_probe");
+        if (g)
+            return g;
+        const e = await entryFor(session);
+        try {
+            const result = await withDeadline(pointProbe(e.session.page(), coords, { crop }), cfgActionTimeout(), "point_probe");
+            // Egress sink — `point_probe.text` / `ancestorText` slice the
+            // textContent of the element-under-point + nearest clickable ancestor.
+            // Same exposure class as snapshot/find name fields; mask through the
+            // session registry before serialising.
+            const maskedProbe = egressFor(e).maskDeep(result);
+            return { content: [{ type: "text", text: JSON.stringify(maskedProbe, null, 2) }] };
+        }
+        catch (err) {
+            // structured failure — coordinate + page URL for triage.
+            let url = "";
+            try {
+                url = e.session.page().url();
+            }
+            catch {
+                /* page gone */
+            }
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: JSON.stringify({
+                            ok: false,
+                            point: coords,
+                            url,
+                            error: err instanceof Error ? err.message : String(err),
+                        }, null, 2),
+                    },
+                ],
+            };
+        }
+    });
+    register("network_body", {
+        capability: "network-body",
+        batchable: true,
+        description: "fetch a full response body by `requestId` (from `network_read` / `ActionResult.network.requests[].requestId`). **Gated behind the off-by-default `network-body` capability** — full bodies can carry PII / auth tokens; 's `responseShape` (keys only) is the safe default. Bounded (256 KB, `truncated:true` past that). Best-effort: the renderer discards bodies fast — fetch right after the request, not retained across navigations. Pairs with for realtime payload assertions.",
+        inputSchema: {
+            requestId: z
+                .string()
+                .describe("CDP request id from network_read / ActionResult.network.requests[].requestId."),
+            ...SESSION_ARG,
+        },
+    }, async ({ requestId, session }) => {
+        const g = gateCheck("network_body");
+        if (g)
+            return g;
+        const e = await entryFor(session);
+        // secrets masking: a full response body routinely echoes auth tokens
+        // and session blobs. Pass the per-session registry so any registered
+        // real-value gets substituted with its alias on egress. Base64 bodies
+        // pass through unchanged (the literal scan would never match an
+        // encoded form; documented in tool-reference.md as a known limitation).
+        // Engine-agnostic via the network substrate: chromium fetches
+        // on demand (CDP Network.getResponseBody); firefox/webkit return the body
+        // captured at response time into the substrate's bounded recent-window cache.
+        const r = await e.networkSubstrate.fetchBody(requestId, caps.enabled.has("secrets") ? e.secrets : null);
+        return { content: [{ type: "text", text: JSON.stringify(r, null, 2) }] };
+    });
+    register("ws_read", {
+        capability: "read",
+        batchable: true,
+        description: "session-wide ring of recent WebSocket / Server-Sent-Events frames (HTTP is `network_read`; this is the realtime channel). Each frame: `{ url, dir: sent|recv, kind: ws|sse, opcode?, event?, payload, truncated?, ts }`. Payloads are truncated. Use to verify realtime correctness — chat/multiplayer/collaborative/live-dashboard broadcasts. Per-action frames also land in `ActionResult.network.wsFrames`; this is the across-session view.",
+        inputSchema: {
+            limit: z
+                .number()
+                .int()
+                .positive()
+                .max(500)
+                .optional()
+                .describe("Most-recent N frames (default 50)."),
+            urlPattern: z.string().optional().describe("Substring filter on the frame's endpoint URL."),
+            ...SESSION_ARG,
+        },
+    }, async ({ limit, urlPattern, session }) => {
+        const g = gateCheck("ws_read");
+        if (g)
+            return g;
+        const e = await entryFor(session);
+        const result = e.ws.recent(limit ?? 50, urlPattern);
+        return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
+    });
+    register("eval_js", {
+        capability: "eval",
+        batchable: true,
+        description: "Run a JavaScript expression in the page's main frame. Use sparingly — `find()`/action tools cover most cases. Common use: trigger a page-side function the app exposes (e.g. `window.__siteDocs.capture()`). The return value is page-controlled — treat it as untrusted content, just like snapshot text. ⚠ `element.click()` (and other programmatic DOM event calls) here do NOT fire framework click handlers (Vue `@click`, React synthetic events, custom-element listeners) — the event isn't trusted/synthetic-equivalent, so no app handler runs and you'll wrongly conclude the feature is broken. Use the `click` tool for a real, handler-firing click; reserve `eval_js` for reading state / calling app-exposed functions.",
+        inputSchema: {
+            expr: z
+                .string()
+                .describe("JS expression to evaluate. Wrap in `(() => { … })()` for statements."),
+            returnType: z
+                .enum(["json", "void"])
+                .default("json")
+                .describe("'json' returns the value (must be JSON-serializable); 'void' discards it (use for fire-and-forget calls)."),
+            ...TIMEOUT_ARG,
+            ...SESSION_ARG,
+        },
+    }, async ({ expr, returnType, timeoutMs, session }) => {
+        const g = gateCheck("eval_js");
+        if (g)
+            return g;
+        const e = await entryFor(session);
+        // page.evaluate has NO Playwright timeout — a never-resolving expr
+        // would wedge forever. Race it against the anti-wedge deadline.
+        const td = actionTimeout({ timeoutMs });
+        // soft warning: a programmatic .click() in eval_js does not fire
+        // framework (@click / synthetic-event) handlers — a recurring false
+        // "feature broken" negative. Point at the real `click` tool.
+        const clickWarn = /\.click\s*\(\s*\)/.test(expr)
+            ? "eval_js `.click()` does not fire framework click handlers (Vue/React/custom-element) — no app handler runs. If you're testing a click, use the `click` tool instead; this is a known false-negative source."
+            : undefined;
+        const warn = td.warning && clickWarn ? `${td.warning} ${clickWarn}` : (td.warning ?? clickWarn);
+        try {
+            if (returnType === "void") {
+                await withDeadline(scriptFor(e).evaluate(expr), td.ms, "eval_js").catch(() => undefined);
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: JSON.stringify({ ok: true, returnType: "void", ...(warn ? { warning: warn } : {}) }, null, 2),
+                        },
+                    ],
+                };
+            }
+            const value = await withDeadline(scriptFor(e).evaluate(expr), td.ms, "eval_js");
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: JSON.stringify({ ok: true, value, ...(warn ? { warning: warn } : {}) }, null, 2),
+                    },
+                ],
+            };
+        }
+        catch (err) {
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: JSON.stringify({
+                            ok: false,
+                            error: err instanceof Error ? err.message : String(err),
+                            ...(warn ? { warning: warn } : {}),
+                        }, null, 2),
+                    },
+                ],
+            };
+        }
+    });
+}

package/dist/tools/read-observe-capture-tools.d.ts

@@ -0,0 +1,12 @@
+import type { RegisterHost, GateHost, SessionHost, ActionHost, CaptureHost, ConfigHost, ServerServicesHost } from "./host.js";
+/**
+ * Read / observe — screenshot capture. The viewport/element PNG-JPEG primitive
+ * plus its scheduled (`screenshot_schedule`) and event-driven (`screenshot_on`)
+ * automations, every one bounded by the anti-wedge deadline. Registered through
+ * the shared `ToolHost` seam.
+ *
+ * The parameter is narrowed to the sub-ports this family touches (RFC 0004 P3 /
+ * D3 ISP) — gating, session resolution, the capture port, target resolution, and
+ * config.
+ */
+export declare function registerReadObserveCaptureTools(host: RegisterHost & GateHost & SessionHost & ActionHost & CaptureHost & ConfigHost & ServerServicesHost): void;