browxai 0.7.0

package/dist/tools/capture-report-export-tools.js

@@ -0,0 +1,246 @@
+import { requireCdp } from "../engine/index.js";
+import { withDeadline } from "../util/deadline.js";
+import { estimateTokens } from "../util/tokens.js";
+import { assetExport } from "../page/asset-export.js";
+import { pdfSave, assertPdfSupported } from "../page/pdf.js";
+import { pageArchive } from "../page/archive.js";
+import { SESSION_ARG } from "./schemas.js";
+/**
+ * Capture + report — page-level artefact export. `asset_export` (persist the
+ * session's network ring to disk), `pdf_save`, and `page_archive` (a
+ * self-contained whole-document capture). Workspace-rooted output by
+ * construction; registered through the shared `ToolHost` seam.
+ */
+export function registerCaptureReportExportTools(host) {
+    const { z, register, gateCheck, engineGate, entryFor, cfgActionTimeout, workspace } = host;
+    // `asset_export` — filter the session's network ring and persist matching
+    // responses to a workspace-rooted dir. Mirrors `download_get`'s file-io
+    // posture (read session-buffered state, write bytes under $BROWX_WORKSPACE).
+    // CORS caveat: when a response body has aged out of the renderer cache the
+    // tool falls back to an in-page `fetch()` against the original URL —
+    // cross-origin URLs without permissive CORS headers will land in
+    // `droppedCount`, not a crash.
+    register("asset_export", {
+        capability: "file-io",
+        description: 'Filter every resource the session has loaded (the always-on `NetworkBuffer` ring) and persist matching responses to a workspace-rooted directory — the first-class alternative to scraping `<img src>` / `<link href>` then re-fetching each one through `eval_js`. Filter shape: `{mime?: string[], urlPattern?: string, minBytes?: number, maxBytes?: number, status?: number[]}`. `mime` is substring match against the captured response `Content-Type` (case-insensitive, any one match wins; e.g. `["image/", "video/"]`). `urlPattern` is a RegExp source matched case-insensitively against the URL (e.g. `"\\\\.(woff2?|ttf|otf)$"`). `minBytes`/`maxBytes` bound the encoded response size when known. `status` defaults to 2xx (200..299). Filenames are derived from the URL path basename, **sanitised** (no path separators / NULs / leading dots / control bytes; length-capped), and collision-resolved with `-N` suffix. `intoDir` defaults to `$BROWX_WORKSPACE/assets/<sessionId>-<ISO>/`; an explicit value is resolved INSIDE `$BROWX_WORKSPACE` (escape rejected). Per-call caps: `maxCount` (default 10000) + `maxBytes` (default 500 MiB) bound runaway exports — callers can raise both up to hard ceilings. **CORS caveat**: when the response body has been discarded by the renderer (bodies are short-lived) the tool falls back to an in-page `fetch()` against the original URL — cross-origin URLs without permissive CORS headers land in `droppedCount`, never a crash. → `{ ok, intoDir, totalCount, matchedCount, persistedCount, droppedCount, manifest: [{url, mime?, status?, sizeBytes, savedAs}], warnings, tokensEstimate }`. The manifest is also written to `<intoDir>/_manifest.json`. `tokensEstimate` sizes the result envelope (the manifest blob), NOT the exported files. Gated by the off-by-default **`file-io`** capability — same posture as `download_get`.',
+        inputSchema: {
+            filter: z
+                .object({
+                mime: z
+                    .array(z.string())
+                    .optional()
+                    .describe("Substring match against response Content-Type (case-insensitive). Any one match wins."),
+                urlPattern: z
+                    .string()
+                    .optional()
+                    .describe("RegExp source matched case-insensitively against the URL."),
+                minBytes: z
+                    .number()
+                    .int()
+                    .nonnegative()
+                    .optional()
+                    .describe("Inclusive lower bound on encoded response byte size (when known)."),
+                maxBytes: z
+                    .number()
+                    .int()
+                    .nonnegative()
+                    .optional()
+                    .describe("Inclusive upper bound on encoded response byte size (when known)."),
+                status: z
+                    .array(z.number().int())
+                    .optional()
+                    .describe("Allow-list of HTTP status codes. Default: 200..299."),
+            })
+                .describe("Filter applied to every entry in the session's network ring."),
+            intoDir: z
+                .string()
+                .optional()
+                .describe("Workspace-rooted output directory. Default `assets/<sessionId>-<ISO>/`. Rejected if it escapes $BROWX_WORKSPACE."),
+            maxCount: z
+                .number()
+                .int()
+                .positive()
+                .optional()
+                .describe("Override the per-call file count cap (default 10000; clamped to hard ceiling 50000)."),
+            maxBytes: z
+                .number()
+                .int()
+                .positive()
+                .optional()
+                .describe("Override the per-call total byte cap (default 500 MiB; clamped to hard ceiling 2 GiB)."),
+            ...SESSION_ARG,
+        },
+    }, async (args) => {
+        const g = gateCheck("asset_export");
+        if (g)
+            return g;
+        const e = await entryFor(args.session);
+        try {
+            const result = await withDeadline(assetExport({
+                cdp: requireCdp(e.session),
+                page: e.session.page(),
+                buffer: e.network,
+                workspaceRoot: workspace.root,
+                sessionId: e.id,
+            }, {
+                filter: args.filter ?? {},
+                intoDir: args.intoDir,
+                maxCount: args.maxCount,
+                maxBytes: args.maxBytes,
+            }), cfgActionTimeout(), "asset_export");
+            const json = JSON.stringify(result);
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: JSON.stringify({ ...result, tokensEstimate: estimateTokens(json) }, null, 2),
+                    },
+                ],
+            };
+        }
+        catch (err) {
+            const body = { ok: false, error: err instanceof Error ? err.message : String(err) };
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: JSON.stringify({ ...body, tokensEstimate: estimateTokens(JSON.stringify(body)) }, null, 2),
+                    },
+                ],
+            };
+        }
+    });
+    // `pdf_save` — print the current page to a workspace-rooted PDF via
+    // Playwright `page.pdf()` (CDP `Page.printToPDF`). The mirror of
+    // `upload_file`: file-io OUT instead of IN. Chromium-only (every browxai
+    // session is Chromium so that's fine); refuses cleanly on `attached`/BYOB
+    // sessions where driving PrintToPDF would surface a print dialog / mutate
+    // the human's window state. Workspace-rooted by construction.
+    register("pdf_save", {
+        capability: "action",
+        deep: true,
+        description: "Print the current page to a workspace-rooted PDF via Playwright `page.pdf()` (CDP `Page.printToPDF`). The first-class alternative to screenshot-and-OCR or driving the browser's print-to-file dialog with `shortcut`. → `{ ok, path, bytes, format, scale, printBackground }`. Defaults: `format:\"A4\"`, `scale:1`, `printBackground:false` (matches browser-print's default — opt in when background colour/imagery matters). Output `path` is resolved INSIDE `$BROWX_WORKSPACE` (a path escaping the workspace is rejected); omit it for a default `pdfs/<sessionId>-<ts>.pdf`. **Refuses on `attached`/BYOB sessions** — `page.pdf()` drives Chromium's PrintToPDF and would mutate the human's window state; open a managed (`persistent`/`incognito`) session and re-run there. Capability `action`.",
+        inputSchema: {
+            path: z
+                .string()
+                .optional()
+                .describe("Workspace-rooted file path for the PDF. Default `pdfs/<sessionId>-<ts>.pdf`. Rejected if it escapes $BROWX_WORKSPACE."),
+            format: z
+                .enum(["Letter", "Legal", "Tabloid", "Ledger", "A0", "A1", "A2", "A3", "A4", "A5", "A6"])
+                .optional()
+                .describe('Paper format. Default "A4".'),
+            scale: z
+                .number()
+                .min(0.1)
+                .max(2.0)
+                .optional()
+                .describe("Render scale. Default 1. Bounded to [0.1, 2.0] (Playwright's CDP-layer clamp)."),
+            printBackground: z
+                .boolean()
+                .optional()
+                .describe("Include CSS background-color / background-image. Default false (matches browser-print default)."),
+            ...SESSION_ARG,
+        },
+    }, async (args) => {
+        const g = gateCheck("pdf_save");
+        if (g)
+            return g;
+        const e = await entryFor(args.session);
+        const eg = engineGate("pdf_save", e);
+        if (eg)
+            return eg;
+        try {
+            const refused = assertPdfSupported({ mode: e.mode });
+            if (refused) {
+                const body = { ok: false, error: refused.error, hint: refused.hint };
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: JSON.stringify({ ...body, tokensEstimate: estimateTokens(JSON.stringify(body)) }, null, 2),
+                        },
+                    ],
+                };
+            }
+            const r = await withDeadline(pdfSave(e.session.page(), workspace.root, e.id, {
+                path: args.path,
+                format: args.format,
+                scale: args.scale,
+                printBackground: args.printBackground,
+            }), cfgActionTimeout(), "pdf_save");
+            return { content: [{ type: "text", text: JSON.stringify(r, null, 2) }] };
+        }
+        catch (err) {
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: JSON.stringify({ ok: false, error: err instanceof Error ? err.message : String(err) }, null, 2),
+                    },
+                ],
+            };
+        }
+    });
+    // `page_archive` — save the current page (HTML + linked resources) as a
+    // self-contained artefact, either as a directory (`index.html` + `assets/`
+    // sidecar) or as a single-file inlined HTML. Workspace-rooted by
+    // construction (same `resolveWorkspacePath` posture as `pdf_save` /
+    // `start_har`). Under the off-by-default `file-io` capability — a deliberate
+    // filesystem egress, not a routine action. The agent is expected to
+    // navigate + settle the page BEFORE calling: the tool does not inject its
+    // own wait. The output is faithfully UNMASKED — see archive.ts header for
+    // the secrets-masking deliberate-gap rationale.
+    register("page_archive", {
+        capability: "file-io",
+        description: "Save the current page as a self-contained archive. Two formats: `directory` (default) writes `<path>/index.html` + `<path>/assets/` sidecar with every linked resource (images, fonts, scripts, stylesheets, CSS background-images surfaced via getComputedStyle); HTML refs rewritten to relative `assets/...` paths. `single-file` writes one HTML at `<path>` with every resource inlined as a `data:` URI (browsers struggle past ~150 MB — large pages should prefer `directory`). `path` is resolved INSIDE `$BROWX_WORKSPACE` (escape rejected); omit for `archives/<sessionId>-<ISO>` (directory) or `archives/<sessionId>-<ISO>.html` (single-file). `maxSizeMb` caps the total archive (default 200) — resources past the budget land in `droppedCount`. Resource fetching runs `await fetch(url)` IN-page (subject to the page's CSP `connect-src` — cross-origin blocks are caught, dropped, and counted). → `{ ok, format, path, sizeBytes, resourceCount, droppedCount, warnings[] }`. **Secrets-masking caveat**: the archive is intentionally UNMASKED — running the egress masking layer would corrupt inline JSON/CSS/binary bytes; treat the archive as sensitive (same posture as `dump_storage_state`). Caller must navigate + settle the page BEFORE calling; `page_archive` does not inject its own wait. Capability `file-io`.",
+        inputSchema: {
+            path: z
+                .string()
+                .optional()
+                .describe("Workspace-rooted output path (directory for `directory` format; .html file for `single-file`). Default `archives/<sessionId>-<ISO>[.html]`. Rejected if it escapes $BROWX_WORKSPACE."),
+            format: z
+                .enum(["directory", "single-file"])
+                .optional()
+                .describe("`directory` (default) → index.html + assets/ sidecar; `single-file` → one HTML with data:-URI-inlined resources."),
+            maxSizeMb: z
+                .number()
+                .positive()
+                .max(10_000)
+                .optional()
+                .describe("Total archive size cap (MB). Default 200. Resources past the budget are dropped + counted."),
+            ...SESSION_ARG,
+        },
+    }, async (args) => {
+        const g = gateCheck("page_archive");
+        if (g)
+            return g;
+        const e = await entryFor(args.session);
+        try {
+            const r = await withDeadline(pageArchive(e.session.page(), workspace.root, e.id, {
+                path: args.path,
+                format: args.format,
+                maxSizeMb: args.maxSizeMb,
+            }), cfgActionTimeout(), "page_archive");
+            const json = JSON.stringify(r);
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: JSON.stringify({ ...r, tokensEstimate: estimateTokens(json) }, null, 2),
+                    },
+                ],
+            };
+        }
+        catch (err) {
+            const body = { ok: false, error: err instanceof Error ? err.message : String(err) };
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: JSON.stringify({ ...body, tokensEstimate: estimateTokens(JSON.stringify(body)) }, null, 2),
+                    },
+                ],
+            };
+        }
+    });
+}

package/dist/tools/capture-report-marks-tools.d.ts

@@ -0,0 +1,9 @@
+import type { ToolHost } from "./host.js";
+/**
+ * Capture + report — region capture & marks composition. `screenshot_region`
+ * (viewport-rect PNG), `screenshot_marks` (set-of-marks overlay), the
+ * `name_region` / `region` named-rect binding, and `cross_session_sample`
+ * (one DOM metric across every open session). Registered through the shared
+ * `ToolHost` seam.
+ */
+export declare function registerCaptureReportMarksTools(host: ToolHost): void;

package/dist/tools/capture-report-marks-tools.js

@@ -0,0 +1,221 @@
+import { withDeadline } from "../util/deadline.js";
+import { screenshotMarks } from "../page/set-of-marks.js";
+import { sampleMetric, ELEMENT_METRICS } from "../page/sample.js";
+import { SESSION_ARG } from "./schemas.js";
+/**
+ * Capture + report — region capture & marks composition. `screenshot_region`
+ * (viewport-rect PNG), `screenshot_marks` (set-of-marks overlay), the
+ * `name_region` / `region` named-rect binding, and `cross_session_sample`
+ * (one DOM metric across every open session). Registered through the shared
+ * `ToolHost` seam.
+ */
+export function registerCaptureReportMarksTools(host) {
+    const { z, register, gateCheck, entryFor, cfgActionTimeout, config, toolHandlers, batchAllowedTools: BATCH_ALLOWED_TOOLS, } = host;
+    const BOX_SCHEMA = z.object({
+        x: z.number(),
+        y: z.number(),
+        width: z.number().positive(),
+        height: z.number().positive(),
+    });
+    register("screenshot_region", {
+        capability: "read",
+        description: "PNG screenshot of an arbitrary viewport rectangle (not an element) — for virtualised timelines / canvas / unlabelled positioned regions where an element-cropped shot doesn't apply.",
+        inputSchema: {
+            box: BOX_SCHEMA.describe("Viewport rect {x,y,width,height} in CSS px."),
+            ...SESSION_ARG,
+        },
+    }, async ({ box, session }) => {
+        const g = gateCheck("screenshot_region");
+        if (g)
+            return g;
+        const e = await entryFor(session);
+        try {
+            const buf = await withDeadline(e.session.page().screenshot({ clip: box, type: "png" }), cfgActionTimeout(), "screenshot_region");
+            return {
+                content: [
+                    {
+                        type: "image",
+                        data: Buffer.from(buf).toString("base64"),
+                        mimeType: "image/png",
+                    },
+                ],
+            };
+        }
+        catch (err) {
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: JSON.stringify({ ok: false, error: err instanceof Error ? err.message : String(err) }, null, 2),
+                    },
+                ],
+            };
+        }
+    });
+    register("screenshot_marks", {
+        capability: "read",
+        batchable: true,
+        description: 'Composed PNG with numbered bounding boxes painted over caller-supplied candidates — the set-of-marks primitive multimodal agents reach for when they want to ground a vision read against a small palette of stable refs ("click 2" instead of estimating a coordinate). Each candidate is either a bare `{ref}` (looked up against the current snapshot for its bbox) OR a full `find()` candidate row passed through (`{ref, role, name, testId, bbox}` — fast path, no extra tree walk). `label:"index"` (default) paints 1..N positions paired with an `{index→ref}` mapping; `label:"ref"` paints the existing `eN` directly; `label:"role"` paints the role for visual grounding. The numbering scheme SHARES the existing `name_ref` / `eN` namespace — no parallel ID space — so `mapping["2"] === "e7"` and the agent can address either way. Pure compose on top of `find()` / `snapshot()` (no new browser interaction beyond a transient in-page overlay removed before return). Candidates with `bbox:null` (clipped / off-screen) are kept in `marks` with `painted:false` so the mapping stays complete. Read-only (`read`).',
+        inputSchema: {
+            candidates: z
+                .array(z.union([z.object({ ref: z.string() }).passthrough(), z.object({}).passthrough()]))
+                .min(1)
+                .max(50)
+                .describe("Either `{ref}` rows (looked up against the current snapshot for bbox) OR full find() candidate rows (passed through). Mix-and-match allowed. Cap 50."),
+            label: z
+                .enum(["index", "ref", "role"])
+                .optional()
+                .describe("How to label each painted box. `index` (default) = 1..N array position, paired with the `{index→ref}` mapping in the result. `ref` = paint the existing `eN` ref directly. `role` = paint the candidate's role."),
+            ...SESSION_ARG,
+        },
+    }, async (args) => {
+        const g = gateCheck("screenshot_marks");
+        if (g)
+            return g;
+        const e = await entryFor(args.session);
+        const candidates = args.candidates;
+        try {
+            const result = await withDeadline(screenshotMarks(e.session.page(), e.snapshotSubstrate, e.refs, {
+                candidates,
+                label: args.label,
+                testAttributes: config.testAttributes,
+            }, e.session.cdp ? e.session.cdp() : undefined), cfgActionTimeout(), "screenshot_marks");
+            const content = [
+                {
+                    type: "text",
+                    text: JSON.stringify({ marks: result.marks, mapping: result.mapping, warnings: result.warnings }, null, 2),
+                },
+                { type: "image", data: result.imageBase64, mimeType: result.mimeType },
+            ];
+            return { content };
+        }
+        catch (err) {
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: JSON.stringify({ ok: false, error: err instanceof Error ? err.message : String(err) }, null, 2),
+                    },
+                ],
+            };
+        }
+    });
+    register("name_region", {
+        capability: "human",
+        description: "Bind a viewport rectangle to a mnemonic so a sub-agent can re-select the same media segment / timeline row without re-deriving coordinates (drift). Resolve it later with `region`. Per-session.",
+        inputSchema: {
+            name: z.string().describe('Mnemonic, e.g. "matching_audio_clip".'),
+            box: BOX_SCHEMA,
+            ...SESSION_ARG,
+        },
+    }, async ({ name, box, session }) => {
+        const g = gateCheck("name_region");
+        if (g)
+            return g;
+        const e = await entryFor(session);
+        return {
+            content: [
+                {
+                    type: "text",
+                    text: JSON.stringify({ ok: true, ...e.regions.set(name, box) }, null, 2),
+                },
+            ],
+        };
+    });
+    register("region", {
+        capability: "human",
+        description: "Resolve a `name_region` mnemonic to its `{ box, center }`. Pass `center` to a coords-based action (`click({coords})`) to act on the bound region.",
+        inputSchema: { name: z.string(), ...SESSION_ARG },
+    }, async ({ name, session }) => {
+        const g = gateCheck("region");
+        if (g)
+            return g;
+        const e = await entryFor(session);
+        const r = e.regions.get(name);
+        return {
+            content: [
+                {
+                    type: "text",
+                    text: JSON.stringify(r
+                        ? { ok: true, ...r }
+                        : {
+                            ok: false,
+                            error: `no region named "${name}" — call name_region first`,
+                            known: e.regions.list().map((x) => x.name),
+                        }, null, 2),
+                },
+            ],
+        };
+    });
+    register("cross_session_sample", {
+        capability: "read",
+        description: "Drive an action in one session and sample a metric in ANOTHER over the same window, in one call — for realtime-propagation assertions (an action in session A should reflect in session B within a freshness budget). `action` is `{tool,args}` from the batch whitelist, dispatched in `actionSession`; the document-scroller `metric` is traced in `sampleSession`. Returns `{ action: <inner result>, sample }`.",
+        inputSchema: {
+            action: z.object({ tool: z.string(), args: z.record(z.unknown()).optional() }),
+            actionSession: z.string().describe("Session the action runs in."),
+            sampleSession: z.string().describe("Session whose page is sampled."),
+            metric: z
+                .enum(ELEMENT_METRICS)
+                .describe("Fixed metric (document scroller of sampleSession)."),
+            durationMs: z.number().int().positive().max(30_000),
+            everyFrame: z.boolean().optional(),
+            intervalMs: z.number().int().positive().max(5000).optional(),
+        },
+    }, async (args) => {
+        const g = gateCheck("cross_session_sample");
+        if (g)
+            return g;
+        const innerTool = args.action.tool;
+        if (!BATCH_ALLOWED_TOOLS.has(innerTool) || innerTool === "cross_session_sample") {
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: JSON.stringify({
+                            ok: false,
+                            error: `cross_session_sample: inner tool "${innerTool}" not allowed (batch whitelist; no batch / await_human / recording / self)`,
+                        }, null, 2),
+                    },
+                ],
+            };
+        }
+        const ig = gateCheck(innerTool);
+        if (ig)
+            return ig;
+        const sampleEntry = await entryFor(args.sampleSession);
+        const samplePromise = sampleMetric(sampleEntry.session.page(), sampleEntry.refs, {
+            metric: args.metric,
+            durationMs: args.durationMs,
+            everyFrame: args.everyFrame,
+            intervalMs: args.intervalMs,
+        });
+        const innerArgs = { ...(args.action.args ?? {}), session: args.actionSession };
+        const parseInner = (resp) => {
+            const first = resp.content[0];
+            if (!first || first.type !== "text" || first.text === undefined)
+                return first ?? null;
+            try {
+                return JSON.parse(first.text);
+            }
+            catch {
+                return first.text;
+            }
+        };
+        const [sRes, aRes] = await Promise.allSettled([
+            samplePromise,
+            toolHandlers[innerTool](innerArgs),
+        ]);
+        const sample = sRes.status === "fulfilled"
+            ? sRes.value
+            : { error: sRes.reason instanceof Error ? sRes.reason.message : String(sRes.reason) };
+        const action = aRes.status === "fulfilled"
+            ? parseInner(aRes.value)
+            : {
+                ok: false,
+                error: aRes.reason instanceof Error ? aRes.reason.message : String(aRes.reason),
+            };
+        return {
+            content: [{ type: "text", text: JSON.stringify({ action, sample }, null, 2) }],
+        };
+    });
+}

package/dist/tools/capture-report-upload-tools.d.ts

@@ -0,0 +1,8 @@
+import type { ToolHost } from "./host.js";
+/**
+ * Capture + report — file ingress & download egress. `upload_file` / `drop_files`
+ * (file ingress into the page), `downloads_capture` / `download_get` (capture and
+ * read browser-initiated downloads under the workspace root). Registered through
+ * the shared `ToolHost` seam.
+ */
+export declare function registerCaptureReportUploadTools(host: ToolHost): void;