@agenticmail/enterprise 0.5.78 → 0.5.79

package/src/browser/routes/agent.debug.ts

@@ -0,0 +1,149 @@
+import crypto from "node:crypto";
+import fs from "node:fs/promises";
+import path from "node:path";
+import type { BrowserRouteContext } from "../server-context.js";
+import {
+  readBody,
+  resolveTargetIdFromBody,
+  resolveTargetIdFromQuery,
+  withPlaywrightRouteContext,
+} from "./agent.shared.js";
+import { DEFAULT_TRACE_DIR, resolvePathWithinRoot } from "./path-output.js";
+import type { BrowserRouteRegistrar } from "./types.js";
+import { toBoolean, toStringOrEmpty } from "./utils.js";
+
+export function registerBrowserAgentDebugRoutes(
+  app: BrowserRouteRegistrar,
+  ctx: BrowserRouteContext,
+) {
+  app.get("/console", async (req, res) => {
+    const targetId = resolveTargetIdFromQuery(req.query);
+    const level = typeof req.query.level === "string" ? req.query.level : "";
+
+    await withPlaywrightRouteContext({
+      req,
+      res,
+      ctx,
+      targetId,
+      feature: "console messages",
+      run: async ({ cdpUrl, tab, pw }) => {
+        const messages = await pw.getConsoleMessagesViaPlaywright({
+          cdpUrl,
+          targetId: tab.targetId,
+          level: level.trim() || undefined,
+        });
+        res.json({ ok: true, messages, targetId: tab.targetId });
+      },
+    });
+  });
+
+  app.get("/errors", async (req, res) => {
+    const targetId = resolveTargetIdFromQuery(req.query);
+    const clear = toBoolean(req.query.clear) ?? false;
+
+    await withPlaywrightRouteContext({
+      req,
+      res,
+      ctx,
+      targetId,
+      feature: "page errors",
+      run: async ({ cdpUrl, tab, pw }) => {
+        const result = await pw.getPageErrorsViaPlaywright({
+          cdpUrl,
+          targetId: tab.targetId,
+          clear,
+        });
+        res.json({ ok: true, targetId: tab.targetId, ...result });
+      },
+    });
+  });
+
+  app.get("/requests", async (req, res) => {
+    const targetId = resolveTargetIdFromQuery(req.query);
+    const filter = typeof req.query.filter === "string" ? req.query.filter : "";
+    const clear = toBoolean(req.query.clear) ?? false;
+
+    await withPlaywrightRouteContext({
+      req,
+      res,
+      ctx,
+      targetId,
+      feature: "network requests",
+      run: async ({ cdpUrl, tab, pw }) => {
+        const result = await pw.getNetworkRequestsViaPlaywright({
+          cdpUrl,
+          targetId: tab.targetId,
+          filter: filter.trim() || undefined,
+          clear,
+        });
+        res.json({ ok: true, targetId: tab.targetId, ...result });
+      },
+    });
+  });
+
+  app.post("/trace/start", async (req, res) => {
+    const body = readBody(req);
+    const targetId = resolveTargetIdFromBody(body);
+    const screenshots = toBoolean(body.screenshots) ?? undefined;
+    const snapshots = toBoolean(body.snapshots) ?? undefined;
+    const sources = toBoolean(body.sources) ?? undefined;
+
+    await withPlaywrightRouteContext({
+      req,
+      res,
+      ctx,
+      targetId,
+      feature: "trace start",
+      run: async ({ cdpUrl, tab, pw }) => {
+        await pw.traceStartViaPlaywright({
+          cdpUrl,
+          targetId: tab.targetId,
+          screenshots,
+          snapshots,
+          sources,
+        });
+        res.json({ ok: true, targetId: tab.targetId });
+      },
+    });
+  });
+
+  app.post("/trace/stop", async (req, res) => {
+    const body = readBody(req);
+    const targetId = resolveTargetIdFromBody(body);
+    const out = toStringOrEmpty(body.path) || "";
+
+    await withPlaywrightRouteContext({
+      req,
+      res,
+      ctx,
+      targetId,
+      feature: "trace stop",
+      run: async ({ cdpUrl, tab, pw }) => {
+        const id = crypto.randomUUID();
+        const dir = DEFAULT_TRACE_DIR;
+        await fs.mkdir(dir, { recursive: true });
+        const tracePathResult = resolvePathWithinRoot({
+          rootDir: dir,
+          requestedPath: out,
+          scopeLabel: "trace directory",
+          defaultFileName: `browser-trace-${id}.zip`,
+        });
+        if (!tracePathResult.ok) {
+          res.status(400).json({ error: tracePathResult.error });
+          return;
+        }
+        const tracePath = tracePathResult.path;
+        await pw.traceStopViaPlaywright({
+          cdpUrl,
+          targetId: tab.targetId,
+          path: tracePath,
+        });
+        res.json({
+          ok: true,
+          targetId: tab.targetId,
+          path: path.resolve(tracePath),
+        });
+      },
+    });
+  });
+}

package/src/browser/routes/agent.shared.ts

@@ -0,0 +1,143 @@
+import type { PwAiModule } from "../pw-ai-module.js";
+import { getPwAiModule as getPwAiModuleBase } from "../pw-ai-module.js";
+import type { BrowserRouteContext, ProfileContext } from "../server-context.js";
+import type { BrowserRequest, BrowserResponse } from "./types.js";
+import { getProfileContext, jsonError } from "./utils.js";
+
+export const SELECTOR_UNSUPPORTED_MESSAGE = [
+  "Error: 'selector' is not supported. Use 'ref' from snapshot instead.",
+  "",
+  "Example workflow:",
+  "1. snapshot action to get page state with refs",
+  '2. act with ref: "e123" to interact with element',
+  "",
+  "This is more reliable for modern SPAs.",
+].join("\n");
+
+export function readBody(req: BrowserRequest): Record<string, unknown> {
+  const body = req.body as Record<string, unknown> | undefined;
+  if (!body || typeof body !== "object" || Array.isArray(body)) {
+    return {};
+  }
+  return body;
+}
+
+export function resolveTargetIdFromBody(body: Record<string, unknown>): string | undefined {
+  const targetId = typeof body.targetId === "string" ? body.targetId.trim() : "";
+  return targetId || undefined;
+}
+
+export function resolveTargetIdFromQuery(query: Record<string, unknown>): string | undefined {
+  const targetId = typeof query.targetId === "string" ? query.targetId.trim() : "";
+  return targetId || undefined;
+}
+
+export function handleRouteError(ctx: BrowserRouteContext, res: BrowserResponse, err: unknown) {
+  const mapped = ctx.mapTabError(err);
+  if (mapped) {
+    return jsonError(res, mapped.status, mapped.message);
+  }
+  jsonError(res, 500, String(err));
+}
+
+export function resolveProfileContext(
+  req: BrowserRequest,
+  res: BrowserResponse,
+  ctx: BrowserRouteContext,
+): ProfileContext | null {
+  const profileCtx = getProfileContext(req, ctx);
+  if ("error" in profileCtx) {
+    jsonError(res, profileCtx.status, profileCtx.error);
+    return null;
+  }
+  return profileCtx;
+}
+
+export async function getPwAiModule(): Promise<PwAiModule | null> {
+  return await getPwAiModuleBase({ mode: "soft" });
+}
+
+export async function requirePwAi(
+  res: BrowserResponse,
+  feature: string,
+): Promise<PwAiModule | null> {
+  const mod = await getPwAiModule();
+  if (mod) {
+    return mod;
+  }
+  jsonError(
+    res,
+    501,
+    [
+      `Playwright is not available in this gateway build; '${feature}' is unsupported.`,
+      "Install the full Playwright package (not playwright-core) and restart the gateway, or reinstall with browser support.",
+      "Docs: /tools/browser#playwright-requirement",
+    ].join("\n"),
+  );
+  return null;
+}
+
+type RouteTabContext = {
+  profileCtx: ProfileContext;
+  tab: Awaited<ReturnType<ProfileContext["ensureTabAvailable"]>>;
+  cdpUrl: string;
+};
+
+type RouteTabPwContext = RouteTabContext & {
+  pw: PwAiModule;
+};
+
+type RouteWithTabParams<T> = {
+  req: BrowserRequest;
+  res: BrowserResponse;
+  ctx: BrowserRouteContext;
+  targetId?: string;
+  run: (ctx: RouteTabContext) => Promise<T>;
+};
+
+export async function withRouteTabContext<T>(
+  params: RouteWithTabParams<T>,
+): Promise<T | undefined> {
+  const profileCtx = resolveProfileContext(params.req, params.res, params.ctx);
+  if (!profileCtx) {
+    return undefined;
+  }
+  try {
+    const tab = await profileCtx.ensureTabAvailable(params.targetId);
+    return await params.run({
+      profileCtx,
+      tab,
+      cdpUrl: profileCtx.profile.cdpUrl,
+    });
+  } catch (err) {
+    handleRouteError(params.ctx, params.res, err);
+    return undefined;
+  }
+}
+
+type RouteWithPwParams<T> = {
+  req: BrowserRequest;
+  res: BrowserResponse;
+  ctx: BrowserRouteContext;
+  targetId?: string;
+  feature: string;
+  run: (ctx: RouteTabPwContext) => Promise<T>;
+};
+
+export async function withPlaywrightRouteContext<T>(
+  params: RouteWithPwParams<T>,
+): Promise<T | undefined> {
+  return await withRouteTabContext({
+    req: params.req,
+    res: params.res,
+    ctx: params.ctx,
+    targetId: params.targetId,
+    run: async ({ profileCtx, tab, cdpUrl }) => {
+      const pw = await requirePwAi(params.res, params.feature);
+      if (!pw) {
+        return undefined as T | undefined;
+      }
+      return await params.run({ profileCtx, tab, cdpUrl, pw });
+    },
+  });
+}

package/src/browser/routes/agent.snapshot.ts

@@ -0,0 +1,333 @@
+import path from "node:path";
+import { ensureMediaDir, saveMediaBuffer } from "../enterprise-compat.js";
+import { captureScreenshot, snapshotAria } from "../cdp.js";
+import {
+  DEFAULT_AI_SNAPSHOT_EFFICIENT_DEPTH,
+  DEFAULT_AI_SNAPSHOT_EFFICIENT_MAX_CHARS,
+  DEFAULT_AI_SNAPSHOT_MAX_CHARS,
+} from "../constants.js";
+import { withBrowserNavigationPolicy } from "../navigation-guard.js";
+import {
+  DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
+  DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE,
+  normalizeBrowserScreenshot,
+} from "../screenshot.js";
+import type { BrowserRouteContext } from "../server-context.js";
+import {
+  getPwAiModule,
+  handleRouteError,
+  readBody,
+  requirePwAi,
+  resolveProfileContext,
+  withPlaywrightRouteContext,
+  withRouteTabContext,
+} from "./agent.shared.js";
+import type { BrowserResponse, BrowserRouteRegistrar } from "./types.js";
+import { jsonError, toBoolean, toNumber, toStringOrEmpty } from "./utils.js";
+
+async function saveBrowserMediaResponse(params: {
+  res: BrowserResponse;
+  buffer: Buffer;
+  contentType: string;
+  maxBytes: number;
+  targetId: string;
+  url: string;
+}) {
+  await ensureMediaDir();
+  const saved = await saveMediaBuffer(
+    params.buffer,
+    params.contentType,
+    "browser",
+    params.maxBytes,
+  );
+  params.res.json({
+    ok: true,
+    path: path.resolve(saved.path),
+    targetId: params.targetId,
+    url: params.url,
+  });
+}
+
+export function registerBrowserAgentSnapshotRoutes(
+  app: BrowserRouteRegistrar,
+  ctx: BrowserRouteContext,
+) {
+  app.post("/navigate", async (req, res) => {
+    const body = readBody(req);
+    const url = toStringOrEmpty(body.url);
+    const targetId = toStringOrEmpty(body.targetId) || undefined;
+    if (!url) {
+      return jsonError(res, 400, "url is required");
+    }
+    await withPlaywrightRouteContext({
+      req,
+      res,
+      ctx,
+      targetId,
+      feature: "navigate",
+      run: async ({ cdpUrl, tab, pw }) => {
+        const result = await pw.navigateViaPlaywright({
+          cdpUrl,
+          targetId: tab.targetId,
+          url,
+          ...withBrowserNavigationPolicy(ctx.state().resolved.ssrfPolicy),
+        });
+        res.json({ ok: true, targetId: tab.targetId, ...result });
+      },
+    });
+  });
+
+  app.post("/pdf", async (req, res) => {
+    const body = readBody(req);
+    const targetId = toStringOrEmpty(body.targetId) || undefined;
+    await withPlaywrightRouteContext({
+      req,
+      res,
+      ctx,
+      targetId,
+      feature: "pdf",
+      run: async ({ cdpUrl, tab, pw }) => {
+        const pdf = await pw.pdfViaPlaywright({
+          cdpUrl,
+          targetId: tab.targetId,
+        });
+        await saveBrowserMediaResponse({
+          res,
+          buffer: pdf.buffer,
+          contentType: "application/pdf",
+          maxBytes: pdf.buffer.byteLength,
+          targetId: tab.targetId,
+          url: tab.url,
+        });
+      },
+    });
+  });
+
+  app.post("/screenshot", async (req, res) => {
+    const body = readBody(req);
+    const targetId = toStringOrEmpty(body.targetId) || undefined;
+    const fullPage = toBoolean(body.fullPage) ?? false;
+    const ref = toStringOrEmpty(body.ref) || undefined;
+    const element = toStringOrEmpty(body.element) || undefined;
+    const type = body.type === "jpeg" ? "jpeg" : "png";
+
+    if (fullPage && (ref || element)) {
+      return jsonError(res, 400, "fullPage is not supported for element screenshots");
+    }
+
+    await withRouteTabContext({
+      req,
+      res,
+      ctx,
+      targetId,
+      run: async ({ profileCtx, tab, cdpUrl }) => {
+        let buffer: Buffer;
+        const shouldUsePlaywright =
+          profileCtx.profile.driver === "extension" ||
+          !tab.wsUrl ||
+          Boolean(ref) ||
+          Boolean(element);
+        if (shouldUsePlaywright) {
+          const pw = await requirePwAi(res, "screenshot");
+          if (!pw) {
+            return;
+          }
+          const snap = await pw.takeScreenshotViaPlaywright({
+            cdpUrl,
+            targetId: tab.targetId,
+            ref,
+            element,
+            fullPage,
+            type,
+          });
+          buffer = snap.buffer;
+        } else {
+          buffer = await captureScreenshot({
+            wsUrl: tab.wsUrl ?? "",
+            fullPage,
+            format: type,
+            quality: type === "jpeg" ? 85 : undefined,
+          });
+        }
+
+        const normalized = await normalizeBrowserScreenshot(buffer, {
+          maxSide: DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE,
+          maxBytes: DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
+        });
+        await saveBrowserMediaResponse({
+          res,
+          buffer: normalized.buffer,
+          contentType: normalized.contentType ?? `image/${type}`,
+          maxBytes: DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
+          targetId: tab.targetId,
+          url: tab.url,
+        });
+      },
+    });
+  });
+
+  app.get("/snapshot", async (req, res) => {
+    const profileCtx = resolveProfileContext(req, res, ctx);
+    if (!profileCtx) {
+      return;
+    }
+    const targetId = typeof req.query.targetId === "string" ? req.query.targetId.trim() : "";
+    const mode = req.query.mode === "efficient" ? "efficient" : undefined;
+    const labels = toBoolean(req.query.labels) ?? undefined;
+    const explicitFormat =
+      req.query.format === "aria" ? "aria" : req.query.format === "ai" ? "ai" : undefined;
+    const format = explicitFormat ?? (mode ? "ai" : (await getPwAiModule()) ? "ai" : "aria");
+    const limitRaw = typeof req.query.limit === "string" ? Number(req.query.limit) : undefined;
+    const hasMaxChars = Object.hasOwn(req.query, "maxChars");
+    const maxCharsRaw =
+      typeof req.query.maxChars === "string" ? Number(req.query.maxChars) : undefined;
+    const limit = Number.isFinite(limitRaw) ? limitRaw : undefined;
+    const maxChars =
+      typeof maxCharsRaw === "number" && Number.isFinite(maxCharsRaw) && maxCharsRaw > 0
+        ? Math.floor(maxCharsRaw)
+        : undefined;
+    const resolvedMaxChars =
+      format === "ai"
+        ? hasMaxChars
+          ? maxChars
+          : mode === "efficient"
+            ? DEFAULT_AI_SNAPSHOT_EFFICIENT_MAX_CHARS
+            : DEFAULT_AI_SNAPSHOT_MAX_CHARS
+        : undefined;
+    const interactiveRaw = toBoolean(req.query.interactive);
+    const compactRaw = toBoolean(req.query.compact);
+    const depthRaw = toNumber(req.query.depth);
+    const refsModeRaw = toStringOrEmpty(req.query.refs).trim();
+    const refsMode: "aria" | "role" | undefined =
+      refsModeRaw === "aria" ? "aria" : refsModeRaw === "role" ? "role" : undefined;
+    const interactive = interactiveRaw ?? (mode === "efficient" ? true : undefined);
+    const compact = compactRaw ?? (mode === "efficient" ? true : undefined);
+    const depth =
+      depthRaw ?? (mode === "efficient" ? DEFAULT_AI_SNAPSHOT_EFFICIENT_DEPTH : undefined);
+    const selector = toStringOrEmpty(req.query.selector);
+    const frameSelector = toStringOrEmpty(req.query.frame);
+    const selectorValue = selector.trim() || undefined;
+    const frameSelectorValue = frameSelector.trim() || undefined;
+
+    try {
+      const tab = await profileCtx.ensureTabAvailable(targetId || undefined);
+      if ((labels || mode === "efficient") && format === "aria") {
+        return jsonError(res, 400, "labels/mode=efficient require format=ai");
+      }
+      if (format === "ai") {
+        const pw = await requirePwAi(res, "ai snapshot");
+        if (!pw) {
+          return;
+        }
+        const wantsRoleSnapshot =
+          labels === true ||
+          mode === "efficient" ||
+          interactive === true ||
+          compact === true ||
+          depth !== undefined ||
+          Boolean(selectorValue) ||
+          Boolean(frameSelectorValue);
+        const roleSnapshotArgs = {
+          cdpUrl: profileCtx.profile.cdpUrl,
+          targetId: tab.targetId,
+          selector: selectorValue,
+          frameSelector: frameSelectorValue,
+          refsMode,
+          options: {
+            interactive: interactive ?? undefined,
+            compact: compact ?? undefined,
+            maxDepth: depth ?? undefined,
+          },
+        };
+
+        const snap = wantsRoleSnapshot
+          ? await pw.snapshotRoleViaPlaywright(roleSnapshotArgs)
+          : await pw
+              .snapshotAiViaPlaywright({
+                cdpUrl: profileCtx.profile.cdpUrl,
+                targetId: tab.targetId,
+                ...(typeof resolvedMaxChars === "number" ? { maxChars: resolvedMaxChars } : {}),
+              })
+              .catch(async (err) => {
+                // Public-API fallback when Playwright's private _snapshotForAI is missing.
+                if (String(err).toLowerCase().includes("_snapshotforai")) {
+                  return await pw.snapshotRoleViaPlaywright(roleSnapshotArgs);
+                }
+                throw err;
+              });
+        if (labels) {
+          const labeled = await pw.screenshotWithLabelsViaPlaywright({
+            cdpUrl: profileCtx.profile.cdpUrl,
+            targetId: tab.targetId,
+            refs: "refs" in snap ? snap.refs : {},
+            type: "png",
+          });
+          const normalized = await normalizeBrowserScreenshot(labeled.buffer, {
+            maxSide: DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE,
+            maxBytes: DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
+          });
+          await ensureMediaDir();
+          const saved = await saveMediaBuffer(
+            normalized.buffer,
+            normalized.contentType ?? "image/png",
+            "browser",
+            DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
+          );
+          const imageType = normalized.contentType?.includes("jpeg") ? "jpeg" : "png";
+          return res.json({
+            ok: true,
+            format,
+            targetId: tab.targetId,
+            url: tab.url,
+            labels: true,
+            labelsCount: labeled.labels,
+            labelsSkipped: labeled.skipped,
+            imagePath: path.resolve(saved.path),
+            imageType,
+            ...snap,
+          });
+        }
+
+        return res.json({
+          ok: true,
+          format,
+          targetId: tab.targetId,
+          url: tab.url,
+          ...snap,
+        });
+      }
+
+      const snap =
+        profileCtx.profile.driver === "extension" || !tab.wsUrl
+          ? (() => {
+              // Extension relay doesn't expose per-page WS URLs; run AX snapshot via Playwright CDP session.
+              // Also covers cases where wsUrl is missing/unusable.
+              return requirePwAi(res, "aria snapshot").then(async (pw) => {
+                if (!pw) {
+                  return null;
+                }
+                return await pw.snapshotAriaViaPlaywright({
+                  cdpUrl: profileCtx.profile.cdpUrl,
+                  targetId: tab.targetId,
+                  limit,
+                });
+              });
+            })()
+          : snapshotAria({ wsUrl: tab.wsUrl ?? "", limit });
+
+      const resolved = await Promise.resolve(snap);
+      if (!resolved) {
+        return;
+      }
+      return res.json({
+        ok: true,
+        format,
+        targetId: tab.targetId,
+        url: tab.url,
+        ...resolved,
+      });
+    } catch (err) {
+      handleRouteError(ctx, res, err);
+    }
+  });
+}