minimal-agent 0.13.0 → 0.14.0

package/src/tools/webbrowser/browserContent.ts

@@ -0,0 +1,438 @@
+import { mkdir, writeFile } from 'node:fs/promises';
+import { basename, extname, join } from 'node:path';
+
+import type { CdpConnection } from './browserAutomation.js';
+
+export type BrowserPageAsset = {
+  kind:
+    | 'image'
+    | 'video'
+    | 'audio'
+    | 'stylesheet'
+    | 'script'
+    | 'font'
+    | 'icon'
+    | 'other';
+  url: string;
+  source: 'dom' | 'performance' | 'inline';
+  name?: string;
+  alt?: string;
+  type?: string;
+  width?: number;
+  height?: number;
+  inline?: boolean;
+  content?: string;
+};
+
+export type BrowserPageContent = {
+  url: string;
+  title: string;
+  text: string;
+  html?: string;
+  format:
+    | 'text'
+    | 'txt'
+    | 'markdown'
+    | 'md'
+    | 'html'
+    | 'json'
+    | 'page'
+    | 'google-workspace'
+    | 'youtube-transcript'
+    | 'pdf'
+    | 'docx'
+    | 'xlsx'
+    | 'csv'
+    | 'pptx';
+};
+
+export type BrowserWorkspaceExportFormat =
+  | 'pdf'
+  | 'docx'
+  | 'xlsx'
+  | 'csv'
+  | 'pptx';
+
+/**
+ * The browser evaluates these fixed expressions. They are deliberately not
+ * user supplied JavaScript: page inspection and clipboard are first-class
+ * browser capabilities, while arbitrary eval remains a separate developer
+ * action.
+ */
+export const PAGE_ASSETS_EXPRESSION = `(() => {
+  const absolute = (value) => { try { return new URL(value, location.href).href; } catch { return ''; } };
+  const result = [];
+  const add = (kind, value, source, extra = {}) => {
+    const url = absolute(value);
+    if (!url || url.startsWith('javascript:')) return;
+    result.push({ kind, url, source, ...extra });
+  };
+  for (const el of document.images) add('image', el.currentSrc || el.src, 'dom', {
+    name: el.getAttribute('name') || undefined,
+    alt: el.alt || undefined,
+    width: Math.round(el.naturalWidth || el.getBoundingClientRect().width || 0) || undefined,
+    height: Math.round(el.naturalHeight || el.getBoundingClientRect().height || 0) || undefined,
+  });
+  for (const el of document.querySelectorAll('video, audio')) add(el.tagName.toLowerCase(), el.currentSrc || el.src, 'dom', {
+    type: el.getAttribute('type') || undefined,
+    width: Math.round(el.getBoundingClientRect().width || 0) || undefined,
+    height: Math.round(el.getBoundingClientRect().height || 0) || undefined,
+  });
+  for (const el of document.querySelectorAll('source')) add(el.parentElement?.tagName.toLowerCase() === 'audio' ? 'audio' : 'video', el.src, 'dom', { type: el.type || undefined });
+  for (const el of document.querySelectorAll('link[rel~="stylesheet"], link[rel~="icon"], link[rel="preload"]')) {
+    const rel = (el.rel || '').toLowerCase();
+    add(rel.includes('icon') ? 'icon' : rel.includes('stylesheet') ? 'stylesheet' : 'other', el.href, 'dom', { type: el.as || el.type || undefined });
+  }
+  for (const el of document.querySelectorAll('script[src]')) add('script', el.src, 'dom', { type: el.type || undefined });
+  for (const el of document.querySelectorAll('style')) {
+    const text = el.textContent || '';
+    if (text) result.push({ kind: 'stylesheet', url: 'inline:style-' + result.length, source: 'inline', inline: true, type: 'text/css', content: text.slice(0, 200000) });
+  }
+  for (const entry of performance.getEntriesByType('resource')) {
+    const name = entry.name || '';
+    if (/\\.(?:woff2?|ttf|otf)(?:[?#]|$)/i.test(name)) add('font', name, 'performance');
+  }
+  const seen = new Set();
+  return result.filter((item) => { const key = item.kind + '|' + item.url; if (seen.has(key)) return false; seen.add(key); return true; });
+})()`;
+
+export const CLIPBOARD_READ_TEXT_EXPRESSION = 'navigator.clipboard.readText()';
+
+/** Fetch one page asset inside the authenticated browser context. */
+export function browserAssetFetchExpression(url: string): string {
+  return `(async () => {
+    const response = await fetch(${JSON.stringify(url)}, { credentials: 'include' });
+    const bytes = new Uint8Array(await response.arrayBuffer());
+    let binary = '';
+    const chunk = 0x8000;
+    for (let i = 0; i < bytes.length; i += chunk) binary += String.fromCharCode(...bytes.subarray(i, i + chunk));
+    return { ok: response.ok, status: response.status, type: response.headers.get('content-type') || '', base64: btoa(binary) };
+  })()`;
+}
+
+export const CLIPBOARD_READ_EXPRESSION = `(() => navigator.clipboard.read().then(async (items) => {
+  const result = [];
+  for (const item of items) {
+    for (const type of item.types) {
+      const blob = await item.getType(type);
+      const bytes = new Uint8Array(await blob.arrayBuffer());
+      let binary = '';
+      const chunk = 0x8000;
+      for (let i = 0; i < bytes.length; i += chunk) binary += String.fromCharCode(...bytes.subarray(i, i + chunk));
+      result.push({ type, base64: btoa(binary), size: bytes.length });
+    }
+  }
+  return result;
+}))()`;
+
+export function clipboardWriteExpression(
+  items: Array<{ type: string; base64: string }>,
+): string {
+  return `(async () => {
+    const items = ${JSON.stringify(items)};
+    await navigator.clipboard.write([new ClipboardItem(Object.fromEntries(items.map((item) => [item.type, fetch('data:' + item.type + ';base64,' + item.base64).then((response) => response.blob())])))]);
+    return true;
+  })()`;
+}
+
+export function clipboardWriteTextExpression(value: string): string {
+  return `navigator.clipboard.writeText(${JSON.stringify(value)}).then(() => true)`;
+}
+
+export function pageContentExpression(
+  format: BrowserPageContent['format'],
+  selector?: string,
+): string {
+  if (['pdf', 'docx', 'xlsx', 'csv', 'pptx'].includes(format)) {
+    return googleWorkspaceExportExpression(
+      format as BrowserWorkspaceExportFormat,
+    );
+  }
+  const selectorLiteral = JSON.stringify(selector || 'body');
+  return `(() => {
+    const root = document.querySelector(${selectorLiteral}) || document.body;
+    const text = (root.innerText || root.textContent || '').trim();
+    const html = root.outerHTML || '';
+    const title = document.title || '';
+    const url = location.href;
+    const value = { url, title, text, html, format: ${JSON.stringify(format)} };
+    if (${JSON.stringify(format)} === 'youtube-transcript') {
+      value.text = Array.from(document.querySelectorAll('ytd-transcript-segment-renderer, [data-purpose="transcript-segment"], .transcript-segment')).map((el) => (el.innerText || el.textContent || '').trim()).filter(Boolean).join('\\n') || text;
+    }
+    if (${JSON.stringify(format)} === 'google-workspace') {
+      value.text = Array.from(root.querySelectorAll('[contenteditable="true"], [role="textbox"], p, li, table')).map((el) => (el.innerText || el.textContent || '').trim()).filter(Boolean).join('\\n');
+    }
+    if (['text', 'txt', 'markdown', 'md', 'youtube-transcript', 'google-workspace'].includes(${JSON.stringify(format)})) delete value.html;
+    return value;
+  })()`;
+}
+
+/**
+ * Export an authenticated Google Workspace document from the current tab.
+ * The browser performs the request so the current login/session cookies are
+ * preserved; the agent never receives or stores cookies itself. The result is
+ * a compact base64 envelope consumed by BrowserBatch's deliverable writer.
+ */
+export function googleWorkspaceExportExpression(
+  format: BrowserWorkspaceExportFormat,
+): string {
+  return `(() => {
+    const href = location.href;
+    const match = href.match(/https?:\\/\\/(?:docs|sheets|slides)\\.google\\.com\\/(document|spreadsheets|presentation)\\/d\\/([^/]+)/i);
+    if (!match) return { error: '当前页面不是可导出的 Google Workspace 文档' };
+    const kind = match[1];
+    const id = match[2];
+    const format = ${JSON.stringify(format)};
+    const base = kind === 'document'
+      ? 'https://docs.google.com/document/d/' + id + '/export?format=' + format
+      : kind === 'spreadsheets'
+        ? 'https://docs.google.com/spreadsheets/d/' + id + '/export?format=' + format
+        : format === 'pptx'
+          ? 'https://docs.google.com/presentation/d/' + id + '/export/pptx'
+          : 'https://docs.google.com/presentation/d/' + id + '/export?format=' + format;
+    return fetch(base, { credentials: 'include' }).then(async (response) => {
+      if (!response.ok) return { error: 'Workspace 导出失败 HTTP ' + response.status, url: base };
+      const bytes = new Uint8Array(await response.arrayBuffer());
+      let binary = '';
+      const chunk = 0x8000;
+      for (let i = 0; i < bytes.length; i += chunk) binary += String.fromCharCode(...bytes.subarray(i, i + chunk));
+      return { __browser_binary_base64: btoa(binary), format, url: base, size: bytes.length };
+    });
+  })()`;
+}
+
+export function markdownFromPageContent(
+  content: Pick<BrowserPageContent, 'title' | 'url' | 'text'>,
+): string {
+  const heading = content.title ? `# ${content.title}\n\n` : '';
+  return `${`${heading}${content.text}${content.url ? `\n\nSource: ${content.url}` : ''}`.trim()}\n`;
+}
+
+export function assetFileName(asset: BrowserPageAsset, index: number): string {
+  if (asset.url.startsWith('inline:'))
+    return `${asset.url.slice('inline:'.length)}.css`;
+  try {
+    const parsed = new URL(asset.url);
+    const name = basename(parsed.pathname) || `asset-${index}`;
+    const extension =
+      extname(name) ||
+      (asset.kind === 'stylesheet'
+        ? '.css'
+        : asset.kind === 'script'
+          ? '.js'
+          : '');
+    return `${String(index + 1).padStart(3, '0')}-${name.replace(/[^a-zA-Z0-9._-]/g, '_')}${extension && !name.endsWith(extension) ? extension : ''}`;
+  } catch {
+    return `${String(index + 1).padStart(3, '0')}-asset`;
+  }
+}
+
+export function assetBundleManifest(
+  assets: BrowserPageAsset[],
+  directory: string,
+): string {
+  return `${JSON.stringify(
+    { version: 1, directory: join(directory, 'assets'), assets },
+    null,
+    2,
+  )}\n`;
+}
+
+export async function bundlePageAssets(
+  assets: BrowserPageAsset[],
+  directory: string,
+  fetchImpl: (url: string) => Promise<Response> = (url) => fetch(url),
+): Promise<{
+  directory: string;
+  files: string[];
+  failures: Array<{ url: string; error: string }>;
+}> {
+  const assetDirectory = join(directory, 'assets');
+  await mkdir(assetDirectory, { recursive: true, mode: 0o700 });
+  const files: string[] = [];
+  const failures: Array<{ url: string; error: string }> = [];
+  for (const [index, asset] of assets.entries()) {
+    if (asset.inline && asset.content) {
+      const path = join(assetDirectory, assetFileName(asset, index));
+      await writeFile(path, asset.content, { mode: 0o600 });
+      files.push(path);
+      continue;
+    }
+    if (!/^https?:\/\//i.test(asset.url)) continue;
+    try {
+      const response = await fetchImpl(asset.url);
+      if (!response.ok) throw new Error(`HTTP ${response.status}`);
+      const path = join(assetDirectory, assetFileName(asset, index));
+      await writeFile(path, Buffer.from(await response.arrayBuffer()), {
+        mode: 0o600,
+      });
+      files.push(path);
+    } catch (error) {
+      failures.push({
+        url: asset.url,
+        error: error instanceof Error ? error.message : String(error),
+      });
+    }
+  }
+  await writeFile(
+    join(directory, 'manifest.json'),
+    assetBundleManifest(assets, directory),
+    { mode: 0o600 },
+  );
+  return { directory, files, failures };
+}
+
+function cdpValue(result: unknown): unknown {
+  if (!result || typeof result !== 'object') return undefined;
+  const value = (result as { result?: { value?: unknown } }).result?.value;
+  return value;
+}
+
+/** Read URLs in short-lived background targets and always close those targets. */
+export async function readUrlsInBackground(
+  cdp: CdpConnection,
+  urls: string[],
+  timeoutMs: number,
+  contentType: 'text' | 'html' | 'domSnapshot' = 'text',
+  concurrency = 3,
+  signal?: AbortSignal,
+): Promise<
+  Array<{
+    url: string;
+    finalUrl?: string;
+    title?: string;
+    text?: string;
+    error?: string;
+  }>
+> {
+  const results: Array<{
+    url: string;
+    finalUrl?: string;
+    title?: string;
+    text?: string;
+    error?: string;
+  }> = [];
+  let next = 0;
+  const worker = async () => {
+    while (true) {
+      const index = next++;
+      if (index >= urls.length) return;
+      if (signal?.aborted) return;
+      const url = urls[index]!;
+      let targetId = '';
+      try {
+        const created = (await cdp.send(
+          'Target.createTarget',
+          { url },
+          undefined,
+          timeoutMs,
+          signal,
+        )) as {
+          targetId?: string;
+        };
+        targetId = created.targetId ?? '';
+        if (!targetId) throw new Error('浏览器未返回后台标签 ID');
+        const attached = (await cdp.send(
+          'Target.attachToTarget',
+          { targetId, flatten: true },
+          undefined,
+          timeoutMs,
+          signal,
+        )) as { sessionId?: string };
+        const sessionId = attached.sessionId;
+        if (!sessionId) throw new Error('无法连接后台标签');
+        const deadline = Date.now() + timeoutMs;
+        let state: unknown;
+        while (Date.now() < deadline) {
+          state = cdpValue(
+            await cdp
+              .send(
+                'Runtime.evaluate',
+                {
+                  expression:
+                    '({ready: document.readyState, url: location.href, title: document.title})',
+                  returnByValue: true,
+                },
+                sessionId,
+                timeoutMs,
+                signal,
+              )
+              .catch(() => ({})),
+          );
+          if (
+            state &&
+            typeof state === 'object' &&
+            ['interactive', 'complete'].includes(
+              String((state as { ready?: unknown }).ready),
+            )
+          )
+            break;
+          await new Promise((resolve) => setTimeout(resolve, 100));
+        }
+        let page: unknown;
+        if (contentType === 'domSnapshot') {
+          const ax = await cdp
+            .send('Accessibility.getFullAXTree', {}, sessionId)
+            .catch(() => undefined);
+          page = {
+            url: String((state as { url?: unknown } | undefined)?.url ?? url),
+            title: String(
+              (state as { title?: unknown } | undefined)?.title ?? '',
+            ),
+            text: JSON.stringify(ax ?? { nodes: [] }).slice(0, 300000),
+          };
+        } else {
+          const expression =
+            contentType === 'html'
+              ? `({url: location.href, title: document.title, text: document.documentElement?.outerHTML?.slice(0, 300000) || ''})`
+              : `({url: location.href, title: document.title, text: (document.body?.innerText || document.body?.textContent || '').slice(0, 200000)})`;
+          page = cdpValue(
+            await cdp.send(
+              'Runtime.evaluate',
+              {
+                expression,
+                returnByValue: true,
+              },
+              sessionId,
+              timeoutMs,
+              signal,
+            ),
+          );
+        }
+        if (!page || typeof page !== 'object')
+          throw new Error('无法读取后台页面内容');
+        results[index] = {
+          url,
+          finalUrl: String((page as { url?: unknown }).url ?? ''),
+          title: String((page as { title?: unknown }).title ?? ''),
+          text: String((page as { text?: unknown }).text ?? ''),
+        };
+      } catch (error) {
+        results[index] = {
+          url,
+          error: error instanceof Error ? error.message : String(error),
+        };
+      } finally {
+        if (targetId)
+          await cdp
+            .send(
+              'Target.closeTarget',
+              { targetId },
+              undefined,
+              timeoutMs,
+              signal,
+            )
+            .catch(() => undefined);
+      }
+    }
+  };
+  await Promise.all(
+    Array.from(
+      { length: Math.max(1, Math.min(concurrency, urls.length)) },
+      () => worker(),
+    ),
+  );
+  return results.filter((result): result is NonNullable<typeof result> =>
+    Boolean(result),
+  );
+}