@pi-harness/pi-harness 0.1.13 → 0.1.14

package/packages/core/src/plugins/browser-fetch.ts

@@ -1,54 +1,162 @@
+import type { LookupAddress } from "node:dns";
 import { lookup } from "node:dns/promises";
-import { isIP } from "node:net";
+import { BlockList, isIP, type LookupFunction } from "node:net";
 import type { Context } from "@deepseek-ai/cordis";
 import z from "@deepseek-ai/schemastery";
 import { Type } from "@earendil-works/pi-ai";
 import { defineTool, type AgentToolResult } from "@earendil-works/pi-coding-agent";
+import { Agent } from "undici";
 
 const maxResponseBytes = 512 * 1024;
+const maxPanelTextChars = 12_000;
 const maxRedirects = 3;
-const requestTimeoutMs = 20_000;
+const maxUrlLength = 4096;
+const defaultRequestTimeoutMs = 20_000;
+const redirectStatuses = new Set([301, 302, 303, 307, 308]);
+const textualApplicationTypes = new Set([
+  "application/ecmascript",
+  "application/graphql",
+  "application/javascript",
+  "application/json",
+  "application/sql",
+  "application/x-httpd-php",
+  "application/x-ndjson",
+  "application/x-www-form-urlencoded",
+  "application/xml",
+]);
 
 type BrowserFetchResult = { url: string; finalUrl: string; status: number; contentType: string; bytes: number; truncated: boolean; text: string };
+type ValidatedTarget = { url: URL; addresses?: LookupAddress[] };
 
-function privateIpv4(address: string): boolean {
-  const octets = address.split(".").map(Number);
-  if (octets.length !== 4 || octets.some((octet) => !Number.isInteger(octet) || octet < 0 || octet > 255)) return true;
-  const first = octets[0] ?? -1;
-  const second = octets[1] ?? -1;
-  return (
-    first === 0 ||
-    first === 10 ||
-    first === 127 ||
-    (first === 100 && second >= 64 && second <= 127) ||
-    (first === 169 && second === 254) ||
-    (first === 172 && second >= 16 && second <= 31) ||
-    (first === 192 && second === 0) ||
-    (first === 192 && second === 168) ||
-    (first === 198 && (second === 18 || second === 19)) ||
-    first >= 224
-  );
-}
+const nonPublicIpv4Networks = new BlockList();
+for (const [network, prefix] of [
+  ["0.0.0.0", 8],
+  ["10.0.0.0", 8],
+  ["100.64.0.0", 10],
+  ["127.0.0.0", 8],
+  ["169.254.0.0", 16],
+  ["172.16.0.0", 12],
+  ["192.0.0.0", 24],
+  ["192.0.2.0", 24],
+  ["192.88.99.0", 24],
+  ["192.168.0.0", 16],
+  ["198.18.0.0", 15],
+  ["198.51.100.0", 24],
+  ["203.0.113.0", 24],
+  ["224.0.0.0", 4],
+  ["240.0.0.0", 4],
+] as const)
+  nonPublicIpv4Networks.addSubnet(network, prefix, "ipv4");
+
+const globallyReachableIpv4SpecialNetworks = new BlockList();
+globallyReachableIpv4SpecialNetworks.addAddress("192.0.0.9", "ipv4");
+globallyReachableIpv4SpecialNetworks.addAddress("192.0.0.10", "ipv4");
+
+const globalIpv6UnicastNetworks = new BlockList();
+globalIpv6UnicastNetworks.addSubnet("2000::", 3, "ipv6");
+
+const globallyReachableIpv6SpecialNetworks = new BlockList();
+for (const [network, prefix] of [
+  ["2001:1::1", 128],
+  ["2001:1::2", 128],
+  ["2001:1::3", 128],
+  ["2001:3::", 32],
+  ["2001:4:112::", 48],
+  ["2001:20::", 28],
+  ["2001:30::", 28],
+] as const)
+  globallyReachableIpv6SpecialNetworks.addSubnet(network, prefix, "ipv6");
+
+const nonPublicIpv6Networks = new BlockList();
+for (const [network, prefix] of [
+  ["::", 128],
+  ["::1", 128],
+  ["::ffff:0:0", 96],
+  ["64:ff9b::", 96],
+  ["64:ff9b:1::", 48],
+  ["100::", 64],
+  ["2001::", 23],
+  ["2001:db8::", 32],
+  ["2002::", 16],
+  ["3fff::", 20],
+  ["fc00::", 7],
+  ["fe80::", 10],
+  ["fec0::", 10],
+  ["ff00::", 8],
+] as const)
+  nonPublicIpv6Networks.addSubnet(network, prefix, "ipv6");
 
 function privateIp(address: string): boolean {
-  if (isIP(address) === 4) return privateIpv4(address);
-  const normalized = address.toLowerCase();
-  return (
-    normalized === "::1" ||
-    normalized.startsWith("fc") ||
-    normalized.startsWith("fd") ||
-    normalized.startsWith("fe8") ||
-    normalized.startsWith("fe9") ||
-    normalized.startsWith("fea") ||
-    normalized.startsWith("feb") ||
-    normalized.startsWith("::ffff:10.") ||
-    normalized.startsWith("::ffff:192.168.") ||
-    normalized.startsWith("::ffff:127.")
-  );
+  const family = isIP(address);
+  if (family === 0) return true;
+  if (family === 4) {
+    if (globallyReachableIpv4SpecialNetworks.check(address, "ipv4")) return false;
+    return nonPublicIpv4Networks.check(address, "ipv4");
+  }
+  if (!globalIpv6UnicastNetworks.check(address, "ipv6")) return true;
+  if (globallyReachableIpv6SpecialNetworks.check(address, "ipv6")) return false;
+  return nonPublicIpv6Networks.check(address, "ipv6");
 }
 
-async function validateTarget(rawUrl: string, allowPrivate: boolean): Promise<URL> {
-  if (rawUrl.length === 0 || rawUrl.length > 4096) throw new Error("Browser URL must be between 1 and 4096 characters");
+function abortError(signal: AbortSignal): Error {
+  return signal.reason instanceof Error ? signal.reason : new Error("Browser fetch aborted", { cause: signal.reason });
+}
+
+function throwIfAborted(signal: AbortSignal): void {
+  if (signal.aborted) throw abortError(signal);
+}
+
+function urlParameter(value: unknown): unknown {
+  if (value === null || typeof value !== "object" || Array.isArray(value)) return undefined;
+  let descriptors: PropertyDescriptorMap;
+  try {
+    const prototype = Object.getPrototypeOf(value) as unknown;
+    if (prototype !== Object.prototype && prototype !== null) throw new Error("Browser fetch parameters must be a plain object");
+    descriptors = Object.getOwnPropertyDescriptors(value);
+  } catch (error) {
+    if (error instanceof Error && error.message === "Browser fetch parameters must be a plain object") throw error;
+    throw new Error("Browser fetch parameters must be an accessible plain object", { cause: error });
+  }
+  if (Reflect.ownKeys(descriptors).some((key) => key !== "url")) throw new Error("Browser fetch parameters contain an unknown property");
+  const descriptor = descriptors.url;
+  if (descriptor === undefined) return undefined;
+  if (!("value" in descriptor)) throw new Error("Browser fetch parameters must use data properties");
+  return descriptor.value as unknown;
+}
+
+function abortable<T>(operation: Promise<T>, signal: AbortSignal): Promise<T> {
+  return new Promise<T>((resolve, reject) => {
+    let settled = false;
+    const cleanup = () => signal.removeEventListener("abort", onAbort);
+    const onAbort = () => {
+      if (settled) return;
+      settled = true;
+      cleanup();
+      reject(abortError(signal));
+    };
+    signal.addEventListener("abort", onAbort, { once: true });
+    if (signal.aborted) onAbort();
+    void operation.then(
+      (value) => {
+        if (settled) return;
+        settled = true;
+        cleanup();
+        resolve(value);
+      },
+      (error: unknown) => {
+        if (settled) return;
+        settled = true;
+        cleanup();
+        reject(error instanceof Error ? error : new Error("Browser target resolution failed", { cause: error }));
+      },
+    );
+  });
+}
+
+async function validateTarget(rawUrl: unknown, allowPrivate: boolean, signal: AbortSignal): Promise<ValidatedTarget> {
+  throwIfAborted(signal);
+  if (typeof rawUrl !== "string") throw new Error("Browser URL must be a string");
+  if (rawUrl.length === 0 || rawUrl.length > maxUrlLength) throw new Error(`Browser URL must be between 1 and ${maxUrlLength} characters`);
   let url: URL;
   try {
     url = new URL(rawUrl);
@@ -58,10 +166,36 @@ async function validateTarget(rawUrl: string, allowPrivate: boolean): Promise<UR
   if (url.protocol !== "http:" && url.protocol !== "https:") throw new Error("Browser fetch only supports http and https URLs");
   if (url.username !== "" || url.password !== "") throw new Error("Browser URL must not contain credentials");
   if (!allowPrivate) {
-    const addresses = await lookup(url.hostname, { all: true, verbatim: true });
+    const hostname = url.hostname.startsWith("[") && url.hostname.endsWith("]") ? url.hostname.slice(1, -1) : url.hostname;
+    const addresses = await abortable(lookup(hostname, { all: true, verbatim: true }), signal);
+    if (addresses.length === 0) throw new Error("Browser target did not resolve to an IP address");
     if (addresses.some(({ address }) => privateIp(address))) throw new Error("Browser fetch blocked a private or local network target");
+    return { url, addresses };
   }
-  return url;
+  return { url };
+}
+
+function createPinnedLookup(addresses: readonly LookupAddress[]): LookupFunction {
+  return (_hostname, options, callback) => {
+    if (options.all === true) {
+      callback(
+        null,
+        addresses.map(({ address, family }) => ({ address, family })),
+      );
+      return;
+    }
+    const requestedFamily = options.family === "IPv4" ? 4 : options.family === "IPv6" ? 6 : options.family;
+    const selected = addresses.find(({ family }) => requestedFamily === undefined || requestedFamily === 0 || requestedFamily === family) ?? addresses[0]!;
+    callback(null, selected.address, selected.family);
+  };
+}
+
+function contentType(response: Response): string {
+  return (response.headers.get("content-type") ?? "text/plain").split(";", 1)[0]!.trim().toLowerCase();
+}
+
+function textualContentType(value: string): boolean {
+  return value.startsWith("text/") || value.endsWith("+json") || value.endsWith("+xml") || value === "image/svg+xml" || textualApplicationTypes.has(value);
 }
 
 async function readBody(response: Response): Promise<{ bytes: number; truncated: boolean; text: string }> {
@@ -89,81 +223,138 @@ async function readBody(response: Response): Promise<{ bytes: number; truncated:
   } finally {
     reader.releaseLock();
   }
-  return { bytes, truncated, text: Buffer.concat(chunks.map((chunk) => Buffer.from(chunk))).toString("utf8") };
+  try {
+    const decoder = new TextDecoder("utf-8", { fatal: true });
+    const text = decoder.decode(Buffer.concat(chunks.map((chunk) => Buffer.from(chunk))));
+    return { bytes, truncated, text };
+  } catch (error) {
+    throw new Error("Browser response body is not valid UTF-8", { cause: error });
+  }
 }
 
-async function fetchPage(rawUrl: string, allowPrivate: boolean): Promise<BrowserFetchResult> {
-  let current = await validateTarget(rawUrl, allowPrivate);
-  const original = current.toString();
-  for (let redirect = 0; redirect <= maxRedirects; redirect += 1) {
-    const controller = new AbortController();
-    const timer = setTimeout(() => controller.abort(), requestTimeoutMs);
-    let response: Response;
-    try {
-      response = await fetch(current, {
-        redirect: "manual",
-        signal: controller.signal,
-        headers: { accept: "text/html, text/plain, application/json;q=0.9, */*;q=0.1", "user-agent": "pi-harness-browser-fetch/0.1" },
-      });
-    } catch (error) {
-      if (error instanceof Error && error.name === "AbortError") throw new Error("Browser fetch timed out after 20 seconds", { cause: error });
-      throw error;
-    } finally {
-      clearTimeout(timer);
-    }
-    if (response.status >= 300 && response.status < 400) {
-      const location = response.headers.get("location");
-      if (location === null) throw new Error(`Browser redirect ${response.status} has no Location header`);
-      if (redirect === maxRedirects) throw new Error(`Browser fetch exceeded the ${maxRedirects}-redirect limit`);
-      current = await validateTarget(new URL(location, current).toString(), allowPrivate);
-      continue;
+async function fetchPage(rawUrl: unknown, allowPrivate: boolean, timeoutMs: number, signal?: AbortSignal): Promise<BrowserFetchResult> {
+  if (signal !== undefined) throwIfAborted(signal);
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  const requestSignal = signal === undefined ? controller.signal : AbortSignal.any([signal, controller.signal]);
+  try {
+    let current = await validateTarget(rawUrl, allowPrivate, requestSignal);
+    const original = current.url.toString();
+    for (let redirect = 0; redirect <= maxRedirects; redirect += 1) {
+      const dispatcher = current.addresses === undefined ? undefined : new Agent({ connect: { lookup: createPinnedLookup(current.addresses) } });
+      try {
+        const requestInit: RequestInit & { dispatcher?: Agent } = {
+          redirect: "manual",
+          signal: requestSignal,
+          ...(dispatcher === undefined ? {} : { dispatcher }),
+          headers: { accept: "text/html, text/plain, application/json;q=0.9, */*;q=0.1", "user-agent": "pi-harness-browser-fetch/0.1" },
+        };
+        const response = await fetch(current.url, requestInit);
+        if (redirectStatuses.has(response.status)) {
+          await response.body?.cancel?.();
+          const location = response.headers.get("location");
+          if (location === null) throw new Error(`Browser redirect ${response.status} has no Location header`);
+          if (redirect === maxRedirects) throw new Error(`Browser fetch exceeded the ${maxRedirects}-redirect limit`);
+          current = await validateTarget(new URL(location, current.url).toString(), allowPrivate, requestSignal);
+          continue;
+        }
+        const responseContentType = contentType(response);
+        if (!textualContentType(responseContentType)) {
+          await response.body?.cancel?.();
+          throw new Error(`Browser fetch rejected unsupported content type: ${responseContentType}`);
+        }
+        const body = await readBody(response);
+        return {
+          url: original,
+          finalUrl: current.url.toString(),
+          status: response.status,
+          contentType: responseContentType,
+          ...body,
+        };
+      } finally {
+        await dispatcher?.close();
+      }
     }
-    const body = await readBody(response);
-    return {
-      url: original,
-      finalUrl: current.toString(),
-      status: response.status,
-      contentType: (response.headers.get("content-type") ?? "text/plain").split(";", 1)[0]!.trim(),
-      ...body,
-    };
+    throw new Error("Browser fetch did not produce a response");
+  } catch (error) {
+    if (signal?.aborted === true) throw abortError(signal);
+    if (controller.signal.aborted) throw new Error(`Browser fetch timed out after ${timeoutMs} ms`, { cause: error });
+    throw error;
+  } finally {
+    clearTimeout(timer);
   }
-  throw new Error("Browser fetch did not produce a response");
 }
 
 export interface BrowserFetchPluginConfig {
   allowPrivate?: boolean;
+  timeoutMs?: number;
 }
 
-export const Config: z<BrowserFetchPluginConfig> = z.object({ allowPrivate: z.boolean().default(false) });
+export const Config: z<BrowserFetchPluginConfig> = z.object({
+  allowPrivate: z.boolean().default(false),
+  timeoutMs: z.number().min(100).max(60_000).step(1).default(defaultRequestTimeoutMs),
+});
 
 export default {
   name: "pi-browser-fetch",
   inject: ["piHarnessLaunch", "piPluginUi", "piTools"],
   Config,
   apply(context: Context, config: BrowserFetchPluginConfig) {
+    const lifecycle = new AbortController();
+    const configuredTimeoutMs = config.timeoutMs ?? defaultRequestTimeoutMs;
+    const timeoutMs = Number.isFinite(configuredTimeoutMs) ? Math.max(100, Math.min(60_000, Math.trunc(configuredTimeoutMs))) : defaultRequestTimeoutMs;
     let latest: BrowserFetchResult | undefined;
-    const unregisterTool = context.piTools.register(
-      defineTool({
-        name: "browser_fetch",
-        label: "Browser fetch",
-        description: "Fetch a public HTTP or HTTPS page as bounded text without executing page scripts.",
-        promptSnippet: "fetch a public web page for inspection",
-        parameters: Type.Object({ url: Type.String({ description: "HTTP or HTTPS URL" }) }),
-        async execute(_toolCallId, params): Promise<AgentToolResult<BrowserFetchResult>> {
-          latest = await fetchPage(params.url, config.allowPrivate === true);
-          return { content: [{ type: "text", text: latest.text }], details: latest };
-        },
-      }),
-    );
-    const disposePanel = context.piPluginUi.register({
-      id: "browser-fetch-panel",
-      pluginId: "@pi-harness/core/plugins/browser-fetch",
-      title: "Browser Fetch",
-      description: "受限抓取公开网页文本，不执行页面脚本。",
-      icon: "◎",
-      read: () => ({ latest: latest ?? null, allowPrivate: config.allowPrivate === true, maxResponseBytes }),
-    });
+    let unregisterTool: () => void = () => undefined;
+    let disposePanel: () => void = () => undefined;
+    try {
+      unregisterTool = context.piTools.register(
+        defineTool({
+          name: "browser_fetch",
+          label: "Browser fetch",
+          description: "Fetch a public HTTP or HTTPS page as bounded text without executing page scripts.",
+          promptSnippet: "fetch a public web page for inspection",
+          parameters: Type.Object(
+            { url: Type.String({ description: "HTTP or HTTPS URL", minLength: 1, maxLength: maxUrlLength }) },
+            { additionalProperties: false },
+          ),
+          executionMode: "sequential",
+          async execute(_toolCallId, params, signal): Promise<AgentToolResult<BrowserFetchResult>> {
+            const executionSignal = signal === undefined ? lifecycle.signal : AbortSignal.any([signal, lifecycle.signal]);
+            latest = await fetchPage(urlParameter(params), config.allowPrivate === true, timeoutMs, executionSignal);
+            return { content: [{ type: "text", text: latest.text }], details: structuredClone(latest) };
+          },
+        }),
+      );
+      disposePanel = context.piPluginUi.register({
+        id: "browser-fetch-panel",
+        pluginId: "@pi-harness/core/plugins/browser-fetch",
+        title: "Browser Fetch",
+        description: "受限抓取公开网页文本，不执行页面脚本。",
+        icon: "◎",
+        read: () => ({
+          latest:
+            latest === undefined
+              ? null
+              : {
+                  ...latest,
+                  text: latest.text.slice(0, maxPanelTextChars),
+                  previewTruncated: latest.text.length > maxPanelTextChars,
+                },
+          allowPrivate: config.allowPrivate === true,
+          maxResponseBytes,
+          maxPanelTextChars,
+          maxRedirects,
+          timeoutMs,
+        }),
+      });
+    } catch (error) {
+      disposePanel();
+      unregisterTool();
+      lifecycle.abort(new Error("Browser fetch plugin registration failed"));
+      throw error;
+    }
     context.effect(() => () => {
+      lifecycle.abort(new Error("Browser fetch plugin disposed"));
       unregisterTool();
       disposePanel();
     });