@staix/agent-hub 0.19.0 → 0.21.0

package/src/hub/workflow.ts

@@ -11,7 +11,7 @@ export interface CriteriaReceipt {
 /** A checklist stored under `phase` before #481 S8. Its task list is shown as ignored; board split-undo still treats it as a reference. */
 export interface LegacyPhaseReceipt extends CriteriaReceipt { tasks: { id: number; after: number[] }[]; }
 const name = (s: unknown): s is string => typeof s === "string" && /^[a-z][a-z0-9-]{0,47}$/.test(s);
-const criterionId = (s: unknown): s is string => typeof s === "string" && /^[A-Za-z][A-Za-z0-9-]{0,47}$/.test(s);
+export const criterionId = (s: unknown): s is string => typeof s === "string" && /^[A-Za-z][A-Za-z0-9-]{0,47}$/.test(s);
 const hash = (s: unknown): s is string => typeof s === "string" && /^[0-9a-f]{40}$/.test(s);
 const fields = (v: unknown, keys: string[], label: string): Record<string, unknown> => {
   if (!v || typeof v !== "object" || Array.isArray(v) || Object.keys(v).some(k => !keys.includes(k))) throw new Error(`${label}: invalid fields`);

package/src/local/deny.ts

@@ -163,6 +163,27 @@ export function hubWriteRegexes(root: string, gitDirs: string[] = [], moduleGit:
   ];
 }
 
+/**
+ * Full SBPL rule lines for the shared object store `<common>/objects` (#864). Every checkout of the repository, and the hub's own host add/commit/push (#845), trusts what is
+ * there without rehashing it, so a sandboxed command may only ADD loose objects: `git add` and `git commit` keep working, and git skips writing an object that exists. Refused:
+ * every write under `objects/info` (alternates, http-alternates, commit-graph) and `objects/pack` (a new pack wins over a loose copy of the same id, since git reads packs
+ * first); the `objects`, `objects/info` and `objects/pack` entries themselves (no rename, replace or removal); removing or renaming a fan-out directory; and changing,
+ * removing, chmod-ing or renaming over an existing loose object. The cost: `git gc`, `git repack`, `git fetch`, `git maintenance` and `git commit-graph write` do not run in
+ * the sandbox. A hardlink to an existing loose object is refused under these rules (verified, into the project and into the command's own temp dir). Residual: an object id
+ * that does not exist yet can be planted before git writes it (the attacker must predict its exact content); a local checkout shows it, a push of it is rejected by the
+ * receiving side's rehash. Seatbelt ignores `{n}`
+ * repeats, so the two fan-out hex digits are written out (verified with the real sandbox-exec).
+ */
+export function objectStoreRules(common: string): string[] {
+  const objects = `${common}/objects`, o = sbplEscape(objects);
+  const re = (s: string) => `(regex ${sbplString(s)})`;
+  return [
+    `(deny file-write* (subpath ${sbplString(`${objects}/info`)}) (subpath ${sbplString(`${objects}/pack`)}) ${[objects, `${objects}/info`, `${objects}/pack`].map((p) => `(literal ${sbplString(p)})`).join(" ")})`,
+    `(deny file-write-unlink ${re(`^${o}/[0-9a-f][0-9a-f]$`)})`,
+    `(deny file-write-data file-write-unlink file-write-mode file-write-flags ${re(`^${o}/[0-9a-f][0-9a-f]/[0-9a-f]+$`)})`,
+  ];
+}
+
 /**
  * Seatbelt regex filters equivalent to isDenied. Seatbelt sees absolute paths, so the `local.deny` substrings are
  * anchored under the project root: a bare "private/" would otherwise match /private/var/... and deny the whole temp tree.

package/src/local/doc-read.ts

@@ -0,0 +1,69 @@
+import { lstatSync } from "node:fs";
+import { extname } from "node:path";
+import { sanitize } from "../hub/envelope.ts";
+import { matchesPolicy } from "../hub/path-policy.ts";
+import { MAX_PROJECTION_INPUT, type ProjectedText } from "../hub/text-projection.ts";
+import { OUTPUT_CAP } from "./sandbox.ts";
+
+// agent-hub #515 (hub task #320): the local worker's `doc_read`. Spec: "On-prem web reading (#515)", AC2 and AC3 for doc_read.
+// The model names a path and nothing else: the converter, its arguments and its sandbox are the person's `text_projection` config, run by `TextProjections`.
+
+/** What the daemon hands the worker when a project opts in (`local.doc_read: true`). Getters, so a policy reload applies to the next call. */
+export interface DocReadPort {
+  /** The projections of this worker's execution root: the same instance the hub's own facts and review projections use. */
+  projections: () => { handles(file: string): boolean; project(file: string): ProjectedText | undefined };
+  /** `pii.paths`: a document under it is PII. */
+  piiPaths: () => readonly string[];
+  /** `signals.pii_patterns`: a document whose text matches is PII. */
+  piiPatterns: () => readonly string[];
+}
+/** The port as a tool call sees it: with whether this turn is a PII task turn. */
+export interface DocReadContext extends DocReadPort { pii: boolean }
+
+export const DEFAULT_DOC_LINES = 2000;
+
+/** The refusal text for a document the turn may not read, so the model is told why instead of retrying. */
+const PII_PATH = "is under pii.paths: only a PII task turn on the on-prem worker may read it; ask for it in a PII task";
+const PII_TEXT = "has text that matches signals.pii_patterns, so it is PII: only a PII task turn on the on-prem worker may read it; ask for it in a PII task";
+
+/**
+ * Text of one document. `file` is already the real path `guardPath` returned (inside the project, not on the denylist, no link escapes). Never throws: every refusal
+ * is text for the model. Read-only: the converter runs in the projection sandbox on a private copy of the bytes and its cache lives in the hub state directory,
+ * so nothing is written beside the document, whatever `data.readonly` says.
+ * ponytail: the conversion is synchronous in the daemon process (bounded by the converter's 8 s timeout, then cached by content); upgrade path is an async spawn if long
+ * documents ever stall the hub.
+ */
+export function docRead(file: string, shown: string, args: Record<string, unknown>, port: DocReadContext, cwd: string): string {
+  const projections = port.projections();
+  if (!projections.handles(file)) {
+    const ext = extname(file).toLowerCase() || "(no extension)";
+    return `error: no converter is configured for ${ext} files (text_projection in .agenthub/config.local.json names one per extension); use read for plain text`;
+  }
+  let st;
+  try { st = lstatSync(file); } catch { return `error: ${shown} cannot be read`; }
+  if (!st.isFile()) return `error: ${shown} is not a regular file`;
+  if (st.nlink !== 1) return `error: ${shown} has several names (a hard link), so its path policy cannot be checked; copy it first`;
+  if (st.size > MAX_PROJECTION_INPUT) return `error: ${shown} is larger than ${MAX_PROJECTION_INPUT / 1024 / 1024} MiB`;
+  // pii.paths decides by the document's own path first: a non-PII turn never even converts it.
+  if (!port.pii && matchesPolicy(cwd, file, [...port.piiPaths()])) return `error: ${shown} ${PII_PATH}`;
+  const projected = projections.project(file);
+  if (!projected) return `error: ${shown} could not be converted (the converter failed or timed out, it is not available here, or the sandbox is unavailable)`;
+  const text = projected.text.replace(/\0/g, "");
+  if (!port.pii) {
+    // A pattern that cannot be evaluated is not a clear verdict.
+    try { if (port.piiPatterns().some(pattern => new RegExp(pattern).test(text))) return `error: ${shown} ${PII_TEXT}`; }
+    catch { return `error: ${shown} could not be screened against signals.pii_patterns, so it is not shown`; }
+  }
+  const lines = sanitize(text).split(/\r?\n/);
+  const from = Math.max(1, Math.floor(Number(args.offset)) || 1), limit = Math.max(1, Math.floor(Number(args.limit)) || DEFAULT_DOC_LINES);
+  if (from > lines.length) return `error: offset ${from} is past the end (${lines.length} lines)`;
+  let out = "", shownTo = from - 1;
+  for (let i = from - 1; i < Math.min(lines.length, from - 1 + limit); i++) {
+    const row = `${i + 1}\t${lines[i]}\n`;
+    if (out.length + row.length > OUTPUT_CAP) { if (!out) out = row.slice(0, OUTPUT_CAP); break; }
+    out += row; shownTo = i + 1;
+  }
+  if (!shownTo || shownTo < from) shownTo = from;
+  out = out.replace(/\n$/, "");
+  return shownTo < lines.length ? `${out}\n(lines ${from}-${shownTo} of ${lines.length}; read again with offset=${shownTo + 1})` : out;
+}

package/src/local/proxy.ts

@@ -31,17 +31,8 @@ export interface EgressProxy {
   close(): Promise<void>;
 }
 
-/** `host` or `host:port`; a name also matches its subdomains, an address only itself. Port 443 unless named. */
-export function allowed(list: string[], host: string, port: number): boolean {
-  const h = host.toLowerCase().replace(/\.$/, "");
-  return list.some((raw) => {
-    const entry = raw.trim().toLowerCase();
-    const at = entry.lastIndexOf(":");
-    const [name, p] = at > 0 && !entry.includes("]") && /^\d+$/.test(entry.slice(at + 1)) ? [entry.slice(0, at), Number(entry.slice(at + 1))] : [entry, 443];
-    if (!name || p !== port) return false;
-    return h === name || (!isIP(name) && h.endsWith(`.${name}`));
-  });
-}
+import { allowed } from "./web-fetch-config.ts";
+export { allowed };
 
 // Parsed, not prefix-matched: an IPv4-mapped IPv6 address in any spelling (`::ffff:7f00:1`) checks as its IPv4 address.
 const INTERNAL = new BlockList();
@@ -81,7 +72,7 @@ const hostOf = (target: string) => {
   }
 };
 
-export function startEgressProxy(opts: { allow: string[]; log: (line: string) => void }): Promise<EgressProxy> {
+export function startEgressProxy(opts: { allow: string[]; log: (line: string) => void; /** Read at each CONNECT instead of `allow`, so a reloaded allowlist applies to the next request (the `web_fetch` proxy uses it). */ allowFn?: () => readonly string[] }): Promise<EgressProxy> {
   const sockets = new Set<Socket>();
   // Half-open on both sides: a client that sends its request and shuts its write side still gets the answer.
   const server: Server = createServer({ allowHalfOpen: true }, (client) => {
@@ -119,7 +110,7 @@ export function startEgressProxy(opts: { allow: string[]; log: (line: string) =>
     const host = m?.[1]!.replace(/^\[|\]$/g, "") ?? "";
     if (!m || !(isIP(host) || /^[a-z0-9.-]+$/i.test(host))) return refuse(client, 400, "bad CONNECT target", "a CONNECT request");
     const port = Number(m[2]);
-    if (!allowed(opts.allow, host, port)) return refuse(client, 403, `${host}:${port} is not in local.network_allow`, `${host}:${port}`);
+    if (!allowed([...(opts.allowFn ? opts.allowFn() : opts.allow)], host, port)) return refuse(client, 403, `${host}:${port} is not in local.network_allow`, `${host}:${port}`);
     let address = host;
     if (!isIP(host)) {
       try {

package/src/local/sandbox.ts

@@ -4,7 +4,7 @@ import { homedir, tmpdir } from "node:os";
 import { dirname, join, resolve } from "node:path";
 import { realPath } from "../hub/project.ts";
 import { hubGitSync } from "../hub/git.ts";
-import { denyRegexes, hubWriteRegexes, sbplString } from "./deny.ts";
+import { denyRegexes, hubWriteRegexes, objectStoreRules, sbplString } from "./deny.ts";
 
 const SANDBOX_EXEC = "/usr/bin/sandbox-exec";
 export const OUTPUT_CAP = 20_000;
@@ -184,6 +184,8 @@ export function profile(cwd: string, network: SandboxNetwork, readAllow: string[
     // .git/modules/ are refused at the depth submodules nest (#281), and the external git dirs get folded
     // hooks/config rules (#270). The cost: git init, clone and worktree add no longer run inside the sandbox.
     `(deny file-write* ${hubWriteRegexes(root, gitDirs, moduleGit, worktrees).join(" ")})`,
+    // The shared object store: only new loose objects may be added (#864).
+    ...(worktrees ? objectStoreRules(dirname(worktrees)) : []),
     `(deny file-read* file-write* ${creds.map((c) => `(subpath ${q(join(home, c))})`).join(" ")})`,
     `(deny file-read* file-write* ${denyRegexes(root, deny).join(" ")})`,
     // Python's own CA bundle (certifi, which pip vendors too): pip, requests and httpx read it instead of the system's (#64).

package/src/local/tools.ts

@@ -2,8 +2,10 @@ import { lstatSync, mkdirSync, readFileSync, statSync, writeFileSync } from "nod
 import { dirname, join, relative, resolve } from "node:path";
 import { realPath } from "../hub/project.ts";
 import { isHubWritePath } from "../hub/permission-mode.ts";
-import { matchesPolicy } from "../hub/path-policy.ts";
+import { insideRoot, matchesPolicy } from "../hub/path-policy.ts";
 import { isDenied } from "./deny.ts";
+import { docRead, type DocReadContext, type DocReadPort } from "./doc-read.ts";
+import { webFetch, type WebFetchContext, type WebFetchPort } from "./web-fetch.ts";
 import { OUTPUT_CAP, sandboxedExec } from "./sandbox.ts";
 
 export { isDenied };
@@ -34,6 +36,10 @@ export interface ToolContext {
   permit: (title: string, tool?: string, signal?: AbortSignal, canonicalTarget?: string, observe?: ApprovalObserver, originalPath?: string, input?: unknown, policy?: ShellPermitPolicy) => Promise<ToolApproval>;
   /** Host-only synchronous recheck, immediately before a shell process can be spawned. */
   shellExecutionCheck?: (input: unknown) => ShellExecutionCheck;
+  /** #515: present only when the project opted in (`local.doc_read`), so `doc_read` exists for this worker and nothing else gets it. `pii`: this turn is a PII task turn. */
+  docRead?: DocReadPort & { pii: boolean };
+  /** #515b: present only when the project opted in (`local.web_fetch.allow` names a host), so `web_fetch` exists for this worker and nothing else gets it. `pii`: this turn is a PII task turn, for which it is refused. */
+  webFetch?: WebFetchPort & { pii: boolean };
   /** Trusted process status, independent of what its stdout says. */
   onProcessExit?: (code: number | null) => void;
   /** Publish a message to other peers mid-turn. Returns a one-line receipt. */
@@ -66,10 +72,14 @@ export function guardPath(ctx: Pick<ToolContext, "cwd" | "deny" | "readOnly">, p
   const real = join(realExisting, relative(existing, abs));
   const rel = relative(root, real);
   if (rel.startsWith("..") || resolve(root, rel) !== real) throw new Error(`${path} is outside the project directory`);
-  if (isDenied(rel, ctx.deny)) throw new Error(`${path} is on the secrets denylist`);
+  // Never trust the resolved name alone (#541): on a hardlinked file Bun's realpath sometimes names another link of the same inode, so the
+  // name the caller typed is judged too. A refusal on either name stands; the typed name only ever adds refusals. Only a typed name INSIDE the project is judged
+  // (`insideRoot`, #555): a name outside it (`../o/.env`, a link into the project) is no project path for any policy to name, and it was never refused before #541.
+  const typed = relative(root, abs), inside = insideRoot(typed);
+  if (isDenied(rel, ctx.deny) || inside && isDenied(typed, ctx.deny)) throw new Error(`${path} is on the secrets denylist`);
   // Not secret, but writing .git or .agenthub changes what runs outside the worker's control. Names are compared
   // folded: a new `.GIT/config` is `.git/config` to a case-insensitive disk, and git then runs what it names.
-  if (mode === "write" && isHubWritePath(rel)) throw new Error(`${path} is not writable by the local worker`);
+  if (mode === "write" && (isHubWritePath(rel) || inside && isHubWritePath(typed))) throw new Error(`${path} is not writable by the local worker`);
   // Identity refusal is stable even when realPath names a protected hardlink alias.
   if (mode === "write" && ctx.readOnly?.length && (lstatSync(real, { throwIfNoEntry: false })?.nlink ?? 1) > 1) throw new Error("hardlinked write target cannot be verified with data.readonly");
   if (mode === "write" && matchesPolicy(root, path, ctx.readOnly ?? [])) throw new Error("target is protected by data.readonly");
@@ -111,6 +121,20 @@ export const TOOL_SCHEMAS = [
   fn("hub_send", "Send a message to the other agents now, before your turn ends. Conclusions only. Your final answer is shared anyway; use this for something that cannot wait.", { text: str, to: { type: "array", items: str } }, ["text"]),
 ];
 
+/**
+ * #515: offered only to a worker whose project set `local.doc_read` (off by default), never part of TOOL_SCHEMAS. The model names a path and a line window; the converter
+ * is the person's `text_projection` entry for the extension, run in the projection sandbox, and the text is bounded and passed through `sanitize()`.
+ */
+export const DOC_READ_SCHEMA = fn("doc_read", "Read a PDF, DOCX or HWPX document inside the project as plain text, through the converters the project configured. Returns numbered lines, bounded; read again with offset for more. Read-only.", { path: str, offset: { type: "integer", minimum: 1 }, limit: { type: "integer", minimum: 1 } }, ["path"]);
+export type { DocReadContext, DocReadPort };
+
+/**
+ * #515b: offered only to a worker whose project listed at least one host in `local.web_fetch.allow` (off by default), never part of TOOL_SCHEMAS. The model names a URL and a line
+ * window and nothing else: no method, header or body. It is refused for the whole turn of a PII task, and otherwise runs only through the egress proxy, to listed hosts.
+ */
+export const WEB_FETCH_SCHEMA = fn("web_fetch", "Read one web page as plain text. https only, and only for hosts the project listed in local.web_fetch.allow; refused in a PII task. A plain GET with no headers or body of yours. Returns numbered lines, bounded; read again with offset for more. Never put anything private or from the task into the URL.", { url: str, offset: { type: "integer", minimum: 1 }, limit: { type: "integer", minimum: 1 } }, ["url"]);
+export type { WebFetchContext, WebFetchPort };
+
 /** Every path-like argument of a call, for the memory capture filter. */
 export function touchedPaths(name: string, args: Record<string, unknown>): string[] {
   if (name === "read" || name === "write" || name === "edit") return [String(args.path ?? "")];
@@ -213,6 +237,16 @@ export async function runTool(name: string, rawArgs: string, ctx: ToolContext):
         try { ctx.onProcessExit?.(res.code); } catch { /* optional status observer never changes a tool result */ }
         return `${res.output}\n(exit ${res.code})`;
       }
+      case "doc_read": {
+        if (!ctx.docRead) return "error: doc_read is not enabled for this project (local.doc_read in .agenthub/config.local.json)";
+        const file = guardPath(ctx, String(a.path), "read");
+        return docRead(file, String(a.path), a, ctx.docRead, ctx.cwd);
+      }
+      case "web_fetch": {
+        // The refusal for a PII turn is the first thing this tool does, before the URL is read (see `webFetch`); the opt-in check only says whether the tool exists.
+        if (!ctx.webFetch) return "error: web_fetch is not enabled for this project (local.web_fetch.allow in .agenthub/config.local.json)";
+        return webFetch(String(a.url ?? ""), a, ctx.webFetch, ctx.signal);
+      }
       case "hub_send":
         return a.text ? ctx.send(String(a.text), Array.isArray(a.to) ? a.to.map(String) : undefined) : "error: text is required";
       default:

package/src/local/web-fetch-config.ts

@@ -0,0 +1,43 @@
+// agent-hub #515 (hub task #332): the `local.web_fetch` config block and the host matcher both the tool and the egress proxy use. Pure on purpose: the config schema reaches the
+// Claude channel bundle, so this file imports nothing that touches the file system, opens a socket or spawns a process. The tool is in `web-fetch.ts`, the proxy in `proxy.ts`.
+import { isIP } from "node:net";
+
+/** `host` or `host:port`; a name also matches its subdomains, an address only itself. Port 443 unless named. */
+export function allowed(list: string[], host: string, port: number): boolean {
+  const h = host.toLowerCase().replace(/\.$/, "");
+  return list.some((raw) => {
+    const entry = raw.trim().toLowerCase();
+    const at = entry.lastIndexOf(":");
+    const [name, p] = at > 0 && !entry.includes("]") && /^\d+$/.test(entry.slice(at + 1)) ? [entry.slice(0, at), Number(entry.slice(at + 1))] : [entry, 443];
+    if (!name || p !== port) return false;
+    return h === name || (!isIP(name) && h.endsWith(`.${name}`));
+  });
+}
+/** At most this many hosts: a list is a small, deliberate thing. */
+export const MAX_WEB_HOSTS = 64;
+/**
+ * Why one `local.web_fetch.allow` entry is not accepted, or undefined. An entry is a host name, optionally `:port`: letters, digits, `-` and `.`, at least one dot, never an
+ * address, a URL, a path, a wildcard or a bare top-level name (`com` would open every site). Checked when the config loads, so a typo fails there and not in a turn.
+ */
+export function webHostProblem(value: unknown): string | undefined {
+  if (typeof value !== "string" || !value || value.length > 253 + 6) return "a host is a non-empty name of at most 253 characters";
+  const match = /^([A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)+)(?::([0-9]{1,5}))?$/.exec(value);
+  if (!match) return "a host is a name such as docs.example.org or docs.example.org:8443: no scheme, path, wildcard, spaces or bare top-level name";
+  if (isIP(match[1]!)) return "a host is a name, never an address";
+  if (match[2] !== undefined && (Number(match[2]) < 1 || Number(match[2]) > 65535)) return "a port is 1 to 65535";
+  return undefined;
+}
+
+/** The checked `local.web_fetch.allow`. Nothing here reads the network or the disk. */
+export function webFetchAllow(input: unknown): string[] {
+  if (input === undefined) return [];
+  if (!Array.isArray(input) || input.length > MAX_WEB_HOSTS) throw new Error(`local.web_fetch.allow takes a list of at most ${MAX_WEB_HOSTS} hosts`);
+  const seen = new Set<string>(), out: string[] = [];
+  input.forEach((host, index) => {
+    const problem = webHostProblem(host);
+    if (problem) throw new Error(`local.web_fetch.allow[${index}]: ${problem}`);
+    const key = (host as string).toLowerCase();
+    if (!seen.has(key)) { seen.add(key); out.push(key); }
+  });
+  return out;
+}

package/src/local/web-fetch.ts

@@ -0,0 +1,183 @@
+import { isIP } from "node:net";
+import { sanitize } from "../hub/envelope.ts";
+import { OUTPUT_CAP } from "./sandbox.ts";
+import { allowed } from "./web-fetch-config.ts";
+export { MAX_WEB_HOSTS, webFetchAllow, webHostProblem } from "./web-fetch-config.ts";
+
+// agent-hub #515 (hub task #332): the local worker's `web_fetch`. Spec: "On-prem web reading (#515)", the owner decision of 2026-10-22.
+// It is refused for the whole turn of a PII task, whatever the host: the URL or a query string could carry PII to the web, and `pii.paths` and `sanitize()` cover
+// what comes back, not what goes out. Outside a PII turn it runs only through the egress proxy, to hosts the person listed for this project, and returns bounded,
+// sanitized text only. Off by default.
+
+/** The refusal for a PII turn. It is the same text for every URL, so it says nothing about the request. */
+export const WEB_FETCH_PII = "error: web_fetch is not available in a PII task turn, whatever the host: the URL or a query could carry PII to the web. Use doc_read or read for what the project already holds";
+
+/** What the daemon hands the worker when a project opts in (`local.web_fetch.allow` names at least one host). Getters, so a policy reload applies to the next call. */
+export interface WebFetchPort {
+  /** `local.web_fetch.allow`: hosts (`host` or `host:port`, subdomains included for a name) the model may read. */
+  allow: () => readonly string[];
+  /** The loopback URL of the egress proxy that opens tunnels to `allow` and nothing else; undefined while it is not running. */
+  proxyUrl: () => string | undefined;
+  /** Test seam: the fetch to use. Production leaves it unset (the global `fetch`). */
+  fetch?: (url: string, init: { proxy: string; redirect: "manual"; signal: AbortSignal; headers: Record<string, string>; method: "GET" }) => Promise<Response>;
+}
+/** The port as a tool call sees it: with whether this turn is a PII task turn. */
+export interface WebFetchContext extends WebFetchPort { pii: boolean }
+
+export const WEB_FETCH_TIMEOUT_MS = 15_000;
+/** Bytes read from a response, however large it says it is. Text past it is cut, never buffered. */
+export const WEB_FETCH_MAX_BYTES = 512 * 1024;
+export const WEB_FETCH_MAX_URL = 2000;
+/** Redirects followed, each one re-checked against the allowlist before it is requested. */
+export const WEB_FETCH_MAX_REDIRECTS = 3;
+export const DEFAULT_WEB_LINES = 2000;
+
+const TEXT_TYPE = /^(?:text\/[a-z0-9.+-]+|application\/(?:json|xml|xhtml\+xml|[a-z0-9.+-]+\+(?:json|xml)))(?:\s*;.*)?$/i;
+
+/**
+ * Whether a URL may be requested at all: https, a host the list names, port 443 unless the entry says otherwise, no userinfo, no address literal, no fragment-only tricks.
+ * A reason string, never a throw. The reason names the host (the model already wrote it) and never anything the person configured beyond "not in the list".
+ */
+export function checkWebUrl(raw: string, allow: readonly string[]): { ok: true; url: URL } | { ok: false; reason: string } {
+  if (typeof raw !== "string" || !raw.trim() || raw.length > WEB_FETCH_MAX_URL || /[\x00-\x20\x7f]/.test(raw)) return { ok: false, reason: "url must be one https URL of at most 2000 characters with no spaces or control characters" };
+  let url: URL;
+  try { url = new URL(raw); } catch { return { ok: false, reason: "url is not a valid URL" }; }
+  if (url.protocol !== "https:") return { ok: false, reason: "only https URLs are fetched" };
+  if (url.username || url.password) return { ok: false, reason: "a URL with a user name or password is not fetched" };
+  const host = url.hostname.replace(/^\[|\]$/g, "");
+  if (isIP(host)) return { ok: false, reason: "an address is not a host: name the site, and the person lists it in local.web_fetch.allow" };
+  const port = url.port ? Number(url.port) : 443;
+  if (!allowed([...allow], host, port)) return { ok: false, reason: `${host}:${port} is not in local.web_fetch.allow` };
+  url.hash = "";
+  return { ok: true, url };
+}
+
+/** The decoded text of a response body, read at most `WEB_FETCH_MAX_BYTES` bytes deep. */
+async function readBounded(res: Response): Promise<{ text: string; cut: boolean }> {
+  const reader = res.body?.getReader();
+  if (!reader) return { text: "", cut: false };
+  const chunks: Uint8Array[] = [];
+  let total = 0, cut = false;
+  for (;;) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    if (total + value.length > WEB_FETCH_MAX_BYTES) { chunks.push(value.subarray(0, WEB_FETCH_MAX_BYTES - total)); total = WEB_FETCH_MAX_BYTES; cut = true; void reader.cancel().catch(() => undefined); break; }
+    chunks.push(value); total += value.length;
+  }
+  return { text: new TextDecoder("utf-8", { fatal: false }).decode(Buffer.concat(chunks)), cut };
+}
+
+const SKIPPED_ELEMENTS = new Set(["script", "style", "noscript", "template", "svg", "head"]);
+/** Their content is never markup, so an unclosed one swallows the rest of the page (what a browser does); the others lose only their tag. */
+const RAW_TEXT_ELEMENTS = new Set(["script", "style"]);
+const BLOCK_CLOSERS = new Set(["p", "div", "li", "tr", "h1", "h2", "h3", "h4", "h5", "h6", "section", "article", "br", "pre", "blockquote", "table"]);
+const isAsciiLetter = (code: number) => (code >= 65 && code <= 90) || (code >= 97 && code <= 122);
+const isNameCode = (code: number) => isAsciiLetter(code) || (code >= 48 && code <= 57) || code === 45;
+
+/**
+ * Markup reduced to its text, enough to read a page: scripts and styles go, tags go, entities are decoded for the few that matter, runs of blank lines collapse.
+ * One forward pass over a body that a hostile page controls: every index only moves ahead, and a search that finds nothing is never repeated for the same
+ * target, so the work is linear in the body whatever is left unclosed (a repeated `<script>`, `<!--` or `<` costs one scan, not one per opener).
+ * The 512 KiB body cap bounds the input; this bounds the CPU, because an AbortSignal cannot interrupt synchronous code.
+ */
+export function htmlText(html: string): string {
+  const out: string[] = [];
+  const closerMissing = new Set<string>();  // a failed search covered the whole suffix, so a later one for the same name would fail too
+  const n = html.length;
+  let i = 0, text = 0;                      // text: start of the plain run not yet copied
+  const flush = (to: number) => { if (to > text) out.push(html.slice(text, to)); };
+  while (i < n) {
+    const lt = html.indexOf("<", i);
+    if (lt === -1) break;
+    const next = html.charCodeAt(lt + 1);
+    if (html.startsWith("<!--", lt)) {
+      flush(lt);
+      const end = html.indexOf("-->", lt + 4);
+      if (end === -1) { text = i = n; break; }   // an unterminated comment runs to the end of the page
+      out.push(" "); text = i = end + 3;
+      continue;
+    }
+    const closing = next === 47;
+    if (!(isAsciiLetter(next) || (closing && isAsciiLetter(html.charCodeAt(lt + 2))) || next === 33 || next === 63)) { i = lt + 1; continue; } // `a < b` is text
+    flush(lt);
+    const gt = html.indexOf(">", lt + 1);
+    if (gt === -1) { text = i = n; break; }       // an unterminated tag at the end of the page is dropped
+    let nameStart = lt + (closing ? 2 : 1), nameEnd = nameStart;
+    while (nameEnd < gt && isNameCode(html.charCodeAt(nameEnd))) nameEnd++;
+    const name = html.slice(nameStart, nameEnd).toLowerCase(), rest = html.slice(nameEnd, gt).trim();
+    i = text = gt + 1;
+    if (closing) { out.push(BLOCK_CLOSERS.has(name) && rest === "" ? "\n" : " "); continue; }
+    if (name === "br" && (rest === "" || rest === "/")) { out.push("\n"); continue; }
+    out.push(" ");
+    if (SKIPPED_ELEMENTS.has(name) && !rest.endsWith("/")) {
+      if (closerMissing.has(name)) continue;   // only a non-raw-text element gets here: an unclosed script or style already ended the page
+      const closer = new RegExp(`</${name}\\s*>`, "gi");
+      closer.lastIndex = i;
+      const found = closer.exec(html);
+      if (found) { i = text = found.index + found[0].length; }
+      else { closerMissing.add(name); if (RAW_TEXT_ELEMENTS.has(name)) i = text = n; }
+    }
+  }
+  flush(n);
+  return out.join("")
+    .replace(/&nbsp;/g, " ").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&#39;|&apos;/g, "'").replace(/&amp;/g, "&")
+    .replace(/[ \t\f\v]+/g, " ").replace(/ *\n */g, "\n").replace(/\n{3,}/g, "\n\n").trim();
+}
+
+/**
+ * One page as text. Never throws: every refusal is text for the model. Order matters and is the contract:
+ *   1. a PII task turn is refused before the URL is read at all, whatever it says, so no host check, error or timing differs for it;
+ *   2. the URL must be an allowed https host, and the proxy must be running (without it there is no way out, so no request is made);
+ *   3. the request is a bare GET through the proxy, with no header or body from the model, and no redirect is followed without being checked again;
+ *   4. only a text type is read, bounded in bytes and time, then `sanitize()` quotes a forged hub header and the output is cut at the shared output cap.
+ */
+export async function webFetch(url: string, args: Record<string, unknown>, port: WebFetchContext, signal?: AbortSignal): Promise<string> {
+  if (port.pii) return WEB_FETCH_PII;
+  const proxy = port.proxyUrl();
+  let current = checkWebUrl(url, port.allow());
+  if (!current.ok) return `error: ${current.reason}`;
+  if (!proxy) return "error: web_fetch has no egress proxy right now, so nothing was requested";
+  const doFetch = port.fetch ?? ((target, init) => fetch(target, init as RequestInit & { proxy: string }));
+  let response: Response | undefined;
+  try {
+    for (let hop = 0; ; hop++) {
+      const abort = AbortSignal.any([AbortSignal.timeout(WEB_FETCH_TIMEOUT_MS), ...(signal ? [signal] : [])]);
+      response = await doFetch(current.url.toString(), { method: "GET", proxy, redirect: "manual", signal: abort, headers: { accept: "text/html, text/plain, application/json;q=0.9, */*;q=0.1", "user-agent": "agent-hub-web-fetch" } });
+      if (response.status < 300 || response.status >= 400) break;
+      const location = response.headers.get("location");
+      void response.body?.cancel().catch(() => undefined);
+      if (!location) return `error: the server answered ${response.status} with no location`;
+      if (hop >= WEB_FETCH_MAX_REDIRECTS) return `error: more than ${WEB_FETCH_MAX_REDIRECTS} redirects`;
+      let next: string;
+      try { next = new URL(location, current.url).toString(); } catch { return "error: the server sent a redirect that is not a URL"; }
+      const checked = checkWebUrl(next, port.allow());
+      if (!checked.ok) return `error: the page redirects to a URL that is not allowed (${checked.reason})`;
+      current = checked;
+    }
+  } catch (error) {
+    if (signal?.aborted) return "error: turn cancelled before the page was read";
+    const name = (error as { name?: string; code?: string })?.name, code = (error as { code?: string })?.code;
+    if (name === "TimeoutError" || name === "AbortError") return `error: the page did not answer within ${WEB_FETCH_TIMEOUT_MS / 1000} seconds`;
+    return `error: the page could not be fetched${typeof code === "string" && /^[A-Z_]{3,40}$/.test(code) ? ` (${code})` : ""}`;
+  }
+  if (!response.ok) { void response.body?.cancel().catch(() => undefined); return `error: the server answered ${response.status}`; }
+  const type = response.headers.get("content-type") ?? "";
+  if (!TEXT_TYPE.test(type)) { void response.body?.cancel().catch(() => undefined); return `error: ${type.split(";")[0] || "an unknown type"} is not text: only text, JSON and XML pages are read`; }
+  let body: { text: string; cut: boolean };
+  try { body = await readBounded(response); } catch { return "error: the page could not be read to the end"; }
+  const isHtml = /^text\/html|^application\/xhtml\+xml/i.test(type);
+  const text = sanitize((isHtml ? htmlText(body.text) : body.text).replace(/\0/g, ""));
+  const lines = text.split(/\r?\n/);
+  const from = Math.max(1, Math.floor(Number(args.offset)) || 1), limit = Math.max(1, Math.floor(Number(args.limit)) || DEFAULT_WEB_LINES);
+  if (from > lines.length) return `error: offset ${from} is past the end (${lines.length} lines)`;
+  let out = "", shownTo = from - 1;
+  for (let i = from - 1; i < Math.min(lines.length, from - 1 + limit); i++) {
+    const row = `${i + 1}\t${lines[i]}\n`;
+    if (out.length + row.length > OUTPUT_CAP) { if (!out) out = row.slice(0, OUTPUT_CAP); break; }
+    out += row; shownTo = i + 1;
+  }
+  if (!shownTo || shownTo < from) shownTo = from;
+  out = out.replace(/\n$/, "");
+  const more = shownTo < lines.length ? `(lines ${from}-${shownTo} of ${lines.length}; read again with offset=${shownTo + 1})` : body.cut ? `(the page was cut at ${WEB_FETCH_MAX_BYTES / 1024} KiB)` : "";
+  return more ? `${out}\n${more}` : out;
+}