@asterxsk/kiln 0.1.0

package/agent/extensions/pi-web-access/test/declared-web-links.test.mjs

@@ -0,0 +1,212 @@
+import assert from "node:assert/strict";
+import { spawnSync } from "node:child_process";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { test } from "node:test";
+import { parseHTML } from "linkedom";
+import {
+	appendDeclaredWebLinks,
+	discoverDeclaredWebLinks,
+} from "../declared-web-links.ts";
+
+const extractModuleUrl = new URL("../extract.ts", import.meta.url).href;
+
+function runChild(script) {
+	const home = mkdtempSync(join(tmpdir(), "pi-web-access-declared-links-"));
+	writeFileSync(
+		join(home, "web-search.json"),
+		JSON.stringify({ fetchRouting: { allowRemoteHostedProviders: true } }) +
+			"\n",
+		"utf8",
+	);
+	const childEnv = {
+		...process.env,
+		HOME: home,
+		USERPROFILE: home,
+		PI_CODING_AGENT_DIR: home,
+	};
+	for (const key of [
+		"GEMINI_API_KEY",
+		"GOOGLE_GEMINI_API_KEY",
+		"GOOGLE_API_KEY",
+		"GOOGLE_GEMINI_BASE_URL",
+		"CLOUDFLARE_API_KEY",
+		"PARALLEL_API_KEY",
+		"TINYFISH_API_KEY",
+		"FIRECRAWL_BASE_URL",
+		"FIRECRAWL_API_KEY",
+		"PI_ALLOW_BROWSER_COOKIES",
+	])
+		delete childEnv[key];
+	try {
+		return spawnSync(process.execPath, ["--input-type=module"], {
+			input: script,
+			encoding: "utf8",
+			env: childEnv,
+			maxBuffer: 2 * 1024 * 1024,
+		});
+	} finally {
+		rmSync(home, { recursive: true, force: true });
+	}
+}
+
+test("discovers registered relations from Link headers and HTML declarations", () => {
+	const { document } = parseHTML(`<!doctype html><html><head>
+		<base href="/v2/">
+		<link rel="stylesheet service-doc" href="docs" type="text/html">
+		<link rel="alternate" href="/feed.xml">
+	</head><body>
+		<a rel="describedby" href="/schema">Schema</a>
+		<a href="/developers">Developer careers</a>
+		<a rel="service-desc" href="javascript:alert(1)">Unsafe</a>
+	</body></html>`);
+	const links = discoverDeclaredWebLinks(
+		document,
+		'</catalog>; title="API, <catalog>"; rel="API-CATALOG"; type="application/linkset+json", ' +
+			'</schema>; rel="service-desc"; type="application/schema+json", ' +
+			'</metadata>; rel="service-meta"; optional, ' +
+			'</quoted>; title="x; rel=service-doc"; rel="alternate", ' +
+			'</anchored>; rel="service-doc"; anchor="/other", ' +
+			'</ignored>; rel="alternate"',
+		"https://example.com/root/start",
+	);
+
+	assert.deepEqual(links, [
+		{
+			url: "https://example.com/catalog",
+			relations: ["api-catalog"],
+			type: "application/linkset+json",
+		},
+		{
+			url: "https://example.com/schema",
+			relations: ["service-desc", "describedby"],
+			type: "application/schema+json",
+		},
+		{
+			url: "https://example.com/metadata",
+			relations: ["service-meta"],
+		},
+		{
+			url: "https://example.com/v2/docs",
+			relations: ["service-doc"],
+			type: "text/html",
+		},
+	]);
+});
+
+test("bounds declarations while preserving their relation annotations", () => {
+	const declarations = Array.from(
+		{ length: 25 },
+		(_, index) => `<link rel="service-doc" href="/docs/${index}">`,
+	).join("");
+	const { document } = parseHTML(`<html><head>${declarations}</head></html>`);
+	const links = discoverDeclaredWebLinks(
+		document,
+		null,
+		"https://example.com/",
+	);
+	assert.equal(links.length, 20);
+
+	const oversized = discoverDeclaredWebLinks(
+		parseHTML("<html></html>").document,
+		`<https://example.com/${"x".repeat(4096)}>; rel="service-doc"`,
+		"https://example.com/",
+	);
+	assert.deepEqual(oversized, []);
+
+	const content = appendDeclaredWebLinks(
+		"Existing: https://example.com/docs/0-extra",
+		links.slice(0, 2),
+	);
+	assert.match(content, /<https:\/\/example\.com\/docs\/0>/);
+	assert.match(content, /<https:\/\/example\.com\/docs\/1>/);
+});
+
+test("HTML extraction surfaces declared documentation links without broad URL heuristics", () => {
+	const child = runChild(`
+		const { extractContent } = await import(${JSON.stringify(extractModuleUrl)});
+		const lookup = async () => [{ address: "93.184.216.34", family: 4 }];
+		const article = "Readable article content remains the primary result. ".repeat(20);
+		const fixtures = {
+			"https://example.com/readable": {
+				html: \`<!doctype html><html><head><title>Readable</title></head><body><article><h1>Readable</h1><p>\${article}</p></article></body></html>\`,
+				link: '</openapi.json>; rel="service-desc"; type="application/vnd.oai.openapi+json;version=3.1"',
+			},
+			"https://example.com/shell": {
+				html: '<!doctype html><html><head><title>API shell</title><link rel="service-doc" href="/docs"></head><body><div id="app"></div></body></html>',
+			},
+			"https://example.com/fallback": {
+				html: '<!doctype html><html><head><title>Rendered API</title></head><body><div id="app"></div></body></html>',
+				link: '</openapi.json>; rel="service-desc"',
+			},
+			"https://example.com/generic": {
+				html: '<!doctype html><html><head><title>Company</title></head><body><a href="/developers">Developer careers</a></body></html>',
+			},
+		};
+		let calls = [];
+		globalThis.fetch = async (input) => {
+			const url = String(input);
+			calls.push(url);
+			if (url === "https://r.jina.ai/https://example.com/fallback") {
+				return new Response("Title: Rendered API\\nMarkdown Content:\\n# Rendered API\\n\\n" + article, { status: 200 });
+			}
+			const fixture = fixtures[url];
+			if (!fixture) return new Response("not available", { status: 404, statusText: "Not Found" });
+			return new Response(fixture.html, {
+				status: 200,
+				headers: {
+					"content-type": "text/html; charset=utf-8",
+					...(fixture.link ? { link: fixture.link } : {}),
+				},
+			});
+		};
+
+		const readable = await extractContent("https://example.com/readable", undefined, { lookup });
+		const readableCalls = calls.splice(0);
+		const shell = await extractContent("https://example.com/shell", undefined, { lookup });
+		const shellCalls = calls.splice(0);
+		const fallback = await extractContent("https://example.com/fallback", undefined, { lookup });
+		const fallbackCalls = calls.splice(0);
+		const generic = await extractContent("https://example.com/generic", undefined, { lookup });
+		console.log(JSON.stringify({ readable, readableCalls, shell, shellCalls, fallback, fallbackCalls, generic }));
+	`);
+	assert.equal(child.status, 0, child.stderr);
+	const output = JSON.parse(child.stdout.trim());
+
+	assert.equal(output.readable.error, null);
+	assert.match(output.readable.content, /Readable article content/);
+	assert.match(output.readable.content, /## Declared links/);
+	assert.match(output.readable.content, /service-desc/);
+	assert.match(
+		output.readable.content,
+		/https:\/\/example\.com\/openapi\.json/,
+	);
+	assert.deepEqual(output.readableCalls, ["https://example.com/readable"]);
+
+	assert.equal(output.shell.error, null);
+	assert.match(output.shell.content, /service-doc/);
+	assert.match(output.shell.content, /https:\/\/example\.com\/docs/);
+	assert.deepEqual(output.shellCalls, [
+		"https://example.com/shell",
+		"https://r.jina.ai/https://example.com/shell",
+	]);
+
+	assert.equal(output.fallback.error, null);
+	assert.match(output.fallback.content, /Rendered API/);
+	assert.match(output.fallback.content, /service-desc/);
+	assert.match(
+		output.fallback.content,
+		/https:\/\/example\.com\/openapi\.json/,
+	);
+	assert.deepEqual(output.fallbackCalls, [
+		"https://example.com/fallback",
+		"https://r.jina.ai/https://example.com/fallback",
+	]);
+
+	assert.doesNotMatch(
+		output.generic.content,
+		/https:\/\/example\.com\/developers/,
+	);
+	assert.ok(output.generic.error);
+});

package/agent/extensions/pi-web-access/test/fetch-answer-storage.test.mjs

@@ -0,0 +1,40 @@
+import assert from "node:assert/strict";
+import { after, test } from "node:test";
+
+import initializeExtension from "../index.ts";
+import { clearResults, getResult } from "../storage.ts";
+
+const originalFetch = globalThis.fetch;
+after(() => { globalThis.fetch = originalFetch; });
+
+test("answer mode stores original fetched content instead of its answer presentation", async () => {
+	clearResults();
+	const originalContent = "Original page content. ".repeat(60);
+	globalThis.fetch = async () => new Response(
+		`<!doctype html><html><head><title>Stored Page</title></head><body><article><h1>Stored Page</h1><p>${originalContent}</p></article></body></html>`,
+		{ status: 200, headers: { "content-type": "text/html" } },
+	);
+	const tools = [];
+	initializeExtension({
+		registerTool(tool) { tools.push(tool); },
+		registerCommand() {},
+		registerShortcut() {},
+		on() {},
+		appendEntry() {},
+	});
+	const tool = tools.find(registered => registered.name === "fetch_content");
+	assert.ok(tool);
+
+	const result = await tool.execute(
+		"call",
+		{ url: "https://93.184.216.34/page", mode: "answer", prompt: "What does it say?" },
+		undefined,
+		undefined,
+		{ model: undefined, modelRegistry: {}, cwd: process.cwd(), isProjectTrusted: () => false },
+	);
+	assert.match(result.details.error, /Page answer failed/);
+	const stored = getResult(result.details.responseId);
+	assert.equal(stored?.type, "fetch");
+	assert.match(stored?.urls?.[0].content ?? "", /Original page content/);
+	assert.doesNotMatch(stored?.urls?.[0].content ?? "", /Page answer failed/);
+});

package/agent/extensions/pi-web-access/test/fetch-cache-storage.test.mjs

@@ -0,0 +1,334 @@
+import assert from "node:assert/strict";
+import { chmodSync, mkdirSync, readFileSync, readdirSync, rmSync, statSync, symlinkSync, truncateSync, unlinkSync, utimesSync, writeFileSync } from "node:fs";
+import { mkdtemp } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { afterEach, test } from "node:test";
+
+import initializeExtension from "../index.ts";
+import { clearResults, deleteResult, getFetchCacheDir, getResult, pruneExpiredFetchCache, restoreFromSession, storeFetchedContentResult } from "../storage.ts";
+
+const originalFetch = globalThis.fetch;
+const originalAgentDir = process.env.PI_CODING_AGENT_DIR;
+const originalCacheDir = process.env.PI_WEB_ACCESS_CACHE_DIR;
+const originalDateNow = Date.now;
+
+afterEach(() => {
+	globalThis.fetch = originalFetch;
+	Date.now = originalDateNow;
+	if (originalAgentDir === undefined) delete process.env.PI_CODING_AGENT_DIR;
+	else process.env.PI_CODING_AGENT_DIR = originalAgentDir;
+	if (originalCacheDir === undefined) delete process.env.PI_WEB_ACCESS_CACHE_DIR;
+	else process.env.PI_WEB_ACCESS_CACHE_DIR = originalCacheDir;
+	clearResults();
+});
+
+async function useTempAgentDir() {
+	const dir = await mkdtemp(join(tmpdir(), "pi-web-access-fetch-cache-"));
+	process.env.PI_CODING_AGENT_DIR = dir;
+	process.env.PI_WEB_ACCESS_CACHE_DIR = join(dir, "web-search-cache");
+	return dir;
+}
+
+function registerTools() {
+	const tools = [];
+	const entries = [];
+	initializeExtension({
+		registerTool(tool) { tools.push(tool); },
+		registerCommand() {},
+		registerShortcut() {},
+		on() {},
+		appendEntry(type, data) { entries.push({ type, data }); },
+	});
+	return {
+		entries,
+		fetchTool: tools.find((tool) => tool.name === "fetch_content"),
+		getContentTool: tools.find((tool) => tool.name === "get_search_content"),
+	};
+}
+
+function restoreEntry(data) {
+	restoreFromSession({
+		sessionManager: {
+			getBranch: () => [{ type: "custom", customType: "web-search-results", data }],
+		},
+	});
+}
+
+function fetchedData(id, content = "cached content") {
+	return {
+		id,
+		type: "fetch",
+		timestamp: Date.now(),
+		urls: [{ url: `https://example.com/${id}`, title: id, content, error: null }],
+	};
+}
+
+test("fetch_content stores full content in cache and writes a bounded session entry", async () => {
+	await useTempAgentDir();
+	const pageContent = "Cached page content. ".repeat(4_000);
+	globalThis.fetch = async () => new Response(pageContent, { status: 200, headers: { "content-type": "text/plain" } });
+
+	const { entries, fetchTool, getContentTool } = registerTools();
+	assert.ok(fetchTool);
+	assert.ok(getContentTool);
+
+	const result = await fetchTool.execute("call", { url: "https://93.184.216.34/page" });
+	const entry = entries.find((candidate) => candidate.type === "web-search-results");
+	assert.ok(entry);
+	assert.equal(entry.data.type, "fetch");
+	assert.equal(entry.data.urls, undefined);
+	assert.ok(entry.data.fetchCache?.key);
+	assert.equal(entry.data.urlMetadata?.[0]?.contentLength, pageContent.length);
+
+	const serialized = JSON.stringify(entry.data);
+	assert.ok(serialized.length < 5_000, `session entry was ${serialized.length} chars`);
+	assert.doesNotMatch(serialized, /Cached page content\. Cached page content\./);
+
+	clearResults();
+	restoreEntry(entry.data);
+	const restored = await getContentTool.execute("call", {
+		responseId: result.details.responseId,
+		urlIndex: 0,
+		offset: pageContent.length - 21,
+		limit: 21,
+	});
+	assert.equal(restored.details.returnedChars, 21);
+	assert.match(restored.content[0].text, /Cached page content\./);
+});
+
+test("legacy inline fetched session entries remain readable", async () => {
+	await useTempAgentDir();
+	const { getContentTool } = registerTools();
+	assert.ok(getContentTool);
+	const legacy = {
+		id: "legacy-fetch",
+		type: "fetch",
+		timestamp: Date.now(),
+		urls: [{ url: "https://example.com/legacy", title: "Legacy", content: "legacy inline content", error: null }],
+	};
+
+	restoreEntry(legacy);
+	const result = await getContentTool.execute("call", { responseId: "legacy-fetch", urlIndex: 0 });
+	assert.match(result.content[0].text, /legacy inline content/);
+	assert.equal(result.details.contentLength, "legacy inline content".length);
+});
+
+test("loaded cached fetched content expires after the result lifetime", async () => {
+	const startedAt = originalDateNow();
+	Date.now = () => startedAt;
+	await useTempAgentDir();
+	const pageContent = "expiring cache-backed content";
+	globalThis.fetch = async () => new Response(pageContent, { status: 200, headers: { "content-type": "text/plain" } });
+
+	const { entries, fetchTool, getContentTool } = registerTools();
+	assert.ok(fetchTool);
+	assert.ok(getContentTool);
+	const result = await fetchTool.execute("call", { url: "https://93.184.216.34/expiring-cache" });
+	const entry = entries.find((candidate) => candidate.type === "web-search-results");
+	assert.ok(entry?.data.fetchCache?.key);
+
+	clearResults();
+	restoreEntry(entry.data);
+	const loaded = await getContentTool.execute("call", { responseId: result.details.responseId, urlIndex: 0 });
+	assert.match(loaded.content[0].text, /expiring cache-backed content/);
+
+	Date.now = () => startedAt + 60 * 60 * 1000;
+	const expired = await getContentTool.execute("call", { responseId: result.details.responseId, urlIndex: 0 });
+	assert.equal(expired.details.error, "Cached fetched content is missing or expired");
+	assert.match(expired.content[0].text, /Cached fetched content is missing or expired/);
+	assert.doesNotMatch(expired.content[0].text, /expiring cache-backed content/);
+});
+
+test("missing cache files return an actionable fetched-content error", async () => {
+	const agentDir = await useTempAgentDir();
+	const pageContent = "cache-backed content";
+	globalThis.fetch = async () => new Response(pageContent, { status: 200, headers: { "content-type": "text/plain" } });
+
+	const { entries, fetchTool, getContentTool } = registerTools();
+	assert.ok(fetchTool);
+	assert.ok(getContentTool);
+	const result = await fetchTool.execute("call", { url: "https://93.184.216.34/missing-cache" });
+	const entry = entries.find((candidate) => candidate.type === "web-search-results");
+	assert.ok(entry?.data.fetchCache?.key);
+	rmSync(join(getFetchCacheDir(), entry.data.fetchCache.key), { force: true });
+
+	clearResults();
+	restoreEntry(entry.data);
+	const missing = await getContentTool.execute("call", { responseId: result.details.responseId, urlIndex: 0 });
+	assert.equal(missing.details.error, "Cached fetched content is missing or expired");
+	assert.match(missing.content[0].text, /Cached fetched content is missing or expired/);
+	rmSync(agentDir, { recursive: true, force: true });
+});
+
+test("cache pruning evicts the oldest entries by count and bytes", async () => {
+	await useTempAgentDir();
+	const cacheDir = getFetchCacheDir();
+	mkdirSync(cacheDir, { recursive: true });
+	const now = Date.now();
+	for (const [name, content, ageSeconds] of [
+		["oldest.json", "1111", 30],
+		["middle.json", "222222", 20],
+		["newest.json", "33333333", 10],
+	]) {
+		const path = join(cacheDir, name);
+		writeFileSync(path, content);
+		utimesSync(path, new Date(now - ageSeconds * 1000), new Date(now - ageSeconds * 1000));
+	}
+
+	pruneExpiredFetchCache(now, { maxEntries: 2, maxBytes: 1024 });
+	assert.deepEqual(readdirSync(cacheDir).sort(), ["middle.json", "newest.json"]);
+
+	pruneExpiredFetchCache(now, { maxEntries: 10, maxBytes: 8 });
+	assert.deepEqual(readdirSync(cacheDir), ["newest.json"]);
+	assert.throws(() => pruneExpiredFetchCache(now, { maxEntries: 0 }), /finite positive integers/);
+	assert.throws(() => pruneExpiredFetchCache(now, { maxBytes: Infinity }), /finite positive integers/);
+});
+
+test("default cache pruning keeps the newest 128 entries", async () => {
+	await useTempAgentDir();
+	const cacheDir = getFetchCacheDir();
+	mkdirSync(cacheDir, { recursive: true });
+	const now = Date.now();
+	for (let index = 0; index < 129; index++) {
+		const path = join(cacheDir, `entry-${index.toString().padStart(3, "0")}.json`);
+		writeFileSync(path, "{}");
+		const modified = new Date(now - (129 - index) * 1000);
+		utimesSync(path, modified, modified);
+	}
+
+	pruneExpiredFetchCache(now);
+	const remaining = readdirSync(cacheDir).sort();
+	assert.equal(remaining.length, 128);
+	assert.equal(remaining.includes("entry-000.json"), false);
+	assert.equal(remaining.includes("entry-128.json"), true);
+});
+
+test("default cache pruning enforces the 128 MiB byte limit", async () => {
+	await useTempAgentDir();
+	const cacheDir = getFetchCacheDir();
+	mkdirSync(cacheDir, { recursive: true });
+	const now = Date.now();
+	for (const [name, ageSeconds] of [["older.json", 20], ["newer.json", 10]]) {
+		const path = join(cacheDir, name);
+		writeFileSync(path, "");
+		truncateSync(path, 65 * 1024 * 1024);
+		const modified = new Date(now - ageSeconds * 1000);
+		utimesSync(path, modified, modified);
+	}
+
+	pruneExpiredFetchCache(now);
+	assert.deepEqual(readdirSync(cacheDir), ["newer.json"]);
+});
+
+test("cache pruning removes only stale owned temp files", async () => {
+	await useTempAgentDir();
+	const cacheDir = getFetchCacheDir();
+	mkdirSync(cacheDir, { recursive: true });
+	const now = Date.now();
+	const stale = ["old.json.123.456.tmp", `new.json.123.456.${"a".repeat(32)}.tmp`];
+	const fresh = `fresh.json.123.456.${"b".repeat(32)}.tmp`;
+	const preserved = ["foreign.tmp", "old.json.not-ours.tmp", fresh];
+	for (const name of [...stale, ...preserved]) {
+		const path = join(cacheDir, name);
+		writeFileSync(path, name);
+		if (name !== fresh) utimesSync(path, new Date(now - 2 * 60 * 60 * 1000), new Date(now - 2 * 60 * 60 * 1000));
+	}
+
+	pruneExpiredFetchCache(now);
+	assert.deepEqual(readdirSync(cacheDir).sort(), preserved.sort());
+});
+
+test("cache pruning corrects directory and entry permissions", { skip: process.platform === "win32" }, async () => {
+	await useTempAgentDir();
+	const cacheDir = getFetchCacheDir();
+	mkdirSync(cacheDir, { recursive: true });
+	const entryPath = join(cacheDir, "permissions.json");
+	writeFileSync(entryPath, "{}");
+	chmodSync(cacheDir, 0o777);
+	chmodSync(entryPath, 0o666);
+
+	pruneExpiredFetchCache();
+	assert.equal(statSync(cacheDir).mode & 0o777, 0o700);
+	assert.equal(statSync(entryPath).mode & 0o777, 0o600);
+});
+
+test("cache write and rename failures degrade without leaving temp files", async () => {
+	await useTempAgentDir();
+	const unwritable = fetchedData("unwritable");
+	unwritable.circular = unwritable;
+	const failed = storeFetchedContentResult("unwritable", unwritable);
+	assert.equal(failed.fetchCache, undefined);
+	assert.match(failed.fetchCacheError, /circular/i);
+	assert.equal(readdirSync(dirname(getFetchCacheDir())).includes("web-search-cache"), false);
+
+	const written = storeFetchedContentResult("normal", fetchedData("normal"));
+	assert.ok(written.fetchCache);
+	mkdirSync(join(getFetchCacheDir(), "blocked.json"));
+	const blocked = storeFetchedContentResult("blocked", fetchedData("blocked"));
+	assert.equal(blocked.fetchCache, undefined);
+	assert.equal(readdirSync(getFetchCacheDir()).some((name) => name.endsWith(".tmp")), false);
+});
+
+test("corrupt cache files remain unavailable without throwing", async () => {
+	await useTempAgentDir();
+	const cacheDir = getFetchCacheDir();
+	mkdirSync(cacheDir, { recursive: true });
+	writeFileSync(join(cacheDir, "corrupt.json"), "not json");
+	restoreEntry({
+		id: "corrupt",
+		type: "fetch",
+		timestamp: Date.now(),
+		fetchCache: { version: 1, key: "corrupt.json", storedAt: Date.now() },
+		urlMetadata: [{ url: "https://example.com/corrupt", title: "corrupt", error: null, contentLength: 8 }],
+	});
+
+	const restored = getResult("corrupt");
+	assert.equal(restored.urls[0].content, "");
+	assert.match(restored.urls[0].error, /could not be read/);
+});
+
+test("cache file symlinks are not followed", { skip: process.platform === "win32" }, async () => {
+	const agentDir = await useTempAgentDir();
+	const cacheDir = getFetchCacheDir();
+	mkdirSync(cacheDir, { recursive: true });
+	const target = join(agentDir, "outside.json");
+	writeFileSync(target, JSON.stringify(fetchedData("linked")));
+	symlinkSync(target, join(cacheDir, "linked.json"));
+	restoreEntry({
+		id: "linked",
+		type: "fetch",
+		timestamp: Date.now(),
+		fetchCache: { version: 1, key: "linked.json", storedAt: Date.now() },
+		urlMetadata: [{ url: "https://example.com/linked", title: "linked", error: null, contentLength: 14 }],
+	});
+
+	const restored = getResult("linked");
+	assert.equal(restored.urls[0].content, "");
+	assert.match(restored.urls[0].error, /not a regular file/);
+	assert.match(readFileSync(target, "utf8"), /cached content/);
+});
+
+test("cache directory symlinks are rejected for writes and deletes", { skip: process.platform === "win32" }, async () => {
+	const agentDir = await useTempAgentDir();
+	const cacheDir = getFetchCacheDir();
+	const outside = join(agentDir, "outside-cache");
+	mkdirSync(dirname(cacheDir), { recursive: true });
+	mkdirSync(outside);
+	symlinkSync(outside, cacheDir);
+
+	const rejected = storeFetchedContentResult("dir-link", fetchedData("dir-link"));
+	assert.equal(rejected.fetchCache, undefined);
+	assert.match(rejected.fetchCacheError, /not a safe directory/);
+	assert.deepEqual(readdirSync(outside), []);
+
+	unlinkSync(cacheDir);
+	const stored = storeFetchedContentResult("delete-link", fetchedData("delete-link"));
+	assert.ok(stored.fetchCache);
+	rmSync(cacheDir, { recursive: true });
+	mkdirSync(outside, { recursive: true });
+	writeFileSync(join(outside, stored.fetchCache.key), "outside");
+	symlinkSync(outside, cacheDir);
+	assert.equal(deleteResult("delete-link"), true);
+	assert.equal(readFileSync(join(outside, stored.fetchCache.key), "utf8"), "outside");
+});

package/agent/extensions/pi-web-access/test/fetch-content-domain-policy.test.mjs

@@ -0,0 +1,95 @@
+import assert from "node:assert/strict";
+import { spawnSync } from "node:child_process";
+import { mkdtemp, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { test } from "node:test";
+
+const extractUrl = new URL("../extract.ts", import.meta.url).href;
+
+async function runExtract(config, urls, optionsByUrl = []) {
+	const root = await mkdtemp(join(tmpdir(), "pi-domain-policy-extract-"));
+	await writeFile(join(root, "web-search.json"), JSON.stringify(config), "utf8");
+	const childEnv = { ...process.env, PI_CODING_AGENT_DIR: root, HOME: root, USERPROFILE: root };
+	for (const key of ["GEMINI_API_KEY", "GOOGLE_GEMINI_API_KEY", "GOOGLE_API_KEY", "CLOUDFLARE_API_KEY", "PARALLEL_API_KEY", "TINYFISH_API_KEY", "FIRECRAWL_BASE_URL", "FIRECRAWL_API_KEY", "BRIGHTDATA_API_KEY", "KAGI_API_KEY", "OLLAMA_API_KEY", "BRIGHTDATA_UNLOCKER_ZONE"]) delete childEnv[key];
+	const child = spawnSync(process.execPath, ["--input-type=module"], {
+		input: `
+			let fetchCalls = [];
+			globalThis.fetch = async (url) => {
+				fetchCalls.push(String(url));
+				return new Response(\`<!doctype html><html><head><title>Allowed</title></head><body><article><h1>Allowed</h1><p>\${"Readable content. ".repeat(60)}</p></article></body></html>\`, { status: 200, headers: { "content-type": "text/html" } });
+			};
+			const { extractContent } = await import(${JSON.stringify(extractUrl)});
+			const urls = ${JSON.stringify(urls)};
+			const optionsByUrl = ${JSON.stringify(optionsByUrl)};
+			const results = [];
+			for (let index = 0; index < urls.length; index += 1) {
+				results.push(await extractContent(urls[index], undefined, { ...optionsByUrl[index], lookup: async () => [{ address: "93.184.216.34", family: 4 }] }));
+			}
+			console.log(JSON.stringify({ fetchCalls, results }));
+		`,
+		encoding: "utf8",
+		env: childEnv,
+		maxBuffer: 2 * 1024 * 1024,
+	});
+	assert.equal(child.status, 0, child.stderr);
+	return JSON.parse(child.stdout.trim());
+}
+
+test("fetch_content enforces domain policy before target network requests", async () => {
+	const output = await runExtract(
+		{ fetchContent: { domainPolicy: { allow: ["allowed.example"], deny: ["blocked.example"] } } },
+		["https://blocked.example/article", "https://allowed.example/article"],
+	);
+	assert.equal(output.fetchCalls.length, 1);
+	assert.equal(output.fetchCalls[0], "https://allowed.example/article");
+	assert.match(output.results[0].error, /Blocked hostname by fetch_content domain policy/);
+	assert.equal(output.results[1].error, null);
+	assert.equal(output.results[1].title, "Allowed");
+});
+
+test("fetch_content domain policy leaves local file inputs outside hostname checks", async () => {
+	const output = await runExtract(
+		{ fetchContent: { domainPolicy: { allow: ["only.example"] } } },
+		["file:///tmp/not-a-web-source.txt"],
+	);
+	assert.deepEqual(output.fetchCalls, []);
+	assert.doesNotMatch(output.results[0].error, /fetch_content domain policy/);
+});
+
+test("fetch_content domain policy blocks YouTube frame and timestamp branches before network helpers", async () => {
+	const output = await runExtract(
+		{ fetchContent: { domainPolicy: { deny: ["youtube.com"] } } },
+		["https://www.youtube.com/watch?v=dQw4w9WgXcQ", "https://www.youtube.com/watch?v=dQw4w9WgXcQ"],
+		[{ frames: 1 }, { timestamp: "1" }],
+	);
+	assert.deepEqual(output.fetchCalls, []);
+	assert.match(output.results[0].error, /Blocked hostname by fetch_content domain policy: www\.youtube\.com/);
+	assert.match(output.results[1].error, /Blocked hostname by fetch_content domain policy: www\.youtube\.com/);
+});
+
+test("fetch_content rejects chunked oversized text responses while reading", async () => {
+	const root = await mkdtemp(join(tmpdir(), "pi-oversized-text-extract-"));
+	const childEnv = { ...process.env, PI_CODING_AGENT_DIR: root, HOME: root, USERPROFILE: root };
+	for (const key of ["GEMINI_API_KEY", "GOOGLE_GEMINI_API_KEY", "GOOGLE_API_KEY", "CLOUDFLARE_API_KEY", "PARALLEL_API_KEY", "TINYFISH_API_KEY", "FIRECRAWL_BASE_URL", "FIRECRAWL_API_KEY"]) delete childEnv[key];
+	const child = spawnSync(process.execPath, ["--input-type=module"], {
+		input: `
+			globalThis.fetch = async () => new Response(new ReadableStream({
+				start(controller) {
+					const chunk = new Uint8Array(1024 * 1024).fill(65);
+					for (let i = 0; i < 6; i += 1) controller.enqueue(chunk);
+					controller.close();
+				},
+			}), { status: 200, headers: { "content-type": "text/html" } });
+			const { extractContent } = await import(${JSON.stringify(extractUrl)});
+			const result = await extractContent("https://allowed.example/article", undefined, { lookup: async () => [{ address: "93.184.216.34", family: 4 }] });
+			console.log(JSON.stringify(result));
+		`,
+		encoding: "utf8",
+		env: childEnv,
+		maxBuffer: 2 * 1024 * 1024,
+	});
+	assert.equal(child.status, 0, child.stderr);
+	const result = JSON.parse(child.stdout.trim());
+	assert.match(result.error, /Response too large \(5MB\)/);
+});