@asterxsk/kiln 0.3.0 → 0.4.2

package/agent/extensions/pi-web-access/test/fetch-content-domain-policy.test.mjs

@@ -1,95 +1,95 @@
-import assert from "node:assert/strict";
-import { spawnSync } from "node:child_process";
-import { mkdtemp, writeFile } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { test } from "node:test";
-
-const extractUrl = new URL("../extract.ts", import.meta.url).href;
-
-async function runExtract(config, urls, optionsByUrl = []) {
-	const root = await mkdtemp(join(tmpdir(), "pi-domain-policy-extract-"));
-	await writeFile(join(root, "web-search.json"), JSON.stringify(config), "utf8");
-	const childEnv = { ...process.env, PI_CODING_AGENT_DIR: root, HOME: root, USERPROFILE: root };
-	for (const key of ["GEMINI_API_KEY", "GOOGLE_GEMINI_API_KEY", "GOOGLE_API_KEY", "CLOUDFLARE_API_KEY", "PARALLEL_API_KEY", "TINYFISH_API_KEY", "FIRECRAWL_BASE_URL", "FIRECRAWL_API_KEY", "BRIGHTDATA_API_KEY", "KAGI_API_KEY", "OLLAMA_API_KEY", "BRIGHTDATA_UNLOCKER_ZONE"]) delete childEnv[key];
-	const child = spawnSync(process.execPath, ["--input-type=module"], {
-		input: `
-			let fetchCalls = [];
-			globalThis.fetch = async (url) => {
-				fetchCalls.push(String(url));
-				return new Response(\`<!doctype html><html><head><title>Allowed</title></head><body><article><h1>Allowed</h1><p>\${"Readable content. ".repeat(60)}</p></article></body></html>\`, { status: 200, headers: { "content-type": "text/html" } });
-			};
-			const { extractContent } = await import(${JSON.stringify(extractUrl)});
-			const urls = ${JSON.stringify(urls)};
-			const optionsByUrl = ${JSON.stringify(optionsByUrl)};
-			const results = [];
-			for (let index = 0; index < urls.length; index += 1) {
-				results.push(await extractContent(urls[index], undefined, { ...optionsByUrl[index], lookup: async () => [{ address: "93.184.216.34", family: 4 }] }));
-			}
-			console.log(JSON.stringify({ fetchCalls, results }));
-		`,
-		encoding: "utf8",
-		env: childEnv,
-		maxBuffer: 2 * 1024 * 1024,
-	});
-	assert.equal(child.status, 0, child.stderr);
-	return JSON.parse(child.stdout.trim());
-}
-
-test("fetch_content enforces domain policy before target network requests", async () => {
-	const output = await runExtract(
-		{ fetchContent: { domainPolicy: { allow: ["allowed.example"], deny: ["blocked.example"] } } },
-		["https://blocked.example/article", "https://allowed.example/article"],
-	);
-	assert.equal(output.fetchCalls.length, 1);
-	assert.equal(output.fetchCalls[0], "https://allowed.example/article");
-	assert.match(output.results[0].error, /Blocked hostname by fetch_content domain policy/);
-	assert.equal(output.results[1].error, null);
-	assert.equal(output.results[1].title, "Allowed");
-});
-
-test("fetch_content domain policy leaves local file inputs outside hostname checks", async () => {
-	const output = await runExtract(
-		{ fetchContent: { domainPolicy: { allow: ["only.example"] } } },
-		["file:///tmp/not-a-web-source.txt"],
-	);
-	assert.deepEqual(output.fetchCalls, []);
-	assert.doesNotMatch(output.results[0].error, /fetch_content domain policy/);
-});
-
-test("fetch_content domain policy blocks YouTube frame and timestamp branches before network helpers", async () => {
-	const output = await runExtract(
-		{ fetchContent: { domainPolicy: { deny: ["youtube.com"] } } },
-		["https://www.youtube.com/watch?v=dQw4w9WgXcQ", "https://www.youtube.com/watch?v=dQw4w9WgXcQ"],
-		[{ frames: 1 }, { timestamp: "1" }],
-	);
-	assert.deepEqual(output.fetchCalls, []);
-	assert.match(output.results[0].error, /Blocked hostname by fetch_content domain policy: www\.youtube\.com/);
-	assert.match(output.results[1].error, /Blocked hostname by fetch_content domain policy: www\.youtube\.com/);
-});
-
-test("fetch_content rejects chunked oversized text responses while reading", async () => {
-	const root = await mkdtemp(join(tmpdir(), "pi-oversized-text-extract-"));
-	const childEnv = { ...process.env, PI_CODING_AGENT_DIR: root, HOME: root, USERPROFILE: root };
-	for (const key of ["GEMINI_API_KEY", "GOOGLE_GEMINI_API_KEY", "GOOGLE_API_KEY", "CLOUDFLARE_API_KEY", "PARALLEL_API_KEY", "TINYFISH_API_KEY", "FIRECRAWL_BASE_URL", "FIRECRAWL_API_KEY"]) delete childEnv[key];
-	const child = spawnSync(process.execPath, ["--input-type=module"], {
-		input: `
-			globalThis.fetch = async () => new Response(new ReadableStream({
-				start(controller) {
-					const chunk = new Uint8Array(1024 * 1024).fill(65);
-					for (let i = 0; i < 6; i += 1) controller.enqueue(chunk);
-					controller.close();
-				},
-			}), { status: 200, headers: { "content-type": "text/html" } });
-			const { extractContent } = await import(${JSON.stringify(extractUrl)});
-			const result = await extractContent("https://allowed.example/article", undefined, { lookup: async () => [{ address: "93.184.216.34", family: 4 }] });
-			console.log(JSON.stringify(result));
-		`,
-		encoding: "utf8",
-		env: childEnv,
-		maxBuffer: 2 * 1024 * 1024,
-	});
-	assert.equal(child.status, 0, child.stderr);
-	const result = JSON.parse(child.stdout.trim());
-	assert.match(result.error, /Response too large \(5MB\)/);
-});
+import assert from "node:assert/strict";
+import { spawnSync } from "node:child_process";
+import { mkdtemp, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { test } from "node:test";
+
+const extractUrl = new URL("../extract.ts", import.meta.url).href;
+
+async function runExtract(config, urls, optionsByUrl = []) {
+	const root = await mkdtemp(join(tmpdir(), "pi-domain-policy-extract-"));
+	await writeFile(join(root, "web-search.json"), JSON.stringify(config), "utf8");
+	const childEnv = { ...process.env, PI_CODING_AGENT_DIR: root, HOME: root, USERPROFILE: root };
+	for (const key of ["GEMINI_API_KEY", "GOOGLE_GEMINI_API_KEY", "GOOGLE_API_KEY", "CLOUDFLARE_API_KEY", "PARALLEL_API_KEY", "TINYFISH_API_KEY", "FIRECRAWL_BASE_URL", "FIRECRAWL_API_KEY", "BRIGHTDATA_API_KEY", "KAGI_API_KEY", "OLLAMA_API_KEY", "BRIGHTDATA_UNLOCKER_ZONE"]) delete childEnv[key];
+	const child = spawnSync(process.execPath, ["--input-type=module"], {
+		input: `
+			let fetchCalls = [];
+			globalThis.fetch = async (url) => {
+				fetchCalls.push(String(url));
+				return new Response(\`<!doctype html><html><head><title>Allowed</title></head><body><article><h1>Allowed</h1><p>\${"Readable content. ".repeat(60)}</p></article></body></html>\`, { status: 200, headers: { "content-type": "text/html" } });
+			};
+			const { extractContent } = await import(${JSON.stringify(extractUrl)});
+			const urls = ${JSON.stringify(urls)};
+			const optionsByUrl = ${JSON.stringify(optionsByUrl)};
+			const results = [];
+			for (let index = 0; index < urls.length; index += 1) {
+				results.push(await extractContent(urls[index], undefined, { ...optionsByUrl[index], lookup: async () => [{ address: "93.184.216.34", family: 4 }] }));
+			}
+			console.log(JSON.stringify({ fetchCalls, results }));
+		`,
+		encoding: "utf8",
+		env: childEnv,
+		maxBuffer: 2 * 1024 * 1024,
+	});
+	assert.equal(child.status, 0, child.stderr);
+	return JSON.parse(child.stdout.trim());
+}
+
+test("fetch_content enforces domain policy before target network requests", async () => {
+	const output = await runExtract(
+		{ fetchContent: { domainPolicy: { allow: ["allowed.example"], deny: ["blocked.example"] } } },
+		["https://blocked.example/article", "https://allowed.example/article"],
+	);
+	assert.equal(output.fetchCalls.length, 1);
+	assert.equal(output.fetchCalls[0], "https://allowed.example/article");
+	assert.match(output.results[0].error, /Blocked hostname by fetch_content domain policy/);
+	assert.equal(output.results[1].error, null);
+	assert.equal(output.results[1].title, "Allowed");
+});
+
+test("fetch_content domain policy leaves local file inputs outside hostname checks", async () => {
+	const output = await runExtract(
+		{ fetchContent: { domainPolicy: { allow: ["only.example"] } } },
+		["file:///tmp/not-a-web-source.txt"],
+	);
+	assert.deepEqual(output.fetchCalls, []);
+	assert.doesNotMatch(output.results[0].error, /fetch_content domain policy/);
+});
+
+test("fetch_content domain policy blocks YouTube frame and timestamp branches before network helpers", async () => {
+	const output = await runExtract(
+		{ fetchContent: { domainPolicy: { deny: ["youtube.com"] } } },
+		["https://www.youtube.com/watch?v=dQw4w9WgXcQ", "https://www.youtube.com/watch?v=dQw4w9WgXcQ"],
+		[{ frames: 1 }, { timestamp: "1" }],
+	);
+	assert.deepEqual(output.fetchCalls, []);
+	assert.match(output.results[0].error, /Blocked hostname by fetch_content domain policy: www\.youtube\.com/);
+	assert.match(output.results[1].error, /Blocked hostname by fetch_content domain policy: www\.youtube\.com/);
+});
+
+test("fetch_content rejects chunked oversized text responses while reading", async () => {
+	const root = await mkdtemp(join(tmpdir(), "pi-oversized-text-extract-"));
+	const childEnv = { ...process.env, PI_CODING_AGENT_DIR: root, HOME: root, USERPROFILE: root };
+	for (const key of ["GEMINI_API_KEY", "GOOGLE_GEMINI_API_KEY", "GOOGLE_API_KEY", "CLOUDFLARE_API_KEY", "PARALLEL_API_KEY", "TINYFISH_API_KEY", "FIRECRAWL_BASE_URL", "FIRECRAWL_API_KEY"]) delete childEnv[key];
+	const child = spawnSync(process.execPath, ["--input-type=module"], {
+		input: `
+			globalThis.fetch = async () => new Response(new ReadableStream({
+				start(controller) {
+					const chunk = new Uint8Array(1024 * 1024).fill(65);
+					for (let i = 0; i < 6; i += 1) controller.enqueue(chunk);
+					controller.close();
+				},
+			}), { status: 200, headers: { "content-type": "text/html" } });
+			const { extractContent } = await import(${JSON.stringify(extractUrl)});
+			const result = await extractContent("https://allowed.example/article", undefined, { lookup: async () => [{ address: "93.184.216.34", family: 4 }] });
+			console.log(JSON.stringify(result));
+		`,
+		encoding: "utf8",
+		env: childEnv,
+		maxBuffer: 2 * 1024 * 1024,
+	});
+	assert.equal(child.status, 0, child.stderr);
+	const result = JSON.parse(child.stdout.trim());
+	assert.match(result.error, /Response too large \(5MB\)/);
+});

package/agent/extensions/pi-web-access/test/fetch-modes.test.mjs

@@ -1,53 +1,53 @@
-import assert from "node:assert/strict";
-import { after, test } from "node:test";
-
-import { extractContent, fetchAllContent } from "../extract.ts";
-
-const originalFetch = globalThis.fetch;
-after(() => { globalThis.fetch = originalFetch; });
-const lookup = async () => [{ address: "93.184.216.34", family: 4 }];
-const png = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", "base64");
-
-test("local HTTP fetch sends the compatible User-Agent", async () => {
-	let userAgent;
-	globalThis.fetch = async (_url, init) => {
-		userAgent = new Headers(init.headers).get("user-agent");
-		return new Response("body", { headers: { "content-type": "text/plain" } });
-	};
-
-	await extractContent("https://example.com/article", undefined, { mode: "raw", lookup });
-	assert.equal(userAgent, "OpenAI File Downloader, XaiImageApiFetch/1.0");
-});
-
-test("raw mode returns textual non-2xx bodies but rejects images", async () => {
-	globalThis.fetch = async (url) => String(url).endsWith(".png")
-		? new Response(png, { status: 200, headers: { "content-type": "image/png" } })
-		: new Response('{"error":"missing"}', { status: 404, headers: { "content-type": "application/json; charset=utf-8" } });
-
-	const text = await extractContent("https://example.com/missing", undefined, { mode: "raw", lookup });
-	assert.equal(text.error, null);
-	assert.equal(text.status, 404);
-	assert.equal(text.content, '{"error":"missing"}');
-
-	const image = await extractContent("https://example.com/pixel.png", undefined, { mode: "raw", lookup });
-	assert.match(image.error, /Unsupported content type in raw mode: image\/png/);
-	assert.equal(image.thumbnail, undefined);
-});
-
-test("raw mode keeps data URIs in the exact HTTP body", async () => {
-	const body = "exact data:text/plain,hello%20world body";
-	globalThis.fetch = async () => new Response(body, { headers: { "content-type": "text/plain" } });
-
-	const [result] = await fetchAllContent(["https://example.com/data"], undefined, { mode: "raw", lookup });
-	assert.equal(result.content, body);
-});
-
-test("readable mode returns supported image content", async () => {
-	globalThis.fetch = async () => new Response(png, { status: 200, headers: { "content-type": "image/png" } });
-	const result = await extractContent("https://example.com/pixel.png", undefined, { lookup });
-
-	assert.equal(result.error, null);
-	assert.equal(result.mimeType, "image/png");
-	assert.equal(result.thumbnail?.mimeType, "image/png");
-	assert.match(result.content, /Image fetched \(1×1, image\/png\)/);
-});
+import assert from "node:assert/strict";
+import { after, test } from "node:test";
+
+import { extractContent, fetchAllContent } from "../extract.ts";
+
+const originalFetch = globalThis.fetch;
+after(() => { globalThis.fetch = originalFetch; });
+const lookup = async () => [{ address: "93.184.216.34", family: 4 }];
+const png = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", "base64");
+
+test("local HTTP fetch sends the compatible User-Agent", async () => {
+	let userAgent;
+	globalThis.fetch = async (_url, init) => {
+		userAgent = new Headers(init.headers).get("user-agent");
+		return new Response("body", { headers: { "content-type": "text/plain" } });
+	};
+
+	await extractContent("https://example.com/article", undefined, { mode: "raw", lookup });
+	assert.equal(userAgent, "OpenAI File Downloader, XaiImageApiFetch/1.0");
+});
+
+test("raw mode returns textual non-2xx bodies but rejects images", async () => {
+	globalThis.fetch = async (url) => String(url).endsWith(".png")
+		? new Response(png, { status: 200, headers: { "content-type": "image/png" } })
+		: new Response('{"error":"missing"}', { status: 404, headers: { "content-type": "application/json; charset=utf-8" } });
+
+	const text = await extractContent("https://example.com/missing", undefined, { mode: "raw", lookup });
+	assert.equal(text.error, null);
+	assert.equal(text.status, 404);
+	assert.equal(text.content, '{"error":"missing"}');
+
+	const image = await extractContent("https://example.com/pixel.png", undefined, { mode: "raw", lookup });
+	assert.match(image.error, /Unsupported content type in raw mode: image\/png/);
+	assert.equal(image.thumbnail, undefined);
+});
+
+test("raw mode keeps data URIs in the exact HTTP body", async () => {
+	const body = "exact data:text/plain,hello%20world body";
+	globalThis.fetch = async () => new Response(body, { headers: { "content-type": "text/plain" } });
+
+	const [result] = await fetchAllContent(["https://example.com/data"], undefined, { mode: "raw", lookup });
+	assert.equal(result.content, body);
+});
+
+test("readable mode returns supported image content", async () => {
+	globalThis.fetch = async () => new Response(png, { status: 200, headers: { "content-type": "image/png" } });
+	const result = await extractContent("https://example.com/pixel.png", undefined, { lookup });
+
+	assert.equal(result.error, null);
+	assert.equal(result.mimeType, "image/png");
+	assert.equal(result.thumbnail?.mimeType, "image/png");
+	assert.match(result.content, /Image fetched \(1×1, image\/png\)/);
+});

package/agent/extensions/pi-web-access/test/fetch-not-found-guidance.test.mjs

@@ -1,92 +1,92 @@
-import assert from "node:assert/strict";
-import { test } from "node:test";
-
-import { extractContent } from "../extract.ts";
-
-const originalFetch = globalThis.fetch;
-const lookup = async () => [{ address: "93.184.216.34", family: 4 }];
-const toolNames = { webSearch: "web_search", fetchContent: "fetch_content" };
-
-function stubStatus(t, status, statusText) {
-	t.after(() => { globalThis.fetch = originalFetch; });
-	globalThis.fetch = async () => new Response("gone", { status, statusText });
-}
-
-test("404 drops the provider checklist and points at the registered tools", async (t) => {
-	stubStatus(t, 404, "Not Found");
-
-	const result = await extractContent("https://example.com/missing-page", undefined, { lookup, toolNames });
-
-	assert.equal(result.status, 404);
-	assert.match(result.error, /HTTP 404: Not Found/);
-	assert.doesNotMatch(result.error, /Fallback options:/);
-	assert.match(result.error, /web_search/);
-	assert.match(result.error, /fetch_content/);
-});
-
-test("410 gets the same not-found guidance", async (t) => {
-	stubStatus(t, 410, "Gone");
-
-	const result = await extractContent("https://example.com/retired", undefined, { lookup, toolNames });
-
-	assert.equal(result.status, 410);
-	assert.doesNotMatch(result.error, /Fallback options:/);
-	assert.match(result.error, /web_search/);
-});
-
-test("guidance uses the registered public tool names", async (t) => {
-	stubStatus(t, 404, "Not Found");
-
-	const result = await extractContent("https://example.com/missing-page", undefined, {
-		lookup,
-		toolNames: { webSearch: "webfinder", fetchContent: "pageget" },
-	});
-
-	assert.match(result.error, /webfinder/);
-	assert.match(result.error, /pageget/);
-	assert.doesNotMatch(result.error, /web_search/);
-});
-
-test("guidance stays generic when the caller does not know the tool names", async (t) => {
-	stubStatus(t, 404, "Not Found");
-
-	const result = await extractContent("https://example.com/missing-page", undefined, { lookup });
-
-	assert.doesNotMatch(result.error, /Fallback options:/);
-	assert.doesNotMatch(result.error, /web_search/);
-	assert.match(result.error, /Find the current URL, then retry the fetch/);
-});
-
-test("guidance omits the fetch tool when only search is registered", async (t) => {
-	stubStatus(t, 404, "Not Found");
-
-	const result = await extractContent("https://example.com/missing-page", undefined, {
-		lookup,
-		toolNames: { webSearch: "web_search" },
-	});
-
-	assert.match(result.error, /web_search/);
-	assert.doesNotMatch(result.error, /fetch_content/);
-});
-
-test("transient errors keep the fallback checklist", async (t) => {
-	stubStatus(t, 500, "Internal Server Error");
-
-	const result = await extractContent("https://example.com/oops", undefined, { lookup, toolNames });
-
-	assert.equal(result.status, 500);
-	assert.match(result.error, /Fallback options:/);
-	assert.match(result.error, /Use web_search to find content about this topic/);
-});
-
-test("the checklist search bullet follows the registered search tool name", async (t) => {
-	stubStatus(t, 500, "Internal Server Error");
-
-	const result = await extractContent("https://example.com/oops", undefined, {
-		lookup,
-		toolNames: { webSearch: "webfinder" },
-	});
-
-	assert.match(result.error, /Use webfinder to find content about this topic/);
-	assert.doesNotMatch(result.error, /web_search/);
-});
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { extractContent } from "../extract.ts";
+
+const originalFetch = globalThis.fetch;
+const lookup = async () => [{ address: "93.184.216.34", family: 4 }];
+const toolNames = { webSearch: "web_search", fetchContent: "fetch_content" };
+
+function stubStatus(t, status, statusText) {
+	t.after(() => { globalThis.fetch = originalFetch; });
+	globalThis.fetch = async () => new Response("gone", { status, statusText });
+}
+
+test("404 drops the provider checklist and points at the registered tools", async (t) => {
+	stubStatus(t, 404, "Not Found");
+
+	const result = await extractContent("https://example.com/missing-page", undefined, { lookup, toolNames });
+
+	assert.equal(result.status, 404);
+	assert.match(result.error, /HTTP 404: Not Found/);
+	assert.doesNotMatch(result.error, /Fallback options:/);
+	assert.match(result.error, /web_search/);
+	assert.match(result.error, /fetch_content/);
+});
+
+test("410 gets the same not-found guidance", async (t) => {
+	stubStatus(t, 410, "Gone");
+
+	const result = await extractContent("https://example.com/retired", undefined, { lookup, toolNames });
+
+	assert.equal(result.status, 410);
+	assert.doesNotMatch(result.error, /Fallback options:/);
+	assert.match(result.error, /web_search/);
+});
+
+test("guidance uses the registered public tool names", async (t) => {
+	stubStatus(t, 404, "Not Found");
+
+	const result = await extractContent("https://example.com/missing-page", undefined, {
+		lookup,
+		toolNames: { webSearch: "webfinder", fetchContent: "pageget" },
+	});
+
+	assert.match(result.error, /webfinder/);
+	assert.match(result.error, /pageget/);
+	assert.doesNotMatch(result.error, /web_search/);
+});
+
+test("guidance stays generic when the caller does not know the tool names", async (t) => {
+	stubStatus(t, 404, "Not Found");
+
+	const result = await extractContent("https://example.com/missing-page", undefined, { lookup });
+
+	assert.doesNotMatch(result.error, /Fallback options:/);
+	assert.doesNotMatch(result.error, /web_search/);
+	assert.match(result.error, /Find the current URL, then retry the fetch/);
+});
+
+test("guidance omits the fetch tool when only search is registered", async (t) => {
+	stubStatus(t, 404, "Not Found");
+
+	const result = await extractContent("https://example.com/missing-page", undefined, {
+		lookup,
+		toolNames: { webSearch: "web_search" },
+	});
+
+	assert.match(result.error, /web_search/);
+	assert.doesNotMatch(result.error, /fetch_content/);
+});
+
+test("transient errors keep the fallback checklist", async (t) => {
+	stubStatus(t, 500, "Internal Server Error");
+
+	const result = await extractContent("https://example.com/oops", undefined, { lookup, toolNames });
+
+	assert.equal(result.status, 500);
+	assert.match(result.error, /Fallback options:/);
+	assert.match(result.error, /Use web_search to find content about this topic/);
+});
+
+test("the checklist search bullet follows the registered search tool name", async (t) => {
+	stubStatus(t, 500, "Internal Server Error");
+
+	const result = await extractContent("https://example.com/oops", undefined, {
+		lookup,
+		toolNames: { webSearch: "webfinder" },
+	});
+
+	assert.match(result.error, /Use webfinder to find content about this topic/);
+	assert.doesNotMatch(result.error, /web_search/);
+});