@asterxsk/kiln 0.1.0 → 0.3.0

package/agent.bak.20260905-190020/extensions/pi-web-access/test/fetch-modes.test.mjs

@@ -0,0 +1,53 @@
+import assert from "node:assert/strict";
+import { after, test } from "node:test";
+
+import { extractContent, fetchAllContent } from "../extract.ts";
+
+const originalFetch = globalThis.fetch;
+after(() => { globalThis.fetch = originalFetch; });
+const lookup = async () => [{ address: "93.184.216.34", family: 4 }];
+const png = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", "base64");
+
+test("local HTTP fetch sends the compatible User-Agent", async () => {
+	let userAgent;
+	globalThis.fetch = async (_url, init) => {
+		userAgent = new Headers(init.headers).get("user-agent");
+		return new Response("body", { headers: { "content-type": "text/plain" } });
+	};
+
+	await extractContent("https://example.com/article", undefined, { mode: "raw", lookup });
+	assert.equal(userAgent, "OpenAI File Downloader, XaiImageApiFetch/1.0");
+});
+
+test("raw mode returns textual non-2xx bodies but rejects images", async () => {
+	globalThis.fetch = async (url) => String(url).endsWith(".png")
+		? new Response(png, { status: 200, headers: { "content-type": "image/png" } })
+		: new Response('{"error":"missing"}', { status: 404, headers: { "content-type": "application/json; charset=utf-8" } });
+
+	const text = await extractContent("https://example.com/missing", undefined, { mode: "raw", lookup });
+	assert.equal(text.error, null);
+	assert.equal(text.status, 404);
+	assert.equal(text.content, '{"error":"missing"}');
+
+	const image = await extractContent("https://example.com/pixel.png", undefined, { mode: "raw", lookup });
+	assert.match(image.error, /Unsupported content type in raw mode: image\/png/);
+	assert.equal(image.thumbnail, undefined);
+});
+
+test("raw mode keeps data URIs in the exact HTTP body", async () => {
+	const body = "exact data:text/plain,hello%20world body";
+	globalThis.fetch = async () => new Response(body, { headers: { "content-type": "text/plain" } });
+
+	const [result] = await fetchAllContent(["https://example.com/data"], undefined, { mode: "raw", lookup });
+	assert.equal(result.content, body);
+});
+
+test("readable mode returns supported image content", async () => {
+	globalThis.fetch = async () => new Response(png, { status: 200, headers: { "content-type": "image/png" } });
+	const result = await extractContent("https://example.com/pixel.png", undefined, { lookup });
+
+	assert.equal(result.error, null);
+	assert.equal(result.mimeType, "image/png");
+	assert.equal(result.thumbnail?.mimeType, "image/png");
+	assert.match(result.content, /Image fetched \(1×1, image\/png\)/);
+});

package/agent.bak.20260905-190020/extensions/pi-web-access/test/fetch-not-found-guidance.test.mjs

@@ -0,0 +1,92 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { extractContent } from "../extract.ts";
+
+const originalFetch = globalThis.fetch;
+const lookup = async () => [{ address: "93.184.216.34", family: 4 }];
+const toolNames = { webSearch: "web_search", fetchContent: "fetch_content" };
+
+function stubStatus(t, status, statusText) {
+	t.after(() => { globalThis.fetch = originalFetch; });
+	globalThis.fetch = async () => new Response("gone", { status, statusText });
+}
+
+test("404 drops the provider checklist and points at the registered tools", async (t) => {
+	stubStatus(t, 404, "Not Found");
+
+	const result = await extractContent("https://example.com/missing-page", undefined, { lookup, toolNames });
+
+	assert.equal(result.status, 404);
+	assert.match(result.error, /HTTP 404: Not Found/);
+	assert.doesNotMatch(result.error, /Fallback options:/);
+	assert.match(result.error, /web_search/);
+	assert.match(result.error, /fetch_content/);
+});
+
+test("410 gets the same not-found guidance", async (t) => {
+	stubStatus(t, 410, "Gone");
+
+	const result = await extractContent("https://example.com/retired", undefined, { lookup, toolNames });
+
+	assert.equal(result.status, 410);
+	assert.doesNotMatch(result.error, /Fallback options:/);
+	assert.match(result.error, /web_search/);
+});
+
+test("guidance uses the registered public tool names", async (t) => {
+	stubStatus(t, 404, "Not Found");
+
+	const result = await extractContent("https://example.com/missing-page", undefined, {
+		lookup,
+		toolNames: { webSearch: "webfinder", fetchContent: "pageget" },
+	});
+
+	assert.match(result.error, /webfinder/);
+	assert.match(result.error, /pageget/);
+	assert.doesNotMatch(result.error, /web_search/);
+});
+
+test("guidance stays generic when the caller does not know the tool names", async (t) => {
+	stubStatus(t, 404, "Not Found");
+
+	const result = await extractContent("https://example.com/missing-page", undefined, { lookup });
+
+	assert.doesNotMatch(result.error, /Fallback options:/);
+	assert.doesNotMatch(result.error, /web_search/);
+	assert.match(result.error, /Find the current URL, then retry the fetch/);
+});
+
+test("guidance omits the fetch tool when only search is registered", async (t) => {
+	stubStatus(t, 404, "Not Found");
+
+	const result = await extractContent("https://example.com/missing-page", undefined, {
+		lookup,
+		toolNames: { webSearch: "web_search" },
+	});
+
+	assert.match(result.error, /web_search/);
+	assert.doesNotMatch(result.error, /fetch_content/);
+});
+
+test("transient errors keep the fallback checklist", async (t) => {
+	stubStatus(t, 500, "Internal Server Error");
+
+	const result = await extractContent("https://example.com/oops", undefined, { lookup, toolNames });
+
+	assert.equal(result.status, 500);
+	assert.match(result.error, /Fallback options:/);
+	assert.match(result.error, /Use web_search to find content about this topic/);
+});
+
+test("the checklist search bullet follows the registered search tool name", async (t) => {
+	stubStatus(t, 500, "Internal Server Error");
+
+	const result = await extractContent("https://example.com/oops", undefined, {
+		lookup,
+		toolNames: { webSearch: "webfinder" },
+	});
+
+	assert.match(result.error, /Use webfinder to find content about this topic/);
+	assert.doesNotMatch(result.error, /web_search/);
+});

package/agent.bak.20260905-190020/extensions/pi-web-access/test/fetch-params.test.mjs

@@ -0,0 +1,86 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { normalizeFetchContentParams } from "../fetch-params.ts";
+
+test("fetch_content params fall back to url when urls is an empty array", () => {
+	const normalized = normalizeFetchContentParams({
+		url: "https://example.com/docs",
+		urls: [],
+	});
+
+	assert.deepEqual(normalized.urlList, ["https://example.com/docs"]);
+});
+
+test("fetch_content params keep non-empty urls precedence over url", () => {
+	const normalized = normalizeFetchContentParams({
+		url: "https://example.com/fallback",
+		urls: ["https://example.com/primary"],
+	});
+
+	assert.deepEqual(normalized.urlList, ["https://example.com/primary"]);
+});
+
+test("fetch_content params ignore blank optional strings and blank urls", () => {
+	const normalized = normalizeFetchContentParams({
+		url: "  https://example.com/one  ",
+		urls: ["", " https://example.com/two ", "https://example.com/one"],
+		prompt: "",
+		timestamp: "   ",
+		model: " gemini-3.6-flash ",
+	});
+
+	assert.deepEqual(normalized.urlList, ["https://example.com/two", "https://example.com/one"]);
+	assert.equal(normalized.options.prompt, undefined);
+	assert.equal(normalized.options.timestamp, undefined);
+	assert.equal(normalized.options.model, "gemini-3.6-flash");
+	assert.equal(normalizeFetchContentParams({ model: "" }).options.model, undefined);
+});
+
+test("fetch_content params preserve forceClone only for boolean values", () => {
+	assert.equal(normalizeFetchContentParams({ forceClone: true }).options.forceClone, true);
+	assert.equal(normalizeFetchContentParams({ forceClone: false }).options.forceClone, false);
+	assert.equal(normalizeFetchContentParams({ forceClone: "true" }).options.forceClone, undefined);
+});
+
+test("fetch_content params drop out-of-range, non-positive, and non-integer frames", () => {
+	assert.equal(normalizeFetchContentParams({ frames: 13, timestamp: "1:23" }).options.frames, undefined);
+	assert.equal(normalizeFetchContentParams({ frames: 0, timestamp: "1:23" }).options.frames, undefined);
+	assert.equal(normalizeFetchContentParams({ frames: -1, timestamp: "1:23" }).options.frames, undefined);
+	assert.equal(normalizeFetchContentParams({ frames: 1.5, timestamp: "1:23" }).options.frames, undefined);
+	assert.equal(normalizeFetchContentParams({ frames: "1", timestamp: "1:23" }).options.frames, undefined);
+});
+
+test("fetch_content params ignore bridge-filled default frames for ordinary page fetches", () => {
+	const normalized = normalizeFetchContentParams({
+		url: "https://example.com/docs",
+		urls: [],
+		frames: 1,
+		prompt: "Summarize this page",
+		timestamp: "",
+	});
+
+	assert.equal(normalized.options.frames, undefined);
+});
+
+test("fetch_content params preserve explicit video frame options", () => {
+	assert.equal(normalizeFetchContentParams({ url: "https://youtu.be/demo", frames: 1, timestamp: "1:23" }).options.frames, 1);
+	assert.equal(normalizeFetchContentParams({ url: "https://youtu.be/demo", frames: 2 }).options.frames, 2);
+});
+
+test("fetch_content params validate fetch and answer modes", () => {
+	assert.deepEqual(
+		normalizeFetchContentParams({ mode: "answer", answerModel: " test/page-model " }).options,
+		{ mode: "answer", answerModel: "test/page-model" },
+	);
+	assert.throws(() => normalizeFetchContentParams({ mode: "invalid" }), /mode must be/);
+});
+
+
+test("fetch_content params validate auth profile input", () => {
+	assert.equal(normalizeFetchContentParams({ auth: true }).options.auth, true);
+	assert.equal(normalizeFetchContentParams({ auth: " work " }).options.auth, "work");
+	assert.equal(normalizeFetchContentParams({ auth: false }).options.auth, undefined);
+	assert.throws(() => normalizeFetchContentParams({ auth: " " }), /auth must be/);
+	assert.throws(() => normalizeFetchContentParams({ auth: 1 }), /auth must be/);
+});

package/agent.bak.20260905-190020/extensions/pi-web-access/test/fetch-render-call.test.mjs

@@ -0,0 +1,34 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import initializeExtension from "../index.ts";
+
+function getFetchTool() {
+	const tools = [];
+	initializeExtension({
+		registerTool(tool) { tools.push(tool); },
+		registerCommand() {},
+		registerShortcut() {},
+		on() {},
+		appendEntry() {},
+	});
+	return tools.find(tool => tool.name === "fetch_content");
+}
+
+const theme = {
+	bold: text => text,
+	fg: (_name, text) => text,
+};
+
+test("fetch_content renderCall falls back to url when urls is empty", () => {
+	const tool = getFetchTool();
+	const lines = tool.renderCall({
+		url: "https://example.com/docs",
+		urls: [],
+		frames: 1,
+		prompt: "",
+		model: "",
+	}, theme).render(120).map(line => line.trimEnd());
+
+	assert.deepEqual(lines, ["fetch https://example.com/docs"]);
+});

package/agent.bak.20260905-190020/extensions/pi-web-access/test/fetch-routing.test.mjs

@@ -0,0 +1,173 @@
+import assert from "node:assert/strict";
+import { spawnSync } from "node:child_process";
+import { mkdtemp, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { test } from "node:test";
+
+const extractUrl = new URL("../extract.ts", import.meta.url).href;
+const featureConfigUrl = new URL("../feature-config.ts", import.meta.url).href;
+
+function cleanProviderEnv(root) {
+	const childEnv = { ...process.env, PI_CODING_AGENT_DIR: root, HOME: root, USERPROFILE: root };
+	for (const key of [
+		"FIRECRAWL_BASE_URL", "FIRECRAWL_API_KEY", "PARALLEL_API_KEY", "TINYFISH_API_KEY",
+		"SEARCH1API_KEY", "SEARCH1API_API_KEY", "QUERIT_API_KEY", "KAGI_API_KEY", "OLLAMA_API_KEY",
+		"BRIGHTDATA_API_KEY", "BRIGHTDATA_UNLOCKER_ZONE", "GEMINI_API_KEY", "GOOGLE_GEMINI_API_KEY", "GOOGLE_API_KEY",
+	]) delete childEnv[key];
+	return childEnv;
+}
+
+async function runExtract(config) {
+	const root = await mkdtemp(join(tmpdir(), "pi-fetch-routing-"));
+	await writeFile(join(root, "web-search.json"), JSON.stringify(config) + "\n", "utf8");
+	const childEnv = cleanProviderEnv(root);
+
+	const child = spawnSync(process.execPath, ["--input-type=module"], {
+		input: `
+			const calls = [];
+			globalThis.fetch = async (url) => {
+				const text = String(url);
+				calls.push(text);
+				if (text.startsWith("https://r.jina.ai/")) {
+					return new Response("Markdown Content:\\n# Routed\\n\\n" + "Jina routed content. ".repeat(12), { status: 200 });
+				}
+				return new Response("blocked", { status: 403 });
+			};
+			const { extractContent } = await import(${JSON.stringify(extractUrl)});
+			const result = await extractContent("https://example.com/routed", undefined, { lookup: async () => [{ address: "93.184.216.34", family: 4 }] });
+			console.log(JSON.stringify({ calls, result }));
+		`,
+		encoding: "utf8",
+		env: childEnv,
+		maxBuffer: 2 * 1024 * 1024,
+	});
+	assert.equal(child.status, 0, child.stderr);
+	return JSON.parse(child.stdout.trim());
+}
+
+async function runTypedExtract(config, contentType) {
+	const root = await mkdtemp(join(tmpdir(), "pi-fetch-routing-typed-"));
+	await writeFile(join(root, "web-search.json"), typeof config === "string" ? config : JSON.stringify(config) + "\n", "utf8");
+	const childEnv = cleanProviderEnv(root);
+	const child = spawnSync(process.execPath, ["--input-type=module"], {
+		input: `
+			const calls = [];
+			globalThis.fetch = async (url) => {
+				const text = String(url);
+				calls.push(text);
+				if (text === "https://example.com/typed") {
+					return new Response("typed", { status: 200, headers: { "content-type": ${JSON.stringify(contentType)} } });
+				}
+				if (text.startsWith("https://r.jina.ai/")) {
+					return new Response("Markdown Content:\\n# Bypassed\\n\\n" + "content ".repeat(80), { status: 200 });
+				}
+				throw new Error("Unexpected fetch " + text);
+			};
+			const { extractContent } = await import(${JSON.stringify(extractUrl)});
+			const result = await extractContent("https://example.com/typed", undefined, { lookup: async () => [{ address: "93.184.216.34", family: 4 }] });
+			console.log(JSON.stringify({ calls, result }));
+		`,
+		encoding: "utf8",
+		env: childEnv,
+		maxBuffer: 2 * 1024 * 1024,
+	});
+	assert.equal(child.status, 0, child.stderr);
+	return JSON.parse(child.stdout.trim());
+}
+
+test("fetchRouting.providers can put Jina first after explicit remote-hosted opt-in", async () => {
+	const output = await runExtract({ fetchRouting: { providers: ["jina", "http"], allowRemoteHostedProviders: true } });
+	assert.deepEqual(output.calls, [
+		"https://example.com/routed",
+		"https://r.jina.ai/https://example.com/routed",
+	]);
+	assert.equal(output.result.error, null);
+	assert.equal(output.result.title, "Routed");
+});
+
+test("remote hosted fetch providers are disabled by default", async () => {
+	const output = await runExtract({});
+	assert.deepEqual(output.calls, ["https://example.com/routed"]);
+	assert.match(output.result.error, /HTTP 403/);
+});
+
+test("fetchRouting without providers uses the default order when remote hosted providers are allowed", async () => {
+	const output = await runExtract({ fetchRouting: { allowRemoteHostedProviders: true } });
+	assert.deepEqual(output.calls, [
+		"https://example.com/routed",
+		"https://r.jina.ai/https://example.com/routed",
+	]);
+	assert.equal(output.result.error, null);
+	assert.equal(output.result.title, "Routed");
+});
+
+test("disabled image fetching does not fall through to hosted providers", async () => {
+	const output = await runTypedExtract({ image: { enabled: false }, fetchRouting: { providers: ["http", "jina"], allowRemoteHostedProviders: true } }, "image/png");
+	assert.deepEqual(output.calls, ["https://example.com/typed"]);
+	assert.match(output.result.error, /Image fetching is disabled by image\.enabled/);
+});
+
+test("disabled PDF extraction does not fall through to hosted providers", async () => {
+	const output = await runTypedExtract({ pdf: { enabled: false }, fetchRouting: { providers: ["http", "jina"], allowRemoteHostedProviders: true } }, "application/pdf");
+	assert.deepEqual(output.calls, ["https://example.com/typed"]);
+	assert.match(output.result.error, /PDF extraction is disabled by pdf\.enabled/);
+});
+
+test("malformed config returns a parse error without hosted fallback", async () => {
+	const output = await runTypedExtract("{", "image/png");
+	assert.deepEqual(output.calls, []);
+	assert.match(output.result.error, /Failed to parse .*web-search\.json/);
+});
+
+test("image attachment gate suppresses malformed config", async () => {
+	const root = await mkdtemp(join(tmpdir(), "pi-feature-config-"));
+	await writeFile(join(root, "web-search.json"), "{", "utf8");
+	const child = spawnSync(process.execPath, ["--input-type=module"], {
+		input: `
+			process.env.PI_CODING_AGENT_DIR = ${JSON.stringify(root)};
+			const { canAttachImages, isImageEnabled } = await import(${JSON.stringify(featureConfigUrl)});
+			let parseError = "";
+			try { isImageEnabled(); } catch (err) { parseError = err instanceof Error ? err.message : String(err); }
+			console.log(JSON.stringify({ canAttach: canAttachImages(), parseError }));
+		`,
+		encoding: "utf8",
+		env: cleanProviderEnv(root),
+	});
+	assert.equal(child.status, 0, child.stderr);
+	const output = JSON.parse(child.stdout.trim());
+	assert.equal(output.canAttach, false);
+	assert.match(output.parseError, /Failed to parse .*web-search\.json/);
+});
+
+test("hosted providers cannot bypass redirect policy validation", async () => {
+	const root = await mkdtemp(join(tmpdir(), "pi-fetch-routing-redirect-"));
+	await writeFile(join(root, "web-search.json"), JSON.stringify({ fetchRouting: { providers: ["jina"], allowRemoteHostedProviders: true } }) + "\n", "utf8");
+	const childEnv = { ...process.env, PI_CODING_AGENT_DIR: root, HOME: root, USERPROFILE: root };
+	const child = spawnSync(process.execPath, ["--input-type=module"], {
+		input: `
+			const calls = [];
+			globalThis.fetch = async (url) => {
+				const text = String(url);
+				calls.push(text);
+				if (text === "https://example.com/redirect") {
+					return new Response("", { status: 302, headers: { location: "http://127.0.0.1/admin" } });
+				}
+				if (text.startsWith("https://r.jina.ai/")) {
+					return new Response("Markdown Content:\\n# Bypassed\\n\\n" + "content ".repeat(80), { status: 200 });
+				}
+				throw new Error("Unexpected fetch " + text);
+			};
+			const { extractContent } = await import(${JSON.stringify(extractUrl)});
+			const result = await extractContent("https://example.com/redirect", undefined, { lookup: async () => [{ address: "93.184.216.34", family: 4 }] });
+			console.log(JSON.stringify({ calls, result }));
+		`,
+		encoding: "utf8",
+		env: childEnv,
+		maxBuffer: 2 * 1024 * 1024,
+	});
+	assert.equal(child.status, 0, child.stderr);
+	const output = JSON.parse(child.stdout.trim());
+	assert.deepEqual(output.calls, ["https://example.com/redirect"]);
+	assert.match(output.result.error, /Blocked internal address/);
+});