@asterxsk/kiln 0.2.0 → 0.3.0

package/agent.bak.20260905-190020/extensions/pi-web-access/auth-fetch.ts

@@ -0,0 +1,148 @@
+import { existsSync, readFileSync } from "node:fs";
+import { getWebSearchConfigPath } from "./utils.ts";
+
+const WEB_SEARCH_CONFIG_PATH = getWebSearchConfigPath();
+const AUTH_PROFILE_NAME_PATTERN = /^[A-Za-z][A-Za-z0-9_-]{0,63}$/;
+
+export type AuthFetchRequest = true | string;
+export type AuthFetchCache = "session" | "off";
+
+export interface AuthFetchProfile {
+	name: string;
+	hosts: string[];
+	chromeProfile?: string;
+	redirects: "same-origin";
+	cache: AuthFetchCache;
+}
+
+interface AuthFetchConfigRoot {
+	authFetch?: unknown;
+}
+
+export function resolveAuthFetchProfile(request: AuthFetchRequest): AuthFetchProfile {
+	const profiles = loadAuthFetchProfiles();
+	if (profiles.length === 0) {
+		throw new Error(`auth requires at least one authFetch profile in ${WEB_SEARCH_CONFIG_PATH}`);
+	}
+	if (request === true) {
+		if (profiles.length !== 1) {
+			throw new Error("auth: true requires exactly one authFetch profile; use a profile name instead");
+		}
+		return profiles[0];
+	}
+	const name = request.trim();
+	const profile = profiles.find(candidate => candidate.name === name);
+	if (!profile) throw new Error(`Unknown authFetch profile: ${name}`);
+	return profile;
+}
+
+export function assertAuthFetchUrl(profile: AuthFetchProfile, rawUrl: string): URL {
+	let url: URL;
+	try {
+		url = new URL(rawUrl);
+	} catch (err) {
+		const message = err instanceof Error ? err.message : String(err);
+		throw new Error(`Authenticated fetch requires an HTTPS URL: ${message}`);
+	}
+	if (url.protocol !== "https:") throw new Error("Authenticated fetch requires an HTTPS URL");
+	const hostname = normalizeHostname(url.hostname);
+	if (!profile.hosts.some(host => hostMatches(hostname, host))) {
+		throw new Error(`URL host ${hostname} is not allowed by authFetch profile ${profile.name}`);
+	}
+	return url;
+}
+
+export function authFetchRedirectGuard(profile: AuthFetchProfile, from: URL, to: URL): void {
+	if (profile.redirects === "same-origin" && to.origin !== from.origin) {
+		throw new Error(`Authenticated fetch refused cross-origin redirect: ${from.origin} -> ${to.origin}`);
+	}
+}
+
+function loadAuthFetchProfiles(): AuthFetchProfile[] {
+	if (!existsSync(WEB_SEARCH_CONFIG_PATH)) return [];
+	const raw = readFileSync(WEB_SEARCH_CONFIG_PATH, "utf-8");
+	let parsed: AuthFetchConfigRoot;
+	try {
+		const value: unknown = JSON.parse(raw);
+		if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("expected a JSON object");
+		parsed = value as AuthFetchConfigRoot;
+	} catch (err) {
+		const message = err instanceof Error ? err.message : String(err);
+		throw new Error(`Failed to parse ${WEB_SEARCH_CONFIG_PATH}: ${message}`);
+	}
+	if (parsed.authFetch === undefined || parsed.authFetch === null) return [];
+	if (typeof parsed.authFetch !== "object" || Array.isArray(parsed.authFetch)) {
+		throw new Error(`authFetch in ${WEB_SEARCH_CONFIG_PATH} must be an object`);
+	}
+	return Object.entries(parsed.authFetch as Record<string, unknown>).map(([name, value]) => parseProfile(name, value));
+}
+
+function parseProfile(name: string, value: unknown): AuthFetchProfile {
+	if (!AUTH_PROFILE_NAME_PATTERN.test(name)) {
+		throw new Error(`authFetch profile name ${JSON.stringify(name)} must start with a letter and contain only letters, numbers, underscores, or hyphens`);
+	}
+	if (Array.isArray(value)) {
+		return { name, hosts: parseHosts(value, `authFetch.${name}`), redirects: "same-origin", cache: "session" };
+	}
+	if (!value || typeof value !== "object") {
+		throw new Error(`authFetch.${name} in ${WEB_SEARCH_CONFIG_PATH} must be an array of hosts or an object`);
+	}
+	const config = value as Record<string, unknown>;
+	if (!Array.isArray(config.hosts)) {
+		throw new Error(`authFetch.${name}.hosts in ${WEB_SEARCH_CONFIG_PATH} must be a non-empty array of hostnames`);
+	}
+	const redirects = config.redirects ?? "same-origin";
+	if (redirects !== "same-origin") {
+		throw new Error(`authFetch.${name}.redirects in ${WEB_SEARCH_CONFIG_PATH} must be "same-origin"`);
+	}
+	const cache = config.cache ?? "session";
+	if (cache !== "session" && cache !== "off") {
+		throw new Error(`authFetch.${name}.cache in ${WEB_SEARCH_CONFIG_PATH} must be "session" or "off"`);
+	}
+	const chromeProfile = parseChromeProfile(config.chromeProfile, `authFetch.${name}.chromeProfile`);
+	return {
+		name,
+		hosts: parseHosts(config.hosts, `authFetch.${name}.hosts`),
+		...(chromeProfile ? { chromeProfile } : {}),
+		redirects,
+		cache,
+	};
+}
+
+function parseHosts(value: unknown[], label: string): string[] {
+	if (value.length === 0) throw new Error(`${label} in ${WEB_SEARCH_CONFIG_PATH} must be a non-empty array of hostnames`);
+	const hosts = value.map((entry) => {
+		if (typeof entry !== "string") throw new Error(`${label} in ${WEB_SEARCH_CONFIG_PATH} must contain only hostnames`);
+		return parseHost(entry, label);
+	});
+	return [...new Set(hosts)];
+}
+
+function parseHost(value: string, label: string): string {
+	const host = normalizeHostname(value.trim());
+	if (!host || host.startsWith(".") || host.endsWith(".") || /\s|[\\/?:#@*]/.test(host)) {
+		throw new Error(`${label} in ${WEB_SEARCH_CONFIG_PATH} contains an invalid hostname: ${JSON.stringify(value)}`);
+	}
+	if (host.length > 253 || !/^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)*[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/i.test(host)) {
+		throw new Error(`${label} in ${WEB_SEARCH_CONFIG_PATH} contains an invalid hostname: ${JSON.stringify(value)}`);
+	}
+	return host;
+}
+
+function parseChromeProfile(value: unknown, label: string): string | undefined {
+	if (value === undefined || value === null) return undefined;
+	if (typeof value !== "string") throw new Error(`${label} in ${WEB_SEARCH_CONFIG_PATH} must be a string`);
+	const normalized = value.trim();
+	if (!normalized || normalized === "." || normalized === ".." || normalized.includes("/") || normalized.includes("\\")) {
+		throw new Error(`${label} in ${WEB_SEARCH_CONFIG_PATH} must be a profile directory name, not a path`);
+	}
+	return normalized;
+}
+
+function normalizeHostname(hostname: string): string {
+	return hostname.toLowerCase().replace(/^\[|\]$/g, "").replace(/\.$/, "");
+}
+
+function hostMatches(hostname: string, allowedHost: string): boolean {
+	return hostname === allowedHost || hostname.endsWith(`.${allowedHost}`);
+}

package/agent.bak.20260905-190020/extensions/pi-web-access/brightdata-unlocker.ts

@@ -0,0 +1,272 @@
+import { existsSync, readFileSync } from "node:fs";
+import { activityMonitor } from "./activity.ts";
+import { hasCredentialSource, redactCredential, resolveCredential } from "./credential-source.ts";
+import type { ExtractedContent, ExtractOptions } from "./extract.ts";
+import { validateRemoteUrl, type Lookup } from "./ssrf-protection.ts";
+import { getWebSearchConfigPath } from "./utils.ts";
+
+const CONFIG_PATH = getWebSearchConfigPath();
+const BRIGHTDATA_REQUEST_URL = "https://api.brightdata.com/request";
+const EXTRACT_TIMEOUT_MS = 60_000;
+const DEFAULT_MAX_REDIRECTS = 5;
+const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
+const ZONE_PATTERN = /^[a-z0-9_-]+$/i;
+
+export interface BrightDataSsrfOptions {
+	allowRanges: string[];
+	trustEnvProxy: boolean;
+}
+
+export interface BrightDataExtractOptions extends Pick<ExtractOptions, "timeoutMs" | "lookup"> {
+	ssrf?: BrightDataSsrfOptions;
+}
+
+interface BrightDataConfig {
+	brightdataApiKey?: unknown;
+	brightdataUnlockerZone?: unknown;
+}
+
+let cachedConfig: BrightDataConfig | null = null;
+
+// V8's JSON.parse message quotes a slice of the source text around the offending
+// token — `JSON.parse('{"brightdataApiKey": bd-live-abc123}')` reports
+// `Unexpected token 'b', ..."aApiKey": bd-live-ab"... is not valid JSON`. This
+// file is where the API key lives, so echoing that message verbatim (which
+// firecrawl.ts:50 and ssrf-protection.ts:39 both do) puts a fragment of the
+// credential into an error string that extract.ts surfaces to the user. Only the
+// position is safe to repeat; the snippet never is. The `Failed to parse ` prefix
+// is preserved because extract.ts's isConfigParseError matches on it.
+function parseFailureDetail(err: unknown): string {
+	const message = err instanceof Error ? err.message : String(err);
+	const position = message.match(/at position \d+(?: \(line \d+ column \d+\))?/);
+	return position ? `invalid JSON ${position[0]}` : "invalid JSON";
+}
+
+function loadConfig(): BrightDataConfig {
+	if (cachedConfig) return cachedConfig;
+	if (!existsSync(CONFIG_PATH)) {
+		cachedConfig = {};
+		return cachedConfig;
+	}
+	const raw = readFileSync(CONFIG_PATH, "utf8");
+	let parsed: unknown;
+	try {
+		parsed = JSON.parse(raw);
+	} catch (err) {
+		throw new Error(`Failed to parse ${CONFIG_PATH}: ${parseFailureDetail(err)}`);
+	}
+	if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+		throw new Error(`Invalid config in ${CONFIG_PATH}: expected a JSON object`);
+	}
+	cachedConfig = parsed as BrightDataConfig;
+	return cachedConfig;
+}
+
+export function clearBrightDataUnlockerConfigCache(): void {
+	cachedConfig = null;
+}
+
+function normalizeZone(value: unknown): string | null {
+	if (typeof value !== "string") return null;
+	const trimmed = value.trim();
+	if (!trimmed) return null;
+	return ZONE_PATTERN.test(trimmed) ? trimmed : null;
+}
+
+interface ZoneSetting {
+	raw: string;
+	label: string;
+}
+
+function zoneSetting(): ZoneSetting | null {
+	const fromEnv = process.env.BRIGHTDATA_UNLOCKER_ZONE;
+	if (typeof fromEnv === "string" && fromEnv.trim()) return { raw: fromEnv.trim(), label: "BRIGHTDATA_UNLOCKER_ZONE" };
+	const configured = loadConfig().brightdataUnlockerZone;
+	if (typeof configured === "string" && configured.trim()) return { raw: configured.trim(), label: `brightdataUnlockerZone in ${CONFIG_PATH}` };
+	return null;
+}
+
+function getZone(): string | null {
+	const setting = zoneSetting();
+	return setting ? normalizeZone(setting.raw) : null;
+}
+
+// A zone is never defaulted. Zones are per-account names bound to a product
+// type, they are billed separately, and a guess is either an HTTP 400 or a
+// charge against the wrong product. The SERP zone from the search provider is
+// a different zone type and cannot serve Web Unlocker requests.
+function requireZone(): string {
+	const setting = zoneSetting();
+	const zone = setting ? normalizeZone(setting.raw) : null;
+	if (zone) return zone;
+	if (setting) {
+		throw new Error(
+			`Invalid Bright Data Unlocker zone: ${setting.label} must be a zone name of letters, digits, "-", or "_". ` +
+			"The zone must be of type \"unblocker\"; a SERP zone will not serve Web Unlocker requests.",
+		);
+	}
+	throw new Error(
+		"Bright Data Web Unlocker zone not configured. Either:\n" +
+		`  1. Set brightdataUnlockerZone in ${CONFIG_PATH}\n` +
+		"  2. Set BRIGHTDATA_UNLOCKER_ZONE environment variable\n" +
+		"The zone must be of type \"unblocker\"; a SERP zone will not serve Web Unlocker requests.",
+	);
+}
+
+async function getApiKey(signal?: AbortSignal): Promise<string> {
+	const apiKey = await resolveCredential({
+		provider: "Bright Data",
+		configuredValue: loadConfig().brightdataApiKey,
+		environmentValue: process.env.BRIGHTDATA_API_KEY,
+		signal,
+	});
+	if (!apiKey) {
+		throw new Error(
+			"Bright Data API key not found. Either:\n" +
+			`  1. Create ${CONFIG_PATH} with { "brightdataApiKey": "your-key" }\n` +
+			"  2. Set BRIGHTDATA_API_KEY environment variable\n" +
+			"Get a key at https://brightdata.com/cp/setting/users",
+		);
+	}
+	return apiKey;
+}
+
+function requestSignal(timeoutMs: number, signal?: AbortSignal): AbortSignal {
+	const timeout = AbortSignal.timeout(timeoutMs);
+	return signal ? AbortSignal.any([timeout, signal]) : timeout;
+}
+
+function errorMessage(err: unknown): string {
+	return err instanceof Error ? err.message : String(err);
+}
+
+function isAbortError(err: unknown): boolean {
+	return errorMessage(err).toLowerCase().includes("abort");
+}
+
+function ssrfOptions(options?: BrightDataExtractOptions): { lookup?: Lookup; allowRanges: string[]; trustEnvProxy: boolean } {
+	return {
+		allowRanges: options?.ssrf?.allowRanges ?? [],
+		trustEnvProxy: options?.ssrf?.trustEnvProxy ?? false,
+		...(options?.lookup ? { lookup: options.lookup } : {}),
+	};
+}
+
+function withoutSensitiveHeaders(headers: Record<string, string>): Record<string, string> {
+	const next = { ...headers };
+	delete next.Authorization;
+	delete next.authorization;
+	delete next.Cookie;
+	delete next.cookie;
+	delete next["X-API-Key"];
+	delete next["x-api-key"];
+	return next;
+}
+
+async function fetchBrightDataApi(
+	url: string,
+	init: { method: string; headers: Record<string, string>; body: string; signal: AbortSignal },
+	options: BrightDataExtractOptions | undefined,
+): Promise<Response> {
+	let current = await validateRemoteUrl(url, ssrfOptions(options));
+	let headers = init.headers;
+	// `redirect: "manual"` is what makes this loop exist: with the default "follow"
+	// the runtime resolves the chain itself, the final response is a 200, and the
+	// per-hop validateRemoteUrl / credential strip / hop cap below all become dead
+	// code. Tests assert the option is present on every hop for that reason.
+	//
+	// The loop is intentionally unbounded (`for (;;)`) with the cap enforced inside:
+	// a `redirects <= DEFAULT_MAX_REDIRECTS` header would make the post-loop
+	// statement provably unreachable, which is exactly the dead line firecrawl.ts
+	// carries at :186. Every exit is a `return` or a `throw` written here.
+	for (let redirects = 0; ; redirects++) {
+		const response = await fetch(current, { ...init, headers, redirect: "manual" });
+		if (!REDIRECT_STATUSES.has(response.status)) return response;
+
+		const location = response.headers.get("location");
+		if (!location) return response;
+		if (redirects === DEFAULT_MAX_REDIRECTS) throw new Error(`Too many redirects fetching ${current.toString()}`);
+
+		const next = await validateRemoteUrl(new URL(location, current), ssrfOptions(options));
+		if (next.origin !== current.origin) headers = withoutSensitiveHeaders(headers);
+		current = next;
+	}
+}
+
+function unlockerBody(url: string, zone: string): Record<string, unknown> {
+	return {
+		url,
+		zone,
+		format: "raw",
+		data_format: "markdown",
+	};
+}
+
+async function brightDataRequest(
+	url: string,
+	zone: string,
+	signal: AbortSignal | undefined,
+	options: BrightDataExtractOptions | undefined,
+): Promise<string> {
+	const apiKey = await getApiKey(signal);
+	const headers: Record<string, string> = {
+		"Content-Type": "application/json",
+		Authorization: `Bearer ${apiKey}`,
+	};
+	const activityId = activityMonitor.logStart({ type: "fetch", url });
+	try {
+		const response = await fetchBrightDataApi(BRIGHTDATA_REQUEST_URL, {
+			method: "POST",
+			headers,
+			body: JSON.stringify(unlockerBody(url, zone)),
+			signal: requestSignal(options?.timeoutMs ?? EXTRACT_TIMEOUT_MS, signal),
+		}, options);
+		if (!response.ok) {
+			const errorText = await response.text().catch(() => "");
+			throw new Error(`Bright Data Web Unlocker error ${response.status}: ${redactCredential(errorText, apiKey).slice(0, 300)}`);
+		}
+		const text = await response.text();
+		activityMonitor.logComplete(activityId, response.status);
+		return text;
+	} catch (err) {
+		const message = errorMessage(err);
+		const redactedMessage = redactCredential(message, apiKey);
+		if (isAbortError(err)) activityMonitor.logComplete(activityId, 0);
+		else activityMonitor.logError(activityId, redactedMessage);
+		if (redactedMessage === message) throw err;
+		const redactedError = new Error(redactedMessage);
+		if (err instanceof Error) redactedError.name = err.name;
+		throw redactedError;
+	}
+}
+
+function headingTitle(text: string): string {
+	const match = text.match(/^#{1,2}\s+(.+)/m);
+	if (!match) return "";
+	return match[1].replace(/\*+/g, "").trim();
+}
+
+export function isBrightDataUnlockerAvailable(): boolean {
+	if (getZone() === null) return false;
+	return hasCredentialSource({
+		provider: "Bright Data",
+		configuredValue: loadConfig().brightdataApiKey,
+		environmentValue: process.env.BRIGHTDATA_API_KEY,
+	});
+}
+
+export async function extractWithBrightDataUnlocker(
+	url: string,
+	signal?: AbortSignal,
+	options?: BrightDataExtractOptions,
+): Promise<ExtractedContent | null> {
+	const zone = requireZone();
+	await validateRemoteUrl(url, ssrfOptions(options));
+	const raw = await brightDataRequest(url, zone, signal, options);
+	// Bright Data bills a successful request whatever its length, so short
+	// content is returned rather than discarded: a paywall stub or consent page
+	// is a useful answer, and silently dropping it would hide the spend.
+	const content = raw.trim();
+	if (!content) return null;
+	return { url, title: headingTitle(content), content, error: null };
+}