@asterxsk/kiln 0.3.0 → 0.4.0

package/agent.bak.20260905-190020/extensions/pi-web-access/auth-fetch.ts

@@ -1,148 +0,0 @@
-import { existsSync, readFileSync } from "node:fs";
-import { getWebSearchConfigPath } from "./utils.ts";
-
-const WEB_SEARCH_CONFIG_PATH = getWebSearchConfigPath();
-const AUTH_PROFILE_NAME_PATTERN = /^[A-Za-z][A-Za-z0-9_-]{0,63}$/;
-
-export type AuthFetchRequest = true | string;
-export type AuthFetchCache = "session" | "off";
-
-export interface AuthFetchProfile {
-	name: string;
-	hosts: string[];
-	chromeProfile?: string;
-	redirects: "same-origin";
-	cache: AuthFetchCache;
-}
-
-interface AuthFetchConfigRoot {
-	authFetch?: unknown;
-}
-
-export function resolveAuthFetchProfile(request: AuthFetchRequest): AuthFetchProfile {
-	const profiles = loadAuthFetchProfiles();
-	if (profiles.length === 0) {
-		throw new Error(`auth requires at least one authFetch profile in ${WEB_SEARCH_CONFIG_PATH}`);
-	}
-	if (request === true) {
-		if (profiles.length !== 1) {
-			throw new Error("auth: true requires exactly one authFetch profile; use a profile name instead");
-		}
-		return profiles[0];
-	}
-	const name = request.trim();
-	const profile = profiles.find(candidate => candidate.name === name);
-	if (!profile) throw new Error(`Unknown authFetch profile: ${name}`);
-	return profile;
-}
-
-export function assertAuthFetchUrl(profile: AuthFetchProfile, rawUrl: string): URL {
-	let url: URL;
-	try {
-		url = new URL(rawUrl);
-	} catch (err) {
-		const message = err instanceof Error ? err.message : String(err);
-		throw new Error(`Authenticated fetch requires an HTTPS URL: ${message}`);
-	}
-	if (url.protocol !== "https:") throw new Error("Authenticated fetch requires an HTTPS URL");
-	const hostname = normalizeHostname(url.hostname);
-	if (!profile.hosts.some(host => hostMatches(hostname, host))) {
-		throw new Error(`URL host ${hostname} is not allowed by authFetch profile ${profile.name}`);
-	}
-	return url;
-}
-
-export function authFetchRedirectGuard(profile: AuthFetchProfile, from: URL, to: URL): void {
-	if (profile.redirects === "same-origin" && to.origin !== from.origin) {
-		throw new Error(`Authenticated fetch refused cross-origin redirect: ${from.origin} -> ${to.origin}`);
-	}
-}
-
-function loadAuthFetchProfiles(): AuthFetchProfile[] {
-	if (!existsSync(WEB_SEARCH_CONFIG_PATH)) return [];
-	const raw = readFileSync(WEB_SEARCH_CONFIG_PATH, "utf-8");
-	let parsed: AuthFetchConfigRoot;
-	try {
-		const value: unknown = JSON.parse(raw);
-		if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("expected a JSON object");
-		parsed = value as AuthFetchConfigRoot;
-	} catch (err) {
-		const message = err instanceof Error ? err.message : String(err);
-		throw new Error(`Failed to parse ${WEB_SEARCH_CONFIG_PATH}: ${message}`);
-	}
-	if (parsed.authFetch === undefined || parsed.authFetch === null) return [];
-	if (typeof parsed.authFetch !== "object" || Array.isArray(parsed.authFetch)) {
-		throw new Error(`authFetch in ${WEB_SEARCH_CONFIG_PATH} must be an object`);
-	}
-	return Object.entries(parsed.authFetch as Record<string, unknown>).map(([name, value]) => parseProfile(name, value));
-}
-
-function parseProfile(name: string, value: unknown): AuthFetchProfile {
-	if (!AUTH_PROFILE_NAME_PATTERN.test(name)) {
-		throw new Error(`authFetch profile name ${JSON.stringify(name)} must start with a letter and contain only letters, numbers, underscores, or hyphens`);
-	}
-	if (Array.isArray(value)) {
-		return { name, hosts: parseHosts(value, `authFetch.${name}`), redirects: "same-origin", cache: "session" };
-	}
-	if (!value || typeof value !== "object") {
-		throw new Error(`authFetch.${name} in ${WEB_SEARCH_CONFIG_PATH} must be an array of hosts or an object`);
-	}
-	const config = value as Record<string, unknown>;
-	if (!Array.isArray(config.hosts)) {
-		throw new Error(`authFetch.${name}.hosts in ${WEB_SEARCH_CONFIG_PATH} must be a non-empty array of hostnames`);
-	}
-	const redirects = config.redirects ?? "same-origin";
-	if (redirects !== "same-origin") {
-		throw new Error(`authFetch.${name}.redirects in ${WEB_SEARCH_CONFIG_PATH} must be "same-origin"`);
-	}
-	const cache = config.cache ?? "session";
-	if (cache !== "session" && cache !== "off") {
-		throw new Error(`authFetch.${name}.cache in ${WEB_SEARCH_CONFIG_PATH} must be "session" or "off"`);
-	}
-	const chromeProfile = parseChromeProfile(config.chromeProfile, `authFetch.${name}.chromeProfile`);
-	return {
-		name,
-		hosts: parseHosts(config.hosts, `authFetch.${name}.hosts`),
-		...(chromeProfile ? { chromeProfile } : {}),
-		redirects,
-		cache,
-	};
-}
-
-function parseHosts(value: unknown[], label: string): string[] {
-	if (value.length === 0) throw new Error(`${label} in ${WEB_SEARCH_CONFIG_PATH} must be a non-empty array of hostnames`);
-	const hosts = value.map((entry) => {
-		if (typeof entry !== "string") throw new Error(`${label} in ${WEB_SEARCH_CONFIG_PATH} must contain only hostnames`);
-		return parseHost(entry, label);
-	});
-	return [...new Set(hosts)];
-}
-
-function parseHost(value: string, label: string): string {
-	const host = normalizeHostname(value.trim());
-	if (!host || host.startsWith(".") || host.endsWith(".") || /\s|[\\/?:#@*]/.test(host)) {
-		throw new Error(`${label} in ${WEB_SEARCH_CONFIG_PATH} contains an invalid hostname: ${JSON.stringify(value)}`);
-	}
-	if (host.length > 253 || !/^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)*[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/i.test(host)) {
-		throw new Error(`${label} in ${WEB_SEARCH_CONFIG_PATH} contains an invalid hostname: ${JSON.stringify(value)}`);
-	}
-	return host;
-}
-
-function parseChromeProfile(value: unknown, label: string): string | undefined {
-	if (value === undefined || value === null) return undefined;
-	if (typeof value !== "string") throw new Error(`${label} in ${WEB_SEARCH_CONFIG_PATH} must be a string`);
-	const normalized = value.trim();
-	if (!normalized || normalized === "." || normalized === ".." || normalized.includes("/") || normalized.includes("\\")) {
-		throw new Error(`${label} in ${WEB_SEARCH_CONFIG_PATH} must be a profile directory name, not a path`);
-	}
-	return normalized;
-}
-
-function normalizeHostname(hostname: string): string {
-	return hostname.toLowerCase().replace(/^\[|\]$/g, "").replace(/\.$/, "");
-}
-
-function hostMatches(hostname: string, allowedHost: string): boolean {
-	return hostname === allowedHost || hostname.endsWith(`.${allowedHost}`);
-}

package/agent.bak.20260905-190020/extensions/pi-web-access/brightdata-unlocker.ts

@@ -1,272 +0,0 @@
-import { existsSync, readFileSync } from "node:fs";
-import { activityMonitor } from "./activity.ts";
-import { hasCredentialSource, redactCredential, resolveCredential } from "./credential-source.ts";
-import type { ExtractedContent, ExtractOptions } from "./extract.ts";
-import { validateRemoteUrl, type Lookup } from "./ssrf-protection.ts";
-import { getWebSearchConfigPath } from "./utils.ts";
-
-const CONFIG_PATH = getWebSearchConfigPath();
-const BRIGHTDATA_REQUEST_URL = "https://api.brightdata.com/request";
-const EXTRACT_TIMEOUT_MS = 60_000;
-const DEFAULT_MAX_REDIRECTS = 5;
-const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
-const ZONE_PATTERN = /^[a-z0-9_-]+$/i;
-
-export interface BrightDataSsrfOptions {
-	allowRanges: string[];
-	trustEnvProxy: boolean;
-}
-
-export interface BrightDataExtractOptions extends Pick<ExtractOptions, "timeoutMs" | "lookup"> {
-	ssrf?: BrightDataSsrfOptions;
-}
-
-interface BrightDataConfig {
-	brightdataApiKey?: unknown;
-	brightdataUnlockerZone?: unknown;
-}
-
-let cachedConfig: BrightDataConfig | null = null;
-
-// V8's JSON.parse message quotes a slice of the source text around the offending
-// token — `JSON.parse('{"brightdataApiKey": bd-live-abc123}')` reports
-// `Unexpected token 'b', ..."aApiKey": bd-live-ab"... is not valid JSON`. This
-// file is where the API key lives, so echoing that message verbatim (which
-// firecrawl.ts:50 and ssrf-protection.ts:39 both do) puts a fragment of the
-// credential into an error string that extract.ts surfaces to the user. Only the
-// position is safe to repeat; the snippet never is. The `Failed to parse ` prefix
-// is preserved because extract.ts's isConfigParseError matches on it.
-function parseFailureDetail(err: unknown): string {
-	const message = err instanceof Error ? err.message : String(err);
-	const position = message.match(/at position \d+(?: \(line \d+ column \d+\))?/);
-	return position ? `invalid JSON ${position[0]}` : "invalid JSON";
-}
-
-function loadConfig(): BrightDataConfig {
-	if (cachedConfig) return cachedConfig;
-	if (!existsSync(CONFIG_PATH)) {
-		cachedConfig = {};
-		return cachedConfig;
-	}
-	const raw = readFileSync(CONFIG_PATH, "utf8");
-	let parsed: unknown;
-	try {
-		parsed = JSON.parse(raw);
-	} catch (err) {
-		throw new Error(`Failed to parse ${CONFIG_PATH}: ${parseFailureDetail(err)}`);
-	}
-	if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
-		throw new Error(`Invalid config in ${CONFIG_PATH}: expected a JSON object`);
-	}
-	cachedConfig = parsed as BrightDataConfig;
-	return cachedConfig;
-}
-
-export function clearBrightDataUnlockerConfigCache(): void {
-	cachedConfig = null;
-}
-
-function normalizeZone(value: unknown): string | null {
-	if (typeof value !== "string") return null;
-	const trimmed = value.trim();
-	if (!trimmed) return null;
-	return ZONE_PATTERN.test(trimmed) ? trimmed : null;
-}
-
-interface ZoneSetting {
-	raw: string;
-	label: string;
-}
-
-function zoneSetting(): ZoneSetting | null {
-	const fromEnv = process.env.BRIGHTDATA_UNLOCKER_ZONE;
-	if (typeof fromEnv === "string" && fromEnv.trim()) return { raw: fromEnv.trim(), label: "BRIGHTDATA_UNLOCKER_ZONE" };
-	const configured = loadConfig().brightdataUnlockerZone;
-	if (typeof configured === "string" && configured.trim()) return { raw: configured.trim(), label: `brightdataUnlockerZone in ${CONFIG_PATH}` };
-	return null;
-}
-
-function getZone(): string | null {
-	const setting = zoneSetting();
-	return setting ? normalizeZone(setting.raw) : null;
-}
-
-// A zone is never defaulted. Zones are per-account names bound to a product
-// type, they are billed separately, and a guess is either an HTTP 400 or a
-// charge against the wrong product. The SERP zone from the search provider is
-// a different zone type and cannot serve Web Unlocker requests.
-function requireZone(): string {
-	const setting = zoneSetting();
-	const zone = setting ? normalizeZone(setting.raw) : null;
-	if (zone) return zone;
-	if (setting) {
-		throw new Error(
-			`Invalid Bright Data Unlocker zone: ${setting.label} must be a zone name of letters, digits, "-", or "_". ` +
-			"The zone must be of type \"unblocker\"; a SERP zone will not serve Web Unlocker requests.",
-		);
-	}
-	throw new Error(
-		"Bright Data Web Unlocker zone not configured. Either:\n" +
-		`  1. Set brightdataUnlockerZone in ${CONFIG_PATH}\n` +
-		"  2. Set BRIGHTDATA_UNLOCKER_ZONE environment variable\n" +
-		"The zone must be of type \"unblocker\"; a SERP zone will not serve Web Unlocker requests.",
-	);
-}
-
-async function getApiKey(signal?: AbortSignal): Promise<string> {
-	const apiKey = await resolveCredential({
-		provider: "Bright Data",
-		configuredValue: loadConfig().brightdataApiKey,
-		environmentValue: process.env.BRIGHTDATA_API_KEY,
-		signal,
-	});
-	if (!apiKey) {
-		throw new Error(
-			"Bright Data API key not found. Either:\n" +
-			`  1. Create ${CONFIG_PATH} with { "brightdataApiKey": "your-key" }\n` +
-			"  2. Set BRIGHTDATA_API_KEY environment variable\n" +
-			"Get a key at https://brightdata.com/cp/setting/users",
-		);
-	}
-	return apiKey;
-}
-
-function requestSignal(timeoutMs: number, signal?: AbortSignal): AbortSignal {
-	const timeout = AbortSignal.timeout(timeoutMs);
-	return signal ? AbortSignal.any([timeout, signal]) : timeout;
-}
-
-function errorMessage(err: unknown): string {
-	return err instanceof Error ? err.message : String(err);
-}
-
-function isAbortError(err: unknown): boolean {
-	return errorMessage(err).toLowerCase().includes("abort");
-}
-
-function ssrfOptions(options?: BrightDataExtractOptions): { lookup?: Lookup; allowRanges: string[]; trustEnvProxy: boolean } {
-	return {
-		allowRanges: options?.ssrf?.allowRanges ?? [],
-		trustEnvProxy: options?.ssrf?.trustEnvProxy ?? false,
-		...(options?.lookup ? { lookup: options.lookup } : {}),
-	};
-}
-
-function withoutSensitiveHeaders(headers: Record<string, string>): Record<string, string> {
-	const next = { ...headers };
-	delete next.Authorization;
-	delete next.authorization;
-	delete next.Cookie;
-	delete next.cookie;
-	delete next["X-API-Key"];
-	delete next["x-api-key"];
-	return next;
-}
-
-async function fetchBrightDataApi(
-	url: string,
-	init: { method: string; headers: Record<string, string>; body: string; signal: AbortSignal },
-	options: BrightDataExtractOptions | undefined,
-): Promise<Response> {
-	let current = await validateRemoteUrl(url, ssrfOptions(options));
-	let headers = init.headers;
-	// `redirect: "manual"` is what makes this loop exist: with the default "follow"
-	// the runtime resolves the chain itself, the final response is a 200, and the
-	// per-hop validateRemoteUrl / credential strip / hop cap below all become dead
-	// code. Tests assert the option is present on every hop for that reason.
-	//
-	// The loop is intentionally unbounded (`for (;;)`) with the cap enforced inside:
-	// a `redirects <= DEFAULT_MAX_REDIRECTS` header would make the post-loop
-	// statement provably unreachable, which is exactly the dead line firecrawl.ts
-	// carries at :186. Every exit is a `return` or a `throw` written here.
-	for (let redirects = 0; ; redirects++) {
-		const response = await fetch(current, { ...init, headers, redirect: "manual" });
-		if (!REDIRECT_STATUSES.has(response.status)) return response;
-
-		const location = response.headers.get("location");
-		if (!location) return response;
-		if (redirects === DEFAULT_MAX_REDIRECTS) throw new Error(`Too many redirects fetching ${current.toString()}`);
-
-		const next = await validateRemoteUrl(new URL(location, current), ssrfOptions(options));
-		if (next.origin !== current.origin) headers = withoutSensitiveHeaders(headers);
-		current = next;
-	}
-}
-
-function unlockerBody(url: string, zone: string): Record<string, unknown> {
-	return {
-		url,
-		zone,
-		format: "raw",
-		data_format: "markdown",
-	};
-}
-
-async function brightDataRequest(
-	url: string,
-	zone: string,
-	signal: AbortSignal | undefined,
-	options: BrightDataExtractOptions | undefined,
-): Promise<string> {
-	const apiKey = await getApiKey(signal);
-	const headers: Record<string, string> = {
-		"Content-Type": "application/json",
-		Authorization: `Bearer ${apiKey}`,
-	};
-	const activityId = activityMonitor.logStart({ type: "fetch", url });
-	try {
-		const response = await fetchBrightDataApi(BRIGHTDATA_REQUEST_URL, {
-			method: "POST",
-			headers,
-			body: JSON.stringify(unlockerBody(url, zone)),
-			signal: requestSignal(options?.timeoutMs ?? EXTRACT_TIMEOUT_MS, signal),
-		}, options);
-		if (!response.ok) {
-			const errorText = await response.text().catch(() => "");
-			throw new Error(`Bright Data Web Unlocker error ${response.status}: ${redactCredential(errorText, apiKey).slice(0, 300)}`);
-		}
-		const text = await response.text();
-		activityMonitor.logComplete(activityId, response.status);
-		return text;
-	} catch (err) {
-		const message = errorMessage(err);
-		const redactedMessage = redactCredential(message, apiKey);
-		if (isAbortError(err)) activityMonitor.logComplete(activityId, 0);
-		else activityMonitor.logError(activityId, redactedMessage);
-		if (redactedMessage === message) throw err;
-		const redactedError = new Error(redactedMessage);
-		if (err instanceof Error) redactedError.name = err.name;
-		throw redactedError;
-	}
-}
-
-function headingTitle(text: string): string {
-	const match = text.match(/^#{1,2}\s+(.+)/m);
-	if (!match) return "";
-	return match[1].replace(/\*+/g, "").trim();
-}
-
-export function isBrightDataUnlockerAvailable(): boolean {
-	if (getZone() === null) return false;
-	return hasCredentialSource({
-		provider: "Bright Data",
-		configuredValue: loadConfig().brightdataApiKey,
-		environmentValue: process.env.BRIGHTDATA_API_KEY,
-	});
-}
-
-export async function extractWithBrightDataUnlocker(
-	url: string,
-	signal?: AbortSignal,
-	options?: BrightDataExtractOptions,
-): Promise<ExtractedContent | null> {
-	const zone = requireZone();
-	await validateRemoteUrl(url, ssrfOptions(options));
-	const raw = await brightDataRequest(url, zone, signal, options);
-	// Bright Data bills a successful request whatever its length, so short
-	// content is returned rather than discarded: a paywall stub or consent page
-	// is a useful answer, and silently dropping it would hide the spend.
-	const content = raw.trim();
-	if (!content) return null;
-	return { url, title: headingTitle(content), content, error: null };
-}