@gajae-code/coding-agent 0.18.11 → 0.18.12

package/src/web/search/providers/insane.ts

@@ -15,6 +15,8 @@ import type { AuthStorage } from "@gajae-code/ai/core";
 
 import type { SearchResponse, SearchSource } from "../../../web/search/types";
 import { SearchProviderError } from "../../../web/search/types";
+import type { AddressResolver } from "../../insane/url-guard";
+import { guardedPublicFetch, validatePublicHttpUrl } from "../../insane/url-guard";
 import { clampNumResults } from "../utils";
 import type { SearchParams } from "./base";
 import { SearchProvider } from "./base";
@@ -24,6 +26,8 @@ import { withHardTimeout } from "./utils";
 const DEFAULT_NUM_RESULTS = 10;
 const MAX_NUM_RESULTS = 20;
 const PUBLIC_ROUTE_TIMEOUT_MS = 15_000;
+const MAX_PUBLIC_ROUTE_REDIRECTS = 5;
+const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
 const DISCOVERY_LIMIT = 8;
 const MAX_PUBLIC_ROUTE_RESPONSE_BYTES = 1024 * 1024;
 
@@ -151,6 +155,10 @@ class ResponseTooLargeError extends Error {
 	override readonly name = "response_too_large";
 }
 
+class PublicUrlRejectedError extends Error {
+	override readonly name = "public_url_rejected";
+}
+
 async function cancelBody(body: ReadableStream<Uint8Array> | null): Promise<void> {
 	if (!body) return;
 	try {
@@ -203,17 +211,51 @@ async function readResponseText(response: Response): Promise<string> {
 async function fetchText(
 	url: string,
 	signal?: AbortSignal,
+	resolverOption?: { resolver?: AddressResolver },
 ): Promise<{ status: number; text: string; contentType: string }> {
-	const response = await fetch(url, {
-		headers: {
-			Accept: "application/json, application/atom+xml, application/rss+xml, text/xml, text/html;q=0.8, */*;q=0.5",
-			"User-Agent": USER_AGENT,
-		},
-		redirect: "follow",
-		signal: withHardTimeout(signal, PUBLIC_ROUTE_TIMEOUT_MS),
-	});
-	const text = await readResponseText(response);
-	return { status: response.status, text, contentType: response.headers.get("content-type") ?? "" };
+	const deadline = withHardTimeout(signal, PUBLIC_ROUTE_TIMEOUT_MS);
+	let currentUrl = url;
+
+	for (let redirectCount = 0; ; redirectCount++) {
+		// Fresh GET each hop: no body and no caller auth headers. guardedPublicFetch forces redirect:"manual".
+		const dial = await guardedPublicFetch(
+			currentUrl,
+			{
+				method: "GET",
+				headers: {
+					Accept:
+						"application/json, application/atom+xml, application/rss+xml, text/xml, text/html;q=0.8, */*;q=0.5",
+					"User-Agent": USER_AGENT,
+				},
+				redirect: "manual",
+				signal: deadline,
+			},
+			resolverOption,
+		);
+		if (!dial.ok) throw new PublicUrlRejectedError(dial.reason);
+
+		const { response, logicalUrl } = dial;
+		if (!REDIRECT_STATUSES.has(response.status)) {
+			const text = await readResponseText(response);
+			return { status: response.status, text, contentType: response.headers.get("content-type") ?? "" };
+		}
+
+		const location = response.headers.get("location")?.trim();
+		await cancelBody(response.body);
+		if (!location) throw new PublicUrlRejectedError("redirect missing Location");
+		if (redirectCount >= MAX_PUBLIC_ROUTE_REDIRECTS) throw new PublicUrlRejectedError("redirect limit exceeded");
+
+		let nextUrl: URL;
+		try {
+			nextUrl = new URL(location, logicalUrl);
+		} catch {
+			throw new PublicUrlRejectedError("invalid redirect Location");
+		}
+		const guard = await validatePublicHttpUrl(nextUrl.href, { signal: deadline, resolver: resolverOption?.resolver });
+		if (deadline.aborted) throw deadline.reason;
+		if (!guard.ok) throw new PublicUrlRejectedError(guard.reason);
+		currentUrl = guard.url.href;
+	}
 }
 
 function attr(input: string, name: string): string | undefined {
@@ -267,11 +309,15 @@ function redditFeedUrls(url: URL): string[] {
 	return [`${base}/.rss`, `${base}.rss`];
 }
 
-async function routeReddit(url: URL, signal?: AbortSignal): Promise<RouteResult> {
+async function routeReddit(
+	url: URL,
+	signal?: AbortSignal,
+	resolverOption?: { resolver?: AddressResolver },
+): Promise<RouteResult> {
 	const attempts: InsaneRouteAttempt[] = [];
 	for (const feedUrl of redditFeedUrls(url)) {
 		try {
-			const response = await fetchText(feedUrl, signal);
+			const response = await fetchText(feedUrl, signal, resolverOption);
 			const ok = response.status === 200 && /<(feed|rss)\b/i.test(response.text) && !isBlockedBody(response.text);
 			attempts.push(attempt("reddit", "rss", ok, response.status, response.text, ok ? "feed" : "no-feed-markers"));
 			if (ok)
@@ -325,13 +371,17 @@ function sourceFromOEmbed(raw: string, url: string): SearchSource | null {
 	return { title, url: data.url ?? url, snippet, author: data.author_name };
 }
 
-async function routeX(url: URL, signal?: AbortSignal): Promise<RouteResult> {
+async function routeX(
+	url: URL,
+	signal?: AbortSignal,
+	resolverOption?: { resolver?: AddressResolver },
+): Promise<RouteResult> {
 	const attempts: InsaneRouteAttempt[] = [];
 	const id = tweetId(url);
 	if (id) {
 		const tweetResultUrl = `https://cdn.syndication.twimg.com/tweet-result?id=${encodeURIComponent(id)}&token=a`;
 		try {
-			const response = await fetchText(tweetResultUrl, signal);
+			const response = await fetchText(tweetResultUrl, signal, resolverOption);
 			const source =
 				response.status === 200 && !isBlockedBody(response.text)
 					? sourceFromTweetJson(response.text, url.toString())
@@ -346,7 +396,7 @@ async function routeX(url: URL, signal?: AbortSignal): Promise<RouteResult> {
 		}
 		const oembedUrl = `https://publish.twitter.com/oembed?url=${encodeURIComponent(`https://twitter.com/i/status/${id}`)}&omit_script=1`;
 		try {
-			const response = await fetchText(oembedUrl, signal);
+			const response = await fetchText(oembedUrl, signal, resolverOption);
 			const source =
 				response.status === 200 && !isBlockedBody(response.text)
 					? sourceFromOEmbed(response.text, oembedUrl)
@@ -363,7 +413,7 @@ async function routeX(url: URL, signal?: AbortSignal): Promise<RouteResult> {
 	if (handle) {
 		const timelineUrl = `https://syndication.twitter.com/srv/timeline-profile/screen-name/${encodeURIComponent(handle)}`;
 		try {
-			const response = await fetchText(timelineUrl, signal);
+			const response = await fetchText(timelineUrl, signal, resolverOption);
 			const ok = response.status === 200 && response.text.includes("__NEXT_DATA__") && !isBlockedBody(response.text);
 			attempts.push(
 				attempt("x", "syndication-timeline", ok, response.status, response.text, ok ? "timeline" : "no-next-data"),
@@ -396,14 +446,18 @@ function youtubeVideoId(url: URL): string | null {
 	return url.searchParams.get("v") ?? url.pathname.match(/\/shorts\/([^/?#]+)/)?.[1] ?? null;
 }
 
-async function routeYouTube(url: URL, signal?: AbortSignal): Promise<RouteResult> {
+async function routeYouTube(
+	url: URL,
+	signal?: AbortSignal,
+	resolverOption?: { resolver?: AddressResolver },
+): Promise<RouteResult> {
 	const attempts: InsaneRouteAttempt[] = [];
 	const videoId = youtubeVideoId(url);
 	if (videoId) {
 		const watchUrl = `https://www.youtube.com/watch?v=${encodeURIComponent(videoId)}`;
 		const oembedUrl = `https://www.youtube.com/oembed?url=${encodeURIComponent(watchUrl)}&format=json`;
 		try {
-			const response = await fetchText(oembedUrl, signal);
+			const response = await fetchText(oembedUrl, signal, resolverOption);
 			const source =
 				response.status === 200 && !isBlockedBody(response.text) ? sourceFromOEmbed(response.text, watchUrl) : null;
 			attempts.push(
@@ -418,7 +472,7 @@ async function routeYouTube(url: URL, signal?: AbortSignal): Promise<RouteResult
 	if (channelId) {
 		const feedUrl = `https://www.youtube.com/feeds/videos.xml?channel_id=${encodeURIComponent(channelId)}`;
 		try {
-			const response = await fetchText(feedUrl, signal);
+			const response = await fetchText(feedUrl, signal, resolverOption);
 			const ok = response.status === 200 && /<feed\b/i.test(response.text) && !isBlockedBody(response.text);
 			attempts.push(attempt("youtube", "feed", ok, response.status, response.text, ok ? "feed" : "no-feed"));
 			if (ok)
@@ -466,13 +520,17 @@ function hnSourceFromItem(raw: string): SearchSource | null {
 	};
 }
 
-async function routeHackerNews(url: URL, signal?: AbortSignal): Promise<RouteResult> {
+async function routeHackerNews(
+	url: URL,
+	signal?: AbortSignal,
+	resolverOption?: { resolver?: AddressResolver },
+): Promise<RouteResult> {
 	const attempts: InsaneRouteAttempt[] = [];
 	const id = hnItemId(url);
 	if (!id) return { platform: "hackernews", attempts };
 	const itemUrl = `https://hacker-news.firebaseio.com/v0/item/${encodeURIComponent(id)}.json`;
 	try {
-		const response = await fetchText(itemUrl, signal);
+		const response = await fetchText(itemUrl, signal, resolverOption);
 		const source = response.status === 200 && !isBlockedBody(response.text) ? hnSourceFromItem(response.text) : null;
 		attempts.push(
 			attempt("hackernews", "firebase-item", !!source, response.status, response.text, source ? "item" : "no-item"),
@@ -487,14 +545,18 @@ async function routeHackerNews(url: URL, signal?: AbortSignal): Promise<RouteRes
 	return { platform: "hackernews", attempts };
 }
 
-export async function routeInsanePublicUrl(rawUrl: string, signal?: AbortSignal): Promise<RouteResult> {
+export async function routeInsanePublicUrl(
+	rawUrl: string,
+	signal?: AbortSignal,
+	resolverOption?: { resolver?: AddressResolver },
+): Promise<RouteResult> {
 	const url = parseHttpUrl(rawUrl);
 	if (!url) return null;
 	const platform = detectPlatform(url);
-	if (platform === "reddit") return routeReddit(url, signal);
-	if (platform === "x") return routeX(url, signal);
-	if (platform === "youtube") return routeYouTube(url, signal);
-	if (platform === "hackernews") return routeHackerNews(url, signal);
+	if (platform === "reddit") return routeReddit(url, signal, resolverOption);
+	if (platform === "x") return routeX(url, signal, resolverOption);
+	if (platform === "youtube") return routeYouTube(url, signal, resolverOption);
+	if (platform === "hackernews") return routeHackerNews(url, signal, resolverOption);
 	return null;
 }
 
@@ -528,9 +590,10 @@ export async function searchInsane(params: {
 	num_results?: number;
 	recency?: "day" | "week" | "month" | "year";
 	signal?: AbortSignal;
+	resolverOption?: { resolver?: AddressResolver };
 }): Promise<SearchResponse> {
 	const numResults = clampNumResults(params.num_results, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
-	const direct = await routeInsanePublicUrl(params.query, params.signal);
+	const direct = await routeInsanePublicUrl(params.query, params.signal, params.resolverOption);
 	if (isSuccess(direct) && direct.sources.length > 0) {
 		return {
 			provider: "insane",
@@ -560,7 +623,7 @@ export async function searchInsane(params: {
 	const seenUrls = new Set<string>();
 	for (const candidate of discovery.sources) {
 		if (routedSources.length >= numResults) break;
-		const routed = await routeInsanePublicUrl(candidate.url, params.signal);
+		const routed = await routeInsanePublicUrl(candidate.url, params.signal, params.resolverOption);
 		if (routed && "attempts" in routed) attempts.push(...routed.attempts);
 		if (!isSuccess(routed)) continue;
 		for (const source of routed.sources) {

package/src/web/search/providers/searxng.ts

@@ -22,6 +22,11 @@
  *   SEARXNG_BASIC_USERNAME - Optional RFC 7617 Basic auth username
  *   SEARXNG_BASIC_PASSWORD - Optional RFC 7617 Basic auth password
  *
+ * A project settings endpoint must not receive a credential that exists only in
+ * the operator environment. The project file can still authenticate with a
+ * credential it sets itself, and an operator endpoint still receives an
+ * operator environment credential.
+ *
  * Reference: https://docs.searxng.org/dev/search_api.html
  */
 
@@ -80,7 +85,6 @@ interface SearXNGAuth {
 	value: string;
 }
 
-/** Find SearXNG endpoint from settings or environment. */
 /**
  * SearXNG environment value, from trusted sources only, preserving an
  * intentionally empty setting.
@@ -122,6 +126,7 @@ export function resolveSearxngConfigForTest(): {
 	};
 }
 
+/** Find SearXNG endpoint from settings or environment. */
 function findEndpoint(activeSettings?: Settings): string | null {
 	try {
 		const endpoint = (activeSettings ?? settings).get("searxng.endpoint");
@@ -132,36 +137,66 @@ function findEndpoint(activeSettings?: Settings): string | null {
 	return trustedSearxngEnv("SEARXNG_ENDPOINT");
 }
 
+/**
+ * True when `endpoint` is the value supplied by the project settings layer.
+ *
+ * `get` is global, then project, then runtime override. `getOverride` and
+ * `getGlobal` are the layers a project file cannot write, so a value that is
+ * present, not overridden, and different from the global layer came from the
+ * project file. An equal global value is the operator endpoint even if the
+ * project file repeats it.
+ */
+function endpointComesFromProjectLayer(activeSettings: Settings | undefined, endpoint: string | null): boolean {
+	if (!endpoint) return false;
+	try {
+		const active = activeSettings ?? settings;
+		if (active.getOverride("searxng.endpoint") !== undefined) return false;
+		if (!active.has("searxng.endpoint")) return false;
+		const effective = active.get("searxng.endpoint");
+		if (effective !== endpoint) return false;
+		const globalValue = active.getGlobal("searxng.endpoint");
+		return globalValue === undefined || globalValue !== effective;
+	} catch {
+		return false;
+	}
+}
+
 /** Find SearXNG bearer token from settings or environment. */
-function findToken(activeSettings?: Settings): string | null {
+function findToken(activeSettings?: Settings, allowEnvironment = true): string | null {
 	try {
 		const token = (activeSettings ?? settings).get("searxng.token");
 		if (token) return token;
 	} catch {
 		// Settings not initialized yet
+		return allowEnvironment ? trustedSearxngEnv("SEARXNG_TOKEN") : null;
 	}
+	if (!allowEnvironment) return null;
 	return trustedSearxngEnv("SEARXNG_TOKEN");
 }
 
 /** Find SearXNG Basic auth username from settings or environment. */
-function findBasicUsername(activeSettings?: Settings): string | null {
+function findBasicUsername(activeSettings?: Settings, allowEnvironment = true): string | null {
 	try {
 		const username = (activeSettings ?? settings).get("searxng.basicUsername");
 		if (username !== undefined) return username;
 	} catch {
 		// Settings not initialized yet
+		return allowEnvironment ? trustedSearxngEnv("SEARXNG_BASIC_USERNAME") : null;
 	}
+	if (!allowEnvironment) return null;
 	return trustedSearxngEnv("SEARXNG_BASIC_USERNAME");
 }
 
 /** Find SearXNG Basic auth password from settings or environment. */
-function findBasicPassword(activeSettings?: Settings): string | null {
+function findBasicPassword(activeSettings?: Settings, allowEnvironment = true): string | null {
 	try {
 		const password = (activeSettings ?? settings).get("searxng.basicPassword");
 		if (password !== undefined) return password;
 	} catch {
 		// Settings not initialized yet
+		return allowEnvironment ? trustedSearxngEnv("SEARXNG_BASIC_PASSWORD") : null;
 	}
+	if (!allowEnvironment) return null;
 	return trustedSearxngEnv("SEARXNG_BASIC_PASSWORD");
 }
 
@@ -175,11 +210,26 @@ function hasControlCharacters(value: string): boolean {
 	return /[\u0000-\u001F\u007F-\u009F]/u.test(value);
 }
 
-/** Find SearXNG authentication from settings or environment. Basic auth takes precedence over bearer tokens. */
-function findAuth(activeSettings?: Settings): SearXNGAuth | null {
-	const basicUsername = findBasicUsername(activeSettings);
-	const basicPassword = findBasicPassword(activeSettings);
-	return resolveBasicOrBearerAuth(basicUsername, basicPassword, findToken(activeSettings));
+/**
+ * Find SearXNG authentication from settings or environment. Basic auth takes
+ * precedence over bearer tokens.
+ *
+ * When `endpoint` came from the project settings layer, environment credentials
+ * are omitted. A complete basic-auth pair or token set in settings is still used,
+ * including one the project file set itself. An incomplete basic-auth pair is not
+ * filled from the environment: the search continues with a project token, or with
+ * no Authorization header, instead of failing closed on the missing half.
+ */
+function findAuth(activeSettings?: Settings, endpoint?: string | null): SearXNGAuth | null {
+	const resolvedEndpoint = endpoint === undefined ? findEndpoint(activeSettings) : endpoint;
+	const allowEnvironment = !endpointComesFromProjectLayer(activeSettings, resolvedEndpoint);
+	const username = findBasicUsername(activeSettings, allowEnvironment);
+	const password = findBasicPassword(activeSettings, allowEnvironment);
+	const token = findToken(activeSettings, allowEnvironment);
+	if (!allowEnvironment && (username === null) !== (password === null)) {
+		return token ? { type: "bearer", value: token } : null;
+	}
+	return resolveBasicOrBearerAuth(username, password, token);
 }
 
 function resolveBasicOrBearerAuth(
@@ -322,7 +372,9 @@ export async function searchSearXNG(params: {
 		);
 	}
 
-	const auth = params.authStorage ? await findAuthFromStorage(params.authStorage) : findAuth(params.settings);
+	const auth = params.authStorage
+		? await findAuthFromStorage(params.authStorage)
+		: findAuth(params.settings, endpoint);
 
 	let categories: string | undefined;
 	let language: string | undefined;