@oh-my-pi/pi-coding-agent 17.2.1 → 17.2.3

package/src/web/search/providers/duckduckgo.ts

@@ -2,8 +2,8 @@ import type { AuthStorage } from "@oh-my-pi/pi-ai";
 import type { SearchResponse, SearchSource } from "../../../web/search/types";
 import { SearchProviderError } from "../../../web/search/types";
 import type { QuerySyntax } from "../query";
-import { formatScraperQuery } from "../query";
-import { clampNumResults } from "../utils";
+import { formatScraperQuery, parseSearchQuery } from "../query";
+import { clampNumResults, dateToAgeSeconds } from "../utils";
 import type { SearchParams } from "./base";
 import { SearchProvider } from "./base";
 import { browserFetch } from "./browser-page";
@@ -35,6 +35,8 @@ interface ParsedResult {
 	title: string;
 	url: string;
 	snippet?: string;
+	/** ISO timestamp lifted from the result row, when DDG emits one. */
+	publishedDate?: string;
 }
 
 /**
@@ -81,6 +83,23 @@ function unwrapResultUrl(href: string): string | undefined {
 	return undefined;
 }
 
+/**
+ * Lift a result row's publication timestamp from DDG markup. Recent DDG HTML
+ * renders it as a bare `<span>&nbsp; &nbsp; 2026-07-30T20:19:00.0000000</span>`
+ * inside `result__extras__url`. Restrict the scan to that container so a
+ * date-shaped value in a `<span class="result__snippet">` is not attributed
+ * as publication metadata.
+ */
+function extractPublishedDate(block: string): string | undefined {
+	const extrasUrl = /<div\b[^>]*\bclass="[^"]*\bresult__extras__url\b[^"]*"[^>]*>([\s\S]*?)<\/div>/i.exec(block)?.[1];
+	if (!extrasUrl) return undefined;
+	for (const match of extrasUrl.matchAll(/<span\b[^>]*>([\s\S]*?)<\/span>/gi)) {
+		const text = decodeHtmlText(match[1]);
+		if (/^\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2}|$)/.test(text)) return text;
+	}
+	return undefined;
+}
+
 /**
  * Walk the HTML page and pull out result blocks in document order.
  *
@@ -106,10 +125,34 @@ function parseHtmlResults(html: string): ParsedResult[] {
 		if (!titleText) continue;
 		const snippet = snippetRe.exec(block);
 		const snippetText = snippet ? decodeHtmlText(snippet[1]) : undefined;
-		results.push({ title: titleText, url, snippet: snippetText || undefined });
+		results.push({
+			title: titleText,
+			url,
+			snippet: snippetText || undefined,
+			publishedDate: extractPublishedDate(block),
+		});
 	}
 	return results;
 }
+/**
+ * Extract the hidden fields from DDG's next-page form.
+ *
+ * Attribute order varies across responses, so each input tag is parsed
+ * independently instead of matching one fixed HTML layout.
+ */
+function parseContinuationForm(html: string): URLSearchParams | undefined {
+	for (const formMatch of html.matchAll(/<form\b[^>]*>([\s\S]*?)<\/form>/gi)) {
+		const form = new URLSearchParams();
+		for (const inputMatch of formMatch[1].matchAll(/<input\b[^>]*>/gi)) {
+			const input = inputMatch[0];
+			const name = /\bname\s*=\s*(["'])(.*?)\1/i.exec(input)?.[2];
+			const value = /\bvalue\s*=\s*(["'])(.*?)\1/i.exec(input)?.[2];
+			if (name && value !== undefined) form.append(decodeHtmlText(name), decodeHtmlText(value));
+		}
+		if (form.has("s") && form.has("vqd")) return form;
+	}
+	return undefined;
+}
 
 /**
  * `true` when the page DDG returned is the bot-challenge modal instead of
@@ -137,19 +180,131 @@ const DDG_QUERY_SYNTAX: QuerySyntax = {
 	filetype: true,
 };
 
-async function callDuckDuckGoHtml(params: SearchParams): Promise<string> {
+/**
+ * DuckDuckGo's documented `kl` values.
+ *
+ * The codes resemble `region-language` locales but contain provider-specific
+ * identifiers (`jp-jp`, `tw-tzh`, `uk-en`) that cannot be derived mechanically.
+ */
+const DDG_KL_CODES = new Set([
+	"xa-ar",
+	"xa-en",
+	"ar-es",
+	"au-en",
+	"at-de",
+	"be-fr",
+	"be-nl",
+	"br-pt",
+	"bg-bg",
+	"ca-en",
+	"ca-fr",
+	"ct-ca",
+	"cl-es",
+	"cn-zh",
+	"co-es",
+	"hr-hr",
+	"cz-cs",
+	"dk-da",
+	"ee-et",
+	"fi-fi",
+	"fr-fr",
+	"de-de",
+	"gr-el",
+	"hk-tzh",
+	"hu-hu",
+	"in-en",
+	"id-id",
+	"id-en",
+	"ie-en",
+	"il-he",
+	"it-it",
+	"jp-jp",
+	"kr-kr",
+	"lv-lv",
+	"lt-lt",
+	"xl-es",
+	"my-ms",
+	"my-en",
+	"mx-es",
+	"nl-nl",
+	"nz-en",
+	"no-no",
+	"pe-es",
+	"ph-en",
+	"ph-tl",
+	"pl-pl",
+	"pt-pt",
+	"ro-ro",
+	"ru-ru",
+	"sg-en",
+	"sk-sk",
+	"sl-sl",
+	"za-en",
+	"es-es",
+	"se-sv",
+	"ch-de",
+	"ch-fr",
+	"ch-it",
+	"tw-tzh",
+	"th-th",
+	"tr-tr",
+	"ua-uk",
+	"uk-en",
+	"us-en",
+	"ue-es",
+	"ve-es",
+	"vn-vi",
+	"wt-wt",
+]);
+
+/** BCP 47 locales whose DDG code does not follow a simple component swap. */
+const DDG_LOCALE_ALIASES: Record<string, string> = {
+	"ca-es": "ct-ca",
+	"en-gb": "uk-en",
+	"es-419": "xl-es",
+	"es-us": "ue-es",
+	"ja-jp": "jp-jp",
+	"ko-kr": "kr-kr",
+	"zh-hk": "hk-tzh",
+	"zh-tw": "tw-tzh",
+};
+
+/**
+ * Map a parsed `lang:` locale onto DuckDuckGo's documented `kl` values.
+ *
+ * Shared queries use `language-region` order while DDG generally uses
+ * `region-language`. Provider-specific exceptions resolve through
+ * {@link DDG_LOCALE_ALIASES}; all other values must survive the documented
+ * allowlist after swapping or the caller keeps its default region.
+ */
+export function localeToKl(lang: string | undefined): string | undefined {
+	if (!lang) return undefined;
+	const locale = lang.toLowerCase().replaceAll("_", "-");
+	const alias = DDG_LOCALE_ALIASES[locale];
+	if (alias) return alias;
+	const match = /^([a-z]{2})-([a-z]{2})$/.exec(locale);
+	if (!match) return undefined;
+	const candidate = `${match[2]}-${match[1]}`;
+	return DDG_KL_CODES.has(candidate) ? candidate : undefined;
+}
+
+function createDuckDuckGoForm(params: SearchParams): URLSearchParams {
+	const parsed = params.parsedQuery ?? parseSearchQuery(params.query);
 	const form = new URLSearchParams({
-		q: formatScraperQuery(params.query, params.parsedQuery, DDG_QUERY_SYNTAX),
-		kl: "us-en",
+		q: formatScraperQuery(params.query, parsed, DDG_QUERY_SYNTAX),
+		kl: localeToKl(parsed.lang) ?? "us-en",
 	});
 	const df = params.recency ? RECENCY_TO_DDG_DF[params.recency] : undefined;
 	if (df) form.set("df", df);
 	// Add b: "" parameter as specified in the browser fetch template to match real browser form submission
 	form.set("b", "");
+	return form;
+}
 
+async function callDuckDuckGoHtml(params: SearchParams, form: URLSearchParams, signal: AbortSignal): Promise<string> {
 	const page = await browserFetch(DUCKDUCKGO_HTML_URL, {
 		fetch: params.fetch ?? fetch,
-		signal: withHardTimeout(params.signal),
+		signal,
 		referer: "https://html.duckduckgo.com/",
 		init: {
 			method: "POST",
@@ -179,16 +334,29 @@ async function callDuckDuckGoHtml(params: SearchParams): Promise<string> {
 /** Execute a DuckDuckGo web search via the no-JS HTML frontend. */
 export async function searchDuckDuckGo(params: SearchParams): Promise<SearchResponse> {
 	const numResults = clampNumResults(params.numSearchResults ?? params.limit, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
-	const html = await callDuckDuckGoHtml(params);
-	const parsed = parseHtmlResults(html);
-
+	const signal = withHardTimeout(params.signal);
 	const sources: SearchSource[] = [];
 	const seen = new Set<string>();
-	for (const result of parsed) {
-		if (seen.has(result.url)) continue;
-		seen.add(result.url);
-		sources.push({ title: result.title, url: result.url, snippet: result.snippet });
-		if (sources.length >= numResults) break;
+	let form: URLSearchParams | undefined = createDuckDuckGoForm(params);
+
+	while (form && sources.length < numResults) {
+		const html = await callDuckDuckGoHtml(params, form, signal);
+		const sourceCount = sources.length;
+		for (const result of parseHtmlResults(html)) {
+			if (seen.has(result.url)) continue;
+			seen.add(result.url);
+			sources.push({
+				title: result.title,
+				url: result.url,
+				snippet: result.snippet,
+				publishedDate: result.publishedDate,
+				ageSeconds: dateToAgeSeconds(result.publishedDate),
+			});
+			if (sources.length >= numResults) break;
+		}
+
+		if (sources.length === sourceCount) break;
+		form = parseContinuationForm(html);
 	}
 
 	return { provider: "duckduckgo", sources };

package/src/web/search/types.ts

@@ -30,7 +30,12 @@ export const SEARCH_PROVIDER_OPTIONS = [
 		label: "OpenAI",
 		description: "OpenAI's native web_search (uses ChatGPT OAuth via /login openai-codex)",
 	},
-	{ value: "xai", label: "xAI", description: "Grok web search via xAI Responses API (requires XAI_API_KEY)" },
+	{
+		value: "xai",
+		label: "xAI",
+		description:
+			"Grok web search via xAI Responses API (uses SuperGrok/X Premium+ OAuth via /login xai-oauth, or XAI_API_KEY)",
+	},
 	{ value: "zai", label: "Z.AI", description: "Calls Z.AI webSearchPrime MCP" },
 	{ value: "exa", label: "Exa", description: "API via /login exa or EXA_API_KEY; explicit keyless fallback via MCP" },
 	{ value: "tinyfish", label: "TinyFish", description: "Requires TINYFISH_API_KEY" },