@oh-my-pi/pi-coding-agent 17.1.1 → 17.1.3

package/src/web/search/providers/searxng.ts

@@ -14,6 +14,9 @@
  *   searxng.basicUsername - Optional RFC 7617 Basic auth username
  *   searxng.basicPassword - Optional RFC 7617 Basic auth password
  *   searxng.categories    - Optional comma-separated categories filter
+ *   searxng.engines       - Optional comma-separated engine names or shortcuts
+ *                           (e.g. "duckduckgo, br, sp"); shortcuts resolve via
+ *                           the instance's /config endpoint
  *   searxng.language      - Optional language code (e.g. en, zh-CN)
  *
  * Environment variable fallbacks:
@@ -22,6 +25,11 @@
  *   SEARXNG_BASIC_USERNAME - Optional RFC 7617 Basic auth username
  *   SEARXNG_BASIC_PASSWORD - Optional RFC 7617 Basic auth password
  *
+ * Bang syntax in queries is passed through: `!ddg foo` selects an engine or
+ * category server-side and the bang token is stripped from the upstream query.
+ * External bangs (`!!g`) are removed client-side because SearXNG answers them
+ * with an HTTP redirect even for JSON requests.
+ *
  * Reference: https://docs.searxng.org/dev/search_api.html
  */
 
@@ -30,6 +38,8 @@ import type { AuthStorage, FetchImpl } from "@oh-my-pi/pi-ai";
 import { settings } from "../../../config/settings";
 import type { SearchResponse, SearchSource } from "../../../web/search/types";
 import { SearchProviderError } from "../../../web/search/types";
+import type { StructuredQuery } from "../query";
+import { formatScraperQuery, parseSearchQuery } from "../query";
 import { clampNumResults, dateToAgeSeconds } from "../utils";
 import type { SearchParams } from "./base";
 import { SearchProvider } from "./base";
@@ -73,6 +83,11 @@ interface SearXNGAuth {
 	value: string;
 }
 
+/** Subset of the SearXNG /config payload used for engine shortcut resolution. */
+interface SearXNGConfig {
+	engines?: Array<{ name?: string; shortcut?: string }>;
+}
+
 /** Find SearXNG endpoint from settings or environment. */
 function findEndpoint(): string | null {
 	try {
@@ -150,6 +165,110 @@ function findAuth(): SearXNGAuth | null {
 	return token ? { type: "bearer", value: token } : null;
 }
 
+/** Find configured engine names/shortcuts from settings. */
+function findEngines(): string | null {
+	try {
+		const engines = settings.get("searxng.engines");
+		if (engines) return engines;
+	} catch {
+		// Settings not initialized yet
+	}
+	return null;
+}
+
+/** Build request headers including authentication. */
+function buildHeaders(auth: SearXNGAuth | null): Record<string, string> {
+	const headers: Record<string, string> = { Accept: "application/json" };
+	if (auth?.type === "basic") {
+		headers.Authorization = `Basic ${auth.value}`;
+	} else if (auth?.type === "bearer") {
+		headers.Authorization = `Bearer ${auth.value}`;
+	}
+	return headers;
+}
+
+/** Per-endpoint cache of shortcut/name → canonical engine name maps. */
+const engineNameMapCache = new Map<string, Promise<Map<string, string> | null>>();
+
+/** Fetch the instance's /config and build a lookup of lowercased engine names
+ *  and shortcuts to canonical engine names. Returns null on any failure. */
+async function fetchEngineNameMap(
+	base: string,
+	auth: SearXNGAuth | null,
+	fetchImpl: FetchImpl | undefined,
+	signal: AbortSignal | undefined,
+): Promise<Map<string, string> | null> {
+	try {
+		const response = await (fetchImpl ?? fetch)(`${base}/config`, {
+			headers: buildHeaders(auth),
+			signal: withHardTimeout(signal),
+		});
+		if (!response.ok) return null;
+		const config = (await response.json()) as SearXNGConfig;
+		const map = new Map<string, string>();
+		for (const engine of config.engines ?? []) {
+			if (!engine.name) continue;
+			map.set(engine.name.toLowerCase(), engine.name);
+			if (engine.shortcut) map.set(engine.shortcut.toLowerCase(), engine.name);
+		}
+		return map.size ? map : null;
+	} catch {
+		return null;
+	}
+}
+
+/** Get the engine name map for an endpoint, cached for the process lifetime.
+ *  Failures are not cached so a transient error retries on the next search. */
+function getEngineNameMap(
+	endpoint: string,
+	auth: SearXNGAuth | null,
+	fetchImpl: FetchImpl | undefined,
+	signal: AbortSignal | undefined,
+): Promise<Map<string, string> | null> {
+	const base = endpoint.replace(/\/+$/, "");
+	let cached = engineNameMapCache.get(base);
+	if (!cached) {
+		cached = fetchEngineNameMap(base, auth, fetchImpl, signal).then(map => {
+			if (!map) engineNameMapCache.delete(base);
+			return map;
+		});
+		engineNameMapCache.set(base, cached);
+	}
+	return cached;
+}
+
+/** Resolve configured engine entries (canonical names or shortcuts like `ddg`)
+ *  to canonical names for SearXNG's `engines=` parameter, which accepts names
+ *  only — shortcuts resolve exclusively through bang syntax. Unknown entries
+ *  pass through verbatim; the server drops them and falls back to categories. */
+async function resolveEngineNames(
+	raw: string,
+	endpoint: string,
+	auth: SearXNGAuth | null,
+	fetchImpl: FetchImpl | undefined,
+	signal: AbortSignal | undefined,
+): Promise<string | undefined> {
+	const entries = raw
+		.split(",")
+		.map(entry => entry.trim())
+		.filter(Boolean);
+	if (!entries.length) return undefined;
+	const map = await getEngineNameMap(endpoint, auth, fetchImpl, signal);
+	if (!map) return entries.join(",");
+	return entries.map(entry => map.get(entry.toLowerCase()) ?? entry).join(",");
+}
+
+/** Strip external bang tokens (`!!g`, bare `!!`): SearXNG answers them with an
+ *  HTTP redirect even for JSON requests, which breaks response parsing.
+ *  Single-bang engine/category selectors (`!ddg`, `!images`) are kept — the
+ *  instance resolves and removes them server-side. */
+function stripExternalBangs(query: string): string {
+	return query
+		.split(/\s+/)
+		.filter(part => !part.startsWith("!!"))
+		.join(" ");
+}
+
 /** Build the search URL and headers for a SearXNG request */
 function buildRequest(
 	endpoint: string,
@@ -158,6 +277,7 @@ function buildRequest(
 		num_results?: number;
 		recency?: "day" | "week" | "month" | "year";
 		categories?: string;
+		engines?: string;
 		language?: string;
 		signal?: AbortSignal;
 	},
@@ -181,19 +301,15 @@ function buildRequest(
 		url.searchParams.set("categories", params.categories);
 	}
 
+	if (params.engines) {
+		url.searchParams.set("engines", params.engines);
+	}
+
 	if (params.language) {
 		url.searchParams.set("language", params.language);
 	}
 
-	const headers: Record<string, string> = {
-		Accept: "application/json",
-	};
-
-	if (auth?.type === "basic") {
-		headers.Authorization = `Basic ${auth.value}`;
-	} else if (auth?.type === "bearer") {
-		headers.Authorization = `Bearer ${auth.value}`;
-	}
+	const headers = buildHeaders(auth);
 
 	return { url, headers };
 }
@@ -205,6 +321,7 @@ async function callSearXNGSearch(
 		num_results?: number;
 		recency?: "day" | "week" | "month" | "year";
 		categories?: string;
+		engines?: string;
 		language?: string;
 		signal?: AbortSignal;
 		fetch?: FetchImpl;
@@ -231,6 +348,7 @@ async function callSearXNGSearch(
 /** Execute SearXNG web search. */
 export async function searchSearXNG(params: {
 	query: string;
+	parsedQuery?: StructuredQuery;
 	num_results?: number;
 	recency?: "day" | "week" | "month" | "year";
 	signal?: AbortSignal;
@@ -255,12 +373,29 @@ export async function searchSearXNG(params: {
 	} catch {
 		// Settings not initialized yet
 	}
+	const configuredEngines = findEngines();
+
+	// SearXNG forwards `q` to downstream engines, so build it with the shared
+	// scraper formatter: operators are canonicalized and scraper-hostile ones
+	// (path-carrying `site:`, `inurl:`) are structurally demoted to plain
+	// terms before formatting, so paren-grouped `site:` filters are covered
+	// too. `lang:` maps onto the native `language` param (overriding the
+	// configured default).
+	const parsed = params.parsedQuery ?? parseSearchQuery(params.query);
+	const query = formatScraperQuery(params.query, parsed);
+	if (parsed.lang) language = parsed.lang;
+
+	const engines = configuredEngines
+		? await resolveEngineNames(configuredEngines, endpoint, auth, params.fetch, params.signal)
+		: undefined;
 
 	const response = await callSearXNGSearch(
 		endpoint,
 		{
 			...params,
+			query: stripExternalBangs(query),
 			categories,
+			engines,
 			language,
 			fetch: params.fetch,
 		},
@@ -315,6 +450,7 @@ export class SearXNGProvider extends SearchProvider {
 
 	search(params: SearchParams): Promise<SearchResponse> {
 		return searchSearXNG({
+			parsedQuery: params.parsedQuery,
 			query: params.query,
 			num_results: params.numSearchResults ?? params.limit,
 			recency: params.recency,

package/src/web/search/providers/startpage.ts

@@ -2,6 +2,7 @@ import type { AuthStorage, FetchImpl } from "@oh-my-pi/pi-ai";
 import { parseHTML } from "linkedom";
 import type { SearchResponse, SearchSource } from "../../../web/search/types";
 import { SearchProviderError } from "../../../web/search/types";
+import { formatScraperQuery } from "../query";
 import { clampNumResults } from "../utils";
 import type { SearchParams } from "./base";
 import { SearchProvider } from "./base";
@@ -136,12 +137,17 @@ async function callStartpageHtml(params: SearchParams): Promise<string> {
 	const fetchImpl = params.fetch ?? fetch;
 	const signal = withHardTimeout(params.signal);
 	const withDate = params.recency ? RECENCY_TO_STARTPAGE_WITH_DATE[params.recency] : undefined;
+	// Startpage proxies Google, so the operator set works inline; rebuild via
+	// the shared scraper formatter to canonicalize aliases (domain: → site:,
+	// since: → after:, …) and demote scraper-hostile operators. Directive-free
+	// queries pass through byte-identical.
+	const query = formatScraperQuery(params.query, params.parsedQuery);
 
 	const formInputs = await fetchFormInputs(fetchImpl, signal);
 	let page: LoadedHtmlPage;
 	if (formInputs) {
 		const form = new URLSearchParams(formInputs);
-		form.set("query", params.query);
+		form.set("query", query);
 		if (withDate) form.set("with_date", withDate);
 		page = await browserFetch(STARTPAGE_SEARCH_URL, {
 			fetch: fetchImpl,
@@ -152,7 +158,7 @@ async function callStartpageHtml(params: SearchParams): Promise<string> {
 		});
 	} else {
 		const url = new URL(STARTPAGE_SEARCH_URL);
-		url.searchParams.set("query", params.query);
+		url.searchParams.set("query", query);
 		if (withDate) url.searchParams.set("with_date", withDate);
 		page = await browserFetch(url.href, {
 			fetch: fetchImpl,

package/src/web/search/providers/synthetic.ts

@@ -8,6 +8,7 @@
 import { type ApiKey, type AuthStorage, type FetchImpl, getEnvApiKey, withAuth } from "@oh-my-pi/pi-ai";
 import type { SearchResponse, SearchSource } from "../../../web/search/types";
 import { SearchProviderError } from "../../../web/search/types";
+import { formatQuery, parseSearchQuery } from "../query";
 import type { SearchParams } from "./base";
 import { SearchProvider } from "./base";
 import { classifyProviderHttpError, withHardTimeout } from "./utils";
@@ -73,8 +74,13 @@ export async function searchSynthetic(params: SearchParamsWithFetch): Promise<Se
 		sessionId: params.sessionId,
 	});
 
+	const parsed = params.parsedQuery ?? parseSearchQuery(params.query);
+	const query = parsed.hasDirectives
+		? formatQuery(parsed, { phrases: true, negation: true, site: true })
+		: params.query;
+
 	const fetchImpl = params.fetch;
-	const data = await withAuth(keyOrResolver, key => callSyntheticSearch(key, params.query, params.signal, fetchImpl), {
+	const data = await withAuth(keyOrResolver, key => callSyntheticSearch(key, query, params.signal, fetchImpl), {
 		signal: params.signal,
 		missingKeyMessage: "Synthetic credentials not found. Set SYNTHETIC_API_KEY or login with 'omp /login synthetic'.",
 	});

package/src/web/search/providers/tavily.ts

@@ -7,6 +7,7 @@
 import { type ApiKey, type AuthStorage, type FetchImpl, getEnvApiKey, withAuth } from "@oh-my-pi/pi-ai";
 import type { SearchResponse, SearchSource } from "../../../web/search/types";
 import { SearchProviderError } from "../../../web/search/types";
+import { formatQuery, parseSearchQuery } from "../query";
 import { clampNumResults, dateToAgeSeconds } from "../utils";
 import type { SearchParams } from "./base";
 import { SearchProvider } from "./base";
@@ -20,6 +21,14 @@ export interface TavilySearchParams {
 	query: string;
 	num_results?: number;
 	recency?: "day" | "week" | "month" | "year";
+	/** `site:` hosts mapped to Tavily's `include_domains`. */
+	include_domains?: string[];
+	/** `-site:` hosts mapped to Tavily's `exclude_domains`. */
+	exclude_domains?: string[];
+	/** `after:` inclusive lower bound, ISO `YYYY-MM-DD`, mapped to `start_date`. */
+	start_date?: string;
+	/** `before:` upper bound, ISO `YYYY-MM-DD`, mapped to `end_date`. */
+	end_date?: string;
 	signal?: AbortSignal;
 	fetch?: FetchImpl;
 }
@@ -83,7 +92,21 @@ export function buildRequestBody(params: TavilySearchParams): Record<string, unk
 		include_answer: "advanced",
 		include_raw_content: false,
 	};
-	if (params.recency) {
+	if (params.include_domains?.length) {
+		body.include_domains = params.include_domains;
+	}
+	if (params.exclude_domains?.length) {
+		body.exclude_domains = params.exclude_domains;
+	}
+	if (params.start_date) {
+		body.start_date = params.start_date;
+	}
+	if (params.end_date) {
+		body.end_date = params.end_date;
+	}
+	// Explicit before:/after: bounds take precedence over the relative recency
+	// window; sending both would over-restrict.
+	if (params.recency && !params.start_date && !params.end_date) {
 		body.time_range = params.recency;
 	}
 	return body;
@@ -148,8 +171,19 @@ function hasRenderableResponse(response: SearchResponse): boolean {
 	return response.sources.length > 0;
 }
 
+/** Bare hosts from `site:` values (path parts are enforced by the central lenient filter). */
+function siteHosts(sites: readonly string[]): string[] {
+	const hosts = new Set<string>();
+	for (const site of sites) {
+		const host = site.split("/", 1)[0];
+		if (host) hosts.add(host);
+	}
+	return [...hosts];
+}
+
 /** Execute Tavily web search. */
 export async function searchTavily(params: SearchParams): Promise<SearchResponse> {
+	const parsed = params.parsedQuery ?? parseSearchQuery(params.query);
 	const tavilyParams: TavilySearchParams = {
 		query: params.query,
 		num_results: params.numSearchResults ?? params.limit,
@@ -157,6 +191,16 @@ export async function searchTavily(params: SearchParams): Promise<SearchResponse
 		signal: params.signal,
 		fetch: params.fetch,
 	};
+	if (parsed.hasDirectives) {
+		// Tavily prefers clean natural text; re-emit only phrases and -exclusions.
+		tavilyParams.query = formatQuery(parsed, { phrases: true, negation: true });
+		const include = siteHosts(parsed.sites);
+		const exclude = siteHosts(parsed.excludedSites);
+		if (include.length > 0) tavilyParams.include_domains = include;
+		if (exclude.length > 0) tavilyParams.exclude_domains = exclude;
+		if (parsed.after) tavilyParams.start_date = parsed.after;
+		if (parsed.before) tavilyParams.end_date = parsed.before;
+	}
 	const keyOrResolver: ApiKey = params.authStorage.resolver("tavily", {
 		sessionId: params.sessionId,
 	});
@@ -171,11 +215,16 @@ export async function searchTavily(params: SearchParams): Promise<SearchResponse
 		withAuth(keyOrResolver, key => callTavilySearch(key, searchParams), authOptions);
 
 	const response = toSearchResponse(await callWithAuth(tavilyParams), numResults);
-	if (!tavilyParams.recency || hasRenderableResponse(response)) {
+	const hasTimeFilter = Boolean(tavilyParams.recency || tavilyParams.start_date || tavilyParams.end_date);
+	if (!hasTimeFilter || hasRenderableResponse(response)) {
 		return response;
 	}
 
-	return toSearchResponse(await callWithAuth({ ...tavilyParams, recency: undefined }), numResults);
+	// Time filters commonly zero out results; retry once without them.
+	return toSearchResponse(
+		await callWithAuth({ ...tavilyParams, recency: undefined, start_date: undefined, end_date: undefined }),
+		numResults,
+	);
 }
 
 /** Search provider for Tavily web search. */

package/src/web/search/providers/tinyfish.ts

@@ -7,6 +7,7 @@
 import { type ApiKey, type AuthStorage, type FetchImpl, getEnvApiKey, withAuth } from "@oh-my-pi/pi-ai";
 import type { SearchResponse, SearchSource } from "../../../web/search/types";
 import { SearchProviderError } from "../../../web/search/types";
+import { formatQuery, parseSearchQuery, type QuerySyntax } from "../query";
 import { clampNumResults } from "../utils";
 import type { SearchParams } from "./base";
 import { SearchProvider } from "./base";
@@ -17,6 +18,9 @@ const DEFAULT_NUM_RESULTS = 10;
 const MAX_NUM_RESULTS = 20;
 const MAX_PAGE = 10;
 
+/** TinyFish is SERP-backed: common Google-style operators pass through. */
+const TINYFISH_QUERY_SYNTAX: QuerySyntax = { phrases: true, negation: true, site: true, filetype: true };
+
 const RECENCY_MINUTES: Record<NonNullable<SearchParams["recency"]>, number> = {
 	day: 1440,
 	week: 10080,
@@ -107,8 +111,9 @@ function appendTinyFishSources(sources: SearchSource[], results: readonly TinyFi
 export async function searchTinyFish(params: SearchParams): Promise<SearchResponse> {
 	const numResults = clampNumResults(params.numSearchResults ?? params.limit, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
 	const pageSize = Math.min(numResults, DEFAULT_NUM_RESULTS);
+	const parsed = params.parsedQuery ?? parseSearchQuery(params.query);
 	const tinyFishParams: TinyFishSearchParams = {
-		query: params.query,
+		query: parsed.hasDirectives ? formatQuery(parsed, TINYFISH_QUERY_SYNTAX) : params.query,
 		num_results: pageSize,
 		recency: params.recency,
 		signal: params.signal,

package/src/web/search/providers/xai.ts

@@ -3,6 +3,7 @@ import { $env } from "@oh-my-pi/pi-utils";
 import { resolveXAIHttpTransport, type XAIHttpProvider, type XAIHttpTransport } from "../../../lib/xai-http";
 import type { SearchCitation, SearchResponse, SearchSource, SearchUsage } from "../../../web/search/types";
 import { SearchProviderError } from "../../../web/search/types";
+import { formatQuery, parseSearchQuery, type QuerySyntax } from "../query";
 import { clampNumResults } from "../utils";
 import type { SearchParams } from "./base";
 import { SearchProvider } from "./base";
@@ -57,14 +58,60 @@ interface XAIResponsesResponse {
 	usage?: XAIResponsesUsage | null;
 }
 
+/**
+ * Query syntax re-emitted for the Grok search agent. `site:`/`-site:` are
+ * stripped because hosts map natively onto the web_search domain filters;
+ * `before:`/`after:` stay in the query text — the Responses web_search tool
+ * has no date parameters (`from_date`/`to_date` exist only on `x_search` and
+ * the deprecated Live Search `search_parameters`, which now returns 410) and
+ * the agent honors the tokens as natural-language hints.
+ */
+const XAI_QUERY_SYNTAX: QuerySyntax = {
+	phrases: true,
+	negation: true,
+	or: true,
+	inUrl: true,
+	inTitle: true,
+	filetype: true,
+	dateRange: true,
+};
+
+/** xAI web_search accepts at most 5 allowed or excluded domains per request. */
+const MAX_DOMAIN_FILTERS = 5;
+
+/** Bare hosts of `site:` values (`github.com/anthropics` → `github.com`), deduped, capped at 5; path parts are enforced by the central constraint filter. */
+function domainFilterList(sites: readonly string[]): string[] {
+	const hosts = new Set<string>();
+	for (const site of sites) {
+		const slash = site.indexOf("/");
+		hosts.add(slash === -1 ? site : site.slice(0, slash));
+		if (hosts.size === MAX_DOMAIN_FILTERS) break;
+	}
+	return [...hosts];
+}
+
 function buildRequestBody(params: SearchParams): Record<string, unknown> {
+	const parsed = params.parsedQuery ?? parseSearchQuery(params.query);
+	const webSearchTool: Record<string, unknown> = { type: "web_search" };
+	let query = params.query;
+	if (parsed.hasDirectives) {
+		query = formatQuery(parsed, XAI_QUERY_SYNTAX);
+		// allowed_domains and excluded_domains are mutually exclusive per
+		// request; prefer the allow list, the central filter enforces exclusions.
+		if (parsed.sites.length > 0) {
+			webSearchTool.filters = { allowed_domains: domainFilterList(parsed.sites) };
+		} else if (parsed.excludedSites.length > 0) {
+			webSearchTool.filters = { excluded_domains: domainFilterList(parsed.excludedSites) };
+		}
+	}
+
 	const body: Record<string, unknown> = {
 		model: XAI_WEB_SEARCH_MODEL,
 		input: [
 			{ role: "system", content: params.systemPrompt },
-			{ role: "user", content: params.query },
+			{ role: "user", content: query },
 		],
-		tools: [{ type: "web_search" }],
+		tools: [webSearchTool],
 		reasoning: { effort: XAI_WEB_SEARCH_REASONING_EFFORT },
 	};
 

package/src/web/search/providers/zai.ts

@@ -8,6 +8,7 @@ import { type ApiKey, type AuthStorage, type FetchImpl, getEnvApiKey, withAuth }
 import { isRecord } from "@oh-my-pi/pi-utils";
 import type { SearchResponse, SearchSource } from "../../../web/search/types";
 import { SearchProviderError } from "../../../web/search/types";
+import { formatQuery, parseSearchQuery, type QuerySyntax } from "../query";
 import { dateToAgeSeconds } from "../utils";
 import type { SearchParams } from "./base";
 import { SearchProvider } from "./base";
@@ -17,6 +18,22 @@ const ZAI_MCP_URL = "https://api.z.ai/api/mcp/web_search_prime/mcp";
 const ZAI_TOOL_NAME = "web_search_prime";
 const DEFAULT_NUM_RESULTS = 10;
 
+/**
+ * webSearchPrime exposes no native filter args (the Web Search API's
+ * `search_domain_filter`/`search_recency_filter` are scoped to the
+ * `search_pro_jina` engine, not `search-prime`), but its Bing-flavored
+ * backend parses the common inline operators. Dates and language are left
+ * to the central lenient post-filter.
+ */
+const ZAI_QUERY_SYNTAX: QuerySyntax = {
+	phrases: true,
+	negation: true,
+	site: true,
+	inTitle: true,
+	inUrl: true,
+	filetype: true,
+};
+
 export interface ZaiSearchParams {
 	query: string;
 	num_results?: number;
@@ -407,8 +424,9 @@ export class ZaiProvider extends SearchProvider {
 
 	search(params: SearchParams): Promise<SearchResponse> {
 		const { fetch: fetchOverride } = params as ZaiProviderSearchParams;
+		const parsed = params.parsedQuery ?? parseSearchQuery(params.query);
 		return searchZai({
-			query: params.query,
+			query: parsed.hasDirectives ? formatQuery(parsed, ZAI_QUERY_SYNTAX) : params.query,
 			num_results: params.numSearchResults ?? params.limit,
 			signal: params.signal,
 			authStorage: params.authStorage,