dsh-web-search-plugin 0.2.0 → 0.3.0

package/lib/index.js

@@ -22,6 +22,9 @@ import { MAX_RESULTS_CAP } from "./shared.js";
 import { TAVILY_DEFAULT_API_KEY_ENV, TAVILY_DEFAULT_BASE_URL, TavilySearchProvider, resolveTavilyOptions } from "./tavily.js";
 import { BRAVE_DEFAULT_API_KEY_ENV, BRAVE_DEFAULT_BASE_URL, BraveSearchProvider, resolveBraveOptions } from "./brave.js";
 import { DEEPSEEK_DEFAULT_API_KEY_ENV, DEEPSEEK_DEFAULT_BASE_URL, DeepSeekSearchProvider, resolveDeepseekOptions } from "./deepseek.js";
+import { TAVILY_USAGE_POLL_MS, UsageTracker } from "./usage.js";
+import { RestSearchProvider, resolveRestProvider } from "./rest.js";
+import { BUILTIN_REST_IDS, REST_PROVIDERS, findProvider } from "./providers.js";
 
 /** Stable id this plugin registers under on `ctx.web`. */
 const SEARCH_PROVIDER_ID = "dsh-web-search";
@@ -33,8 +36,8 @@ const inject = ["web"];
 
 /** Plugin config (all optional — `apply` fills defaults). */
 const Config = z.object({
-	/** Active backend: `deepseek-official`, `tavily` (default), or `brave`. */
-	provider: z.string().default("tavily"),
+	/** Active backend: `deepseek-official` (default), `tavily`, `brave`, `serper`, `serpapi`, `exa`, or `searxng`. */
+	provider: z.string().default("deepseek-official"),
 	/** Tavily auth mode: `keyless` (default) or `keyed`. */
 	mode: z.string().default("keyless"),
 	/** Literal Tavily API key; prefer {@link apiKeyEnv}. */
@@ -65,6 +68,38 @@ const Config = z.object({
 	freshness: z.string().default(""),
 	/** Optional HTTP(S) proxy URL for Brave (falls back to HTTPS_PROXY / HTTP_PROXY). */
 	proxy: z.string().default(""),
+	/** Literal Serper API key; prefer {@link serperApiKeyEnv}. */
+	serperApiKey: z.string().role("secret").default(""),
+	/** Credential reference for Serper search. */
+	serperApiKeyEnv: z.string().role("credential-ref").default("SERPER_API_KEY"),
+	/** Serper endpoint base. `/search` is appended. */
+	serperBaseURL: z.string().default(""),
+	/** Literal SerpApi API key; prefer {@link serpapiApiKeyEnv}. */
+	serpapiApiKey: z.string().role("secret").default(""),
+	/** Credential reference for SerpApi search. */
+	serpapiApiKeyEnv: z.string().role("credential-ref").default("SERPAPI_API_KEY"),
+	/** SerpApi endpoint base. `/search.json` is appended. */
+	serpapiBaseURL: z.string().default(""),
+	/** Literal Exa API key; prefer {@link exaApiKeyEnv}. */
+	exaApiKey: z.string().role("secret").default(""),
+	/** Credential reference for Exa search. */
+	exaApiKeyEnv: z.string().role("credential-ref").default("EXA_API_KEY"),
+	/** Exa endpoint base. `/search` is appended. */
+	exaBaseURL: z.string().default(""),
+	/** Optional SearXNG instance base URL (self-hosted). `/search` is appended. */
+	searxngBaseURL: z.string().default(""),
+	/** Literal Scavio API key; prefer {@link scavioApiKeyEnv}. */
+	scavioApiKey: z.string().role("secret").default(""),
+	/** Credential reference for Scavio search. */
+	scavioApiKeyEnv: z.string().role("credential-ref").default("SCAVIO_API_KEY"),
+	/** Scavio endpoint base. `/api/v2/google` is appended. */
+	scavioBaseURL: z.string().default(""),
+	/** Literal Firecrawl API key; prefer {@link firecrawlApiKeyEnv}. */
+	firecrawlApiKey: z.string().role("secret").default(""),
+	/** Credential reference for Firecrawl search. */
+	firecrawlApiKeyEnv: z.string().role("credential-ref").default("FIRECRAWL_API_KEY"),
+	/** Firecrawl endpoint base. `/v2/search` is appended. */
+	firecrawlBaseURL: z.string().default(""),
 	/** Literal DeepSeek API key; prefer {@link deepseekApiKeyEnv}. */
 	deepseekApiKey: z.string().role("secret").default(""),
 	/** Credential reference for DeepSeek search. */
@@ -94,11 +129,24 @@ const WEB_SEARCH_TAVILY_SETTINGS_NAMESPACE = WEB_SEARCH_SETTINGS_NAMESPACE;
 class PluginSearchProvider {
 	id = SEARCH_PROVIDER_ID;
 
-	constructor(resolveOptions) {
+	constructor(resolveOptions, usage) {
 		this.resolveOptions = resolveOptions;
 		this.deepseek = new DeepSeekSearchProvider(() => this.resolveOptions().deepseek);
-		this.tavily = new TavilySearchProvider(() => this.resolveOptions().tavily);
-		this.brave = new BraveSearchProvider(() => this.resolveOptions().brave);
+		this.tavily = new TavilySearchProvider(() => ({
+			...this.resolveOptions().tavily,
+			onUsage: (credits) => usage?.recordTavilyCredits(credits)
+		}));
+		this.brave = new BraveSearchProvider(() => ({
+			...this.resolveOptions().brave,
+			onRateLimit: (headers) => usage?.recordBraveHeaders(headers)
+		}));
+		this.rest = new Map();
+		for (const providerId of BUILTIN_REST_IDS) {
+			this.rest.set(providerId, new RestSearchProvider(providerId, () => this.resolveOptions().rest[providerId], {
+				onUsage: providerId === "tavily" ? (credits) => usage?.recordTavilyCredits(credits) : void 0,
+				onRateLimit: providerId === "brave" ? (headers) => usage?.recordBraveHeaders(headers) : void 0
+			}));
+		}
 	}
 
 	available() {
@@ -110,24 +158,98 @@ class PluginSearchProvider {
 	}
 
 	backend() {
-		const provider = this.resolveOptions().provider;
-		if (provider === "brave") return this.brave;
+		const options = this.resolveOptions();
+		const provider = options.provider;
+		const restInstance = this.rest.get(provider);
+		if (restInstance !== void 0) return restInstance;
 		if (provider === "deepseek-official") return this.deepseek;
-		return this.tavily;
+		// Fallback: provider points at something unknown → official default.
+		return this.deepseek;
 	}
 }
 
 /** Project one resolved section into options for each backend. */
 function resolveOptions(ctx, config) {
-	const provider = config.provider === "brave" || config.provider === "deepseek-official" ? config.provider : "tavily";
+	const isBuiltIn = config.provider === "tavily" || config.provider === "brave" || config.provider === "deepseek-official" || BUILTIN_REST_IDS.includes(config.provider);
+	const provider = isBuiltIn ? config.provider : "deepseek-official";
+	// Per-provider literal key / optional endpoint override, used when the
+	// metadata row's default is not enough (self-hosted SearXNG, proxied
+	// providers, custom key refs).
+	const keySourceByProvider = {
+		brave: { literal: config.braveApiKey, envName: config.braveApiKeyEnv ?? BRAVE_DEFAULT_API_KEY_ENV, baseURL: config.braveBaseURL ?? "", proxy: config.proxy },
+		tavily: { literal: config.apiKey, envName: config.apiKeyEnv ?? TAVILY_DEFAULT_API_KEY_ENV, baseURL: config.baseURL ?? "" },
+		serper: { literal: config.serperApiKey, envName: config.serperApiKeyEnv ?? "SERPER_API_KEY", baseURL: config.serperBaseURL ?? "" },
+		serpapi: { literal: config.serpapiApiKey, envName: config.serpapiApiKeyEnv ?? "SERPAPI_API_KEY", baseURL: config.serpapiBaseURL ?? "" },
+		exa: { literal: config.exaApiKey, envName: config.exaApiKeyEnv ?? "EXA_API_KEY", baseURL: config.exaBaseURL ?? "" },
+		searxng: { baseURL: config.searxngBaseURL ?? "" },
+		scavio: { literal: config.scavioApiKey, envName: config.scavioApiKeyEnv ?? "SCAVIO_API_KEY", baseURL: config.scavioBaseURL ?? "" },
+		firecrawl: { literal: config.firecrawlApiKey, envName: config.firecrawlApiKeyEnv ?? "FIRECRAWL_API_KEY", baseURL: config.firecrawlBaseURL ?? "" }
+	};
+	const settings = {
+		country: config.country,
+		searchLang: config.searchLang,
+		freshness: config.freshness,
+		searchDepth: config.searchDepth,
+		topic: config.topic,
+		includeAnswer: config.includeAnswer
+	};
+	const rest = {};
+	for (const providerId of BUILTIN_REST_IDS) {
+		const row = findProvider(providerId);
+		if (row === void 0) continue;
+		const source = keySourceByProvider[providerId] ?? {};
+		const resolved = resolveRestProvider(ctx, row, {
+			config,
+			settings,
+			keyed: providerId === "tavily" ? config.mode === "keyed" : false,
+			keySource: source
+		});
+		// Endpoint override: an explicit config baseURL wins over the row default.
+		if (source.baseURL != null && source.baseURL.length > 0) resolved.baseURL = source.baseURL;
+		if (providerId === "brave" && (source.proxy ?? "").length > 0) resolved.proxy = source.proxy;
+		rest[providerId] = resolved;
+	}
 	return {
 		provider,
 		deepseek: resolveDeepseekOptions(ctx, config),
 		tavily: resolveTavilyOptions(ctx, config),
-		brave: resolveBraveOptions(ctx, config)
+		brave: resolveBraveOptions(ctx, config),
+		rest
 	};
 }
 
+/** JSON body helper for the usage refresh POST. */
+function readJsonBody(req) {
+	return new Promise((resolve, reject) => {
+		let body = "";
+		req.on("data", (chunk) => {
+			body += chunk;
+			if (body.length > 1e6) {
+				req.destroy();
+				reject(new Error("Payload too large"));
+			}
+		});
+		req.on("end", () => {
+			try {
+				resolve(body ? JSON.parse(body) : {});
+			} catch {
+				reject(new Error("Invalid JSON"));
+			}
+		});
+		req.on("error", reject);
+	});
+}
+
+function sendJson(res, statusCode, data) {
+	const body = JSON.stringify(data);
+	res.writeHead(statusCode, {
+		"Content-Type": "application/json; charset=utf-8",
+		"Cache-Control": "no-store",
+		"Content-Length": Buffer.byteLength(body)
+	});
+	res.end(body);
+}
+
 /** Register the dispatched search provider with `ctx.web`. */
 function apply(ctx, config) {
 	let current = () => config;
@@ -137,24 +259,85 @@ function apply(ctx, config) {
 		},
 		onChange: () => {}
 	});
-	ctx.web.registerSearchProvider(new PluginSearchProvider(() => resolveOptions(ctx, current())));
+	const live = () => resolveOptions(ctx, current());
+	const usage = new UsageTracker({
+		getTavilyOptions: () => live().tavily
+	});
+	ctx.effect(() => {
+		let cancelled = false;
+		void usage.hydrate().then(() => {
+			if (cancelled) return;
+			if (live().provider === "tavily" && live().tavily.mode === "keyed") return usage.refreshTavily(undefined, true);
+		}).catch(() => {});
+		return () => {
+			cancelled = true;
+			usage.dispose();
+		};
+	}, "dsh-web-search-plugin: usage hydrate");
+	ctx.effect(() => {
+		const tick = () => {
+			if (live().provider === "tavily" && live().tavily.mode === "keyed") void usage.refreshTavily().catch(() => {});
+		};
+		const timer = setInterval(tick, TAVILY_USAGE_POLL_MS);
+		return () => clearInterval(timer);
+	}, "dsh-web-search-plugin: tavily usage poll");
+	ctx.web.registerSearchProvider(new PluginSearchProvider(live, usage));
+	ctx.inject(["webServer"], (webCtx) => {
+		webCtx.effect(() => webCtx.webServer.register({
+			kind: "exact",
+			path: "/dsh-web-search/usage",
+			async handler(req, res) {
+				if (req.method !== "GET" && req.method !== "HEAD" && req.method !== "POST") {
+					res.writeHead(405, { Allow: "GET, HEAD, POST" });
+					res.end();
+					return;
+				}
+				const url = new URL(req.url ?? "/", "http://127.0.0.1");
+				const force = req.method === "POST" || url.searchParams.get("force") === "1" || url.searchParams.get("force") === "true";
+				const options = live();
+				const tavilyState = usage.snapshot().tavily;
+				const missingIdentity = tavilyState.plan == null && tavilyState.limit == null;
+				if ((force || missingIdentity) && options.provider === "tavily" && options.tavily.mode === "keyed") {
+					try {
+						if (req.method === "POST") await readJsonBody(req).catch(() => ({}));
+						await usage.refreshTavily(undefined, force);
+					} catch {
+						/* serialize still returns the last cache */
+					}
+				}
+				if (req.method === "HEAD") {
+					res.writeHead(200, { "Content-Type": "application/json; charset=utf-8", "Cache-Control": "no-store" });
+					res.end();
+					return;
+				}
+				sendJson(res, 200, usage.serialize(options.provider, options.tavily.mode));
+			}
+		}), "dsh-web-search-plugin: usage route");
+	});
 }
 
 export {
 	BRAVE_DEFAULT_API_KEY_ENV,
 	BRAVE_DEFAULT_BASE_URL,
+	BUILTIN_REST_IDS,
 	BraveSearchProvider,
 	Config,
 	DEEPSEEK_DEFAULT_API_KEY_ENV,
 	DEEPSEEK_DEFAULT_BASE_URL,
 	DeepSeekSearchProvider,
+	REST_PROVIDERS,
+	RestSearchProvider,
 	SEARCH_PROVIDER_ID,
 	TAVILY_DEFAULT_API_KEY_ENV,
 	TAVILY_DEFAULT_BASE_URL,
+	TAVILY_USAGE_POLL_MS,
 	TavilySearchProvider,
+	UsageTracker,
 	WEB_SEARCH_SETTINGS_NAMESPACE,
 	WEB_SEARCH_TAVILY_SETTINGS_NAMESPACE,
 	apply,
+	findProvider,
 	inject,
-	name
+	name,
+	resolveRestProvider
 };

package/lib/providers.js

@@ -0,0 +1,212 @@
+/**
+ * Static metadata table for every built-in REST web-search provider.
+ *
+ * This module is intentionally **data, not code**: adding a new REST provider
+ * means adding one row here (plus an optional official "get a key" link in the
+ * settings card), with zero per-provider execution code. The generic backend in
+ * `lib/rest.js` reads a row and performs `method` / `path` / `queryParam` /
+ * `countParam` / `params` / `auth` / `response` / `hooks`.
+ *
+ * Only **pure REST providers** ("one query → one HTTP request → one result
+ * array") belong here. Model-native search tool (DeepSeek's Anthropic Messages
+ * `web_search_20250305`, Perplexity sonar tools, OpenAI Responses) is **not**
+ * REST and stays on its dedicated backend (`lib/deepseek.js`).
+ *
+ * @module dsh-web-search-plugin/providers
+ */
+
+/**
+ * Built-in REST providers.
+ * `auth.type` ∈ { `bearer`, `header`, `none`, `query` }:
+ *   - `bearer` — `Authorization: Bearer <key>`
+ *   - `header` — key in a named header (`auth.header`)
+ *   - `none`   — no key (SearXNG, keyless)
+ *   - `query`  — key as a query-string parameter (`auth.param`, e.g. SerpApi)
+ * `params[].when` ∈ { `always`, `nonEmpty`, `keyed`, `keyless` }.
+ */
+export const REST_PROVIDERS = [
+	{
+		id: "brave",
+		name: "Brave Search",
+		kind: "rest",
+		method: "GET",
+		baseURL: "https://api.search.brave.com/res/v1/web/search",
+		path: "",
+		queryIn: "query",
+		queryParam: "q",
+		countParam: "count",
+		auth: { type: "header", header: "X-Subscription-Token" },
+		keyRequired: true,
+		apiKeyRef: "BRAVE_API_KEY",
+		response: "brave",
+		params: [
+			{ key: "country", setting: "country", when: "nonEmpty" },
+			{ key: "search_lang", setting: "searchLang", when: "nonEmpty" },
+			{ key: "freshness", setting: "freshness", when: "nonEmpty" }
+		],
+		officialUrl: "https://api-dashboard.search.brave.com/",
+		hooks: ["rate-limit", "proxy"]
+	},
+	{
+		id: "tavily",
+		name: "Tavily",
+		kind: "rest",
+		method: "POST",
+		baseURL: "https://api.tavily.com",
+		path: "/search",
+		queryIn: "body",
+		queryParam: "query",
+		countParam: "max_results",
+		auth: { type: "bearer" },
+		keyRequired: false, // keyless by default; keyed only when a key is configured
+		apiKeyRef: "TAVILY_API_KEY",
+		response: "tavily",
+		params: [
+			{ key: "search_depth", setting: "searchDepth", when: "nonEmpty" },
+			{ key: "topic", setting: "topic", when: "nonEmpty" },
+			{ key: "include_answer", setting: "includeAnswer", when: "nonEmpty" },
+			{ key: "include_images", value: false, when: "always" },
+			{ key: "include_usage", value: true, when: "keyed" }
+		],
+		keylessHeaders: { "x-tavily-access-mode": "keyless" },
+		keylessHeaderWhen: "keyless",
+		officialUrl: "https://app.tavily.com/",
+		hooks: ["usage"]
+	},
+	{
+		id: "serper",
+		name: "Serper",
+		kind: "rest",
+		method: "POST",
+		baseURL: "https://google.serper.dev",
+		path: "/search",
+		queryIn: "body",
+		queryParam: "q",
+		countParam: "num",
+		auth: { type: "header", header: "X-API-KEY" },
+		keyRequired: true,
+		apiKeyRef: "SERPER_API_KEY",
+		response: "serper",
+		params: [],
+		officialUrl: "https://serper.dev/",
+		hooks: []
+	},
+	{
+		id: "serpapi",
+		name: "SerpApi",
+		kind: "rest",
+		method: "GET",
+		baseURL: "https://serpapi.com",
+		path: "/search.json",
+		queryIn: "query",
+		queryParam: "q",
+		countParam: "num",
+		auth: { type: "query", param: "api_key" },
+		keyRequired: true,
+		apiKeyRef: "SERPAPI_API_KEY",
+		response: "serper",
+		params: [],
+		officialUrl: "https://serpapi.com/",
+		hooks: []
+	},
+	{
+		id: "exa",
+		name: "Exa",
+		kind: "rest",
+		method: "POST",
+		baseURL: "https://api.exa.ai",
+		path: "/search",
+		queryIn: "body",
+		queryParam: "query",
+		countParam: "num_results",
+		auth: { type: "bearer" },
+		keyRequired: true,
+		apiKeyRef: "EXA_API_KEY",
+		response: "exa",
+		params: [
+			{ key: "contents", value: "text", when: "always" }
+		],
+		officialUrl: "https://dashboard.exa.ai/",
+		hooks: []
+	},
+	{
+		id: "searxng",
+		name: "SearXNG",
+		kind: "rest",
+		method: "GET",
+		baseURL: "https://searx.be",
+		path: "/search",
+		queryIn: "query",
+		queryParam: "q",
+		countParam: "count",
+		auth: { type: "none" },
+		keyRequired: false,
+		apiKeyRef: "",
+		response: "searxng",
+		params: [
+			{ key: "format", value: "json", when: "always" }
+		],
+		officialUrl: "https://docs.searxng.org/",
+		hooks: []
+	},
+	{
+		id: "scavio",
+		name: "Scavio",
+		kind: "rest",
+		method: "POST",
+		baseURL: "https://api.scavio.dev",
+		path: "/api/v2/google",
+		queryIn: "body",
+		queryParam: "query",
+		countParam: "",
+		auth: { type: "bearer" },
+		keyRequired: true,
+		apiKeyRef: "SCAVIO_API_KEY",
+		response: "serper",
+		params: [
+			{ key: "hl", setting: "searchLang", when: "nonEmpty" },
+			{ key: "gl", setting: "country", when: "nonEmpty" }
+		],
+		officialUrl: "https://scavio.dev/",
+		hooks: []
+	},
+	{
+		id: "firecrawl",
+		name: "Firecrawl",
+		kind: "rest",
+		method: "POST",
+		baseURL: "https://api.firecrawl.dev",
+		path: "/v2/search",
+		queryIn: "body",
+		queryParam: "query",
+		countParam: "limit",
+		auth: { type: "bearer" },
+		keyRequired: true,
+		apiKeyRef: "FIRECRAWL_API_KEY",
+		response: "firecrawl",
+		params: [
+			{ key: "sources", value: "web", when: "always" }
+		],
+		officialUrl: "https://firecrawl.dev/",
+		hooks: []
+	}
+];
+
+/** Location of one provider by id (the id doubles as the `provider` value). */
+export function findProvider(id) {
+	for (const entry of REST_PROVIDERS) if (entry.id === id) return entry;
+	return void 0;
+}
+
+/** Set of ids that live in the metadata table. */
+export function restProviderIds() {
+	const ids = /* @__PURE__ */ new Set();
+	for (const entry of REST_PROVIDERS) ids.add(entry.id);
+	return ids;
+}
+
+/** The built-in REST provider ids (seam-facing `provider` values). */
+export const BUILTIN_REST_IDS = REST_PROVIDERS.map((entry) => entry.id);
+
+/** The model-native (non-REST) provider id, kept on its own backend. */
+export const TOOL_PROVIDER_IDS = ["deepseek-official"];