dsh-web-search-plugin 0.2.1 → 0.3.1

package/lib/index.js

@@ -18,11 +18,14 @@
  */
 import z from "@deepseek-ai/schemastery";
 import { installSettingsSection, settingsNamespace } from "@deepseek-ai/dsh-settings";
+import { launchEnvironmentOf } from "@deepseek-ai/dsh-launch-environment";
 import { MAX_RESULTS_CAP } from "./shared.js";
 import { TAVILY_DEFAULT_API_KEY_ENV, TAVILY_DEFAULT_BASE_URL, TavilySearchProvider, resolveTavilyOptions } from "./tavily.js";
 import { BRAVE_DEFAULT_API_KEY_ENV, BRAVE_DEFAULT_BASE_URL, BraveSearchProvider, resolveBraveOptions } from "./brave.js";
 import { DEEPSEEK_DEFAULT_API_KEY_ENV, DEEPSEEK_DEFAULT_BASE_URL, DeepSeekSearchProvider, resolveDeepseekOptions } from "./deepseek.js";
 import { TAVILY_USAGE_POLL_MS, UsageTracker } from "./usage.js";
+import { RestSearchProvider, resolveRestProvider } from "./rest.js";
+import { BUILTIN_REST_IDS, REST_PROVIDERS, findProvider } from "./providers.js";
 
 /** Stable id this plugin registers under on `ctx.web`. */
 const SEARCH_PROVIDER_ID = "dsh-web-search";
@@ -34,8 +37,8 @@ const inject = ["web"];
 
 /** Plugin config (all optional — `apply` fills defaults). */
 const Config = z.object({
-	/** Active backend: `deepseek-official`, `tavily` (default), or `brave`. */
-	provider: z.string().default("tavily"),
+	/** Active backend: `deepseek-official` (default), `tavily`, `brave`, `serper`, `serpapi`, `exa`, or `searxng`. */
+	provider: z.string().default("deepseek-official"),
 	/** Tavily auth mode: `keyless` (default) or `keyed`. */
 	mode: z.string().default("keyless"),
 	/** Literal Tavily API key; prefer {@link apiKeyEnv}. */
@@ -66,6 +69,38 @@ const Config = z.object({
 	freshness: z.string().default(""),
 	/** Optional HTTP(S) proxy URL for Brave (falls back to HTTPS_PROXY / HTTP_PROXY). */
 	proxy: z.string().default(""),
+	/** Literal Serper API key; prefer {@link serperApiKeyEnv}. */
+	serperApiKey: z.string().role("secret").default(""),
+	/** Credential reference for Serper search. */
+	serperApiKeyEnv: z.string().role("credential-ref").default("SERPER_API_KEY"),
+	/** Serper endpoint base. `/search` is appended. */
+	serperBaseURL: z.string().default("https://google.serper.dev"),
+	/** Literal SerpApi API key; prefer {@link serpapiApiKeyEnv}. */
+	serpapiApiKey: z.string().role("secret").default(""),
+	/** Credential reference for SerpApi search. */
+	serpapiApiKeyEnv: z.string().role("credential-ref").default("SERPAPI_API_KEY"),
+	/** SerpApi endpoint base. `/search.json` is appended. */
+	serpapiBaseURL: z.string().default("https://serpapi.com"),
+	/** Literal Exa API key; prefer {@link exaApiKeyEnv}. */
+	exaApiKey: z.string().role("secret").default(""),
+	/** Credential reference for Exa search. */
+	exaApiKeyEnv: z.string().role("credential-ref").default("EXA_API_KEY"),
+	/** Exa endpoint base. `/search` is appended. */
+	exaBaseURL: z.string().default("https://api.exa.ai"),
+	/** Optional SearXNG instance base URL (self-hosted). `/search` is appended. */
+	searxngBaseURL: z.string().default("https://searx.be"),
+	/** Literal Scavio API key; prefer {@link scavioApiKeyEnv}. */
+	scavioApiKey: z.string().role("secret").default(""),
+	/** Credential reference for Scavio search. */
+	scavioApiKeyEnv: z.string().role("credential-ref").default("SCAVIO_API_KEY"),
+	/** Scavio endpoint base. `/api/v2/google` is appended. */
+	scavioBaseURL: z.string().default("https://api.scavio.dev"),
+	/** Literal Firecrawl API key; prefer {@link firecrawlApiKeyEnv}. */
+	firecrawlApiKey: z.string().role("secret").default(""),
+	/** Credential reference for Firecrawl search. */
+	firecrawlApiKeyEnv: z.string().role("credential-ref").default("FIRECRAWL_API_KEY"),
+	/** Firecrawl endpoint base. `/v2/search` is appended. */
+	firecrawlBaseURL: z.string().default("https://api.firecrawl.dev"),
 	/** Literal DeepSeek API key; prefer {@link deepseekApiKeyEnv}. */
 	deepseekApiKey: z.string().role("secret").default(""),
 	/** Credential reference for DeepSeek search. */
@@ -106,6 +141,13 @@ class PluginSearchProvider {
 			...this.resolveOptions().brave,
 			onRateLimit: (headers) => usage?.recordBraveHeaders(headers)
 		}));
+		this.rest = new Map();
+		for (const providerId of BUILTIN_REST_IDS) {
+			this.rest.set(providerId, new RestSearchProvider(providerId, () => this.resolveOptions().rest[providerId], {
+				onUsage: providerId === "tavily" ? (credits) => usage?.recordTavilyCredits(credits) : void 0,
+				onRateLimit: providerId === "brave" ? (headers) => usage?.recordBraveHeaders(headers) : void 0
+			}));
+		}
 	}
 
 	available() {
@@ -117,21 +159,63 @@ class PluginSearchProvider {
 	}
 
 	backend() {
-		const provider = this.resolveOptions().provider;
-		if (provider === "brave") return this.brave;
+		const options = this.resolveOptions();
+		const provider = options.provider;
+		const restInstance = this.rest.get(provider);
+		if (restInstance !== void 0) return restInstance;
 		if (provider === "deepseek-official") return this.deepseek;
-		return this.tavily;
+		// Fallback: provider points at something unknown → official default.
+		return this.deepseek;
 	}
 }
 
 /** Project one resolved section into options for each backend. */
 function resolveOptions(ctx, config) {
-	const provider = config.provider === "brave" || config.provider === "deepseek-official" ? config.provider : "tavily";
+	const isBuiltIn = config.provider === "tavily" || config.provider === "brave" || config.provider === "deepseek-official" || BUILTIN_REST_IDS.includes(config.provider);
+	const provider = isBuiltIn ? config.provider : "deepseek-official";
+	// Per-provider literal key / optional endpoint override, used when the
+	// metadata row's default is not enough (self-hosted SearXNG, proxied
+	// providers, custom key refs).
+	const keySourceByProvider = {
+		brave: { literal: config.braveApiKey, envName: config.braveApiKeyEnv ?? BRAVE_DEFAULT_API_KEY_ENV, baseURL: config.braveBaseURL ?? "", proxy: config.proxy || launchEnvironmentOf(ctx).get("HTTPS_PROXY")?.value || launchEnvironmentOf(ctx).get("HTTP_PROXY")?.value || "" },
+		tavily: { literal: config.apiKey, envName: config.apiKeyEnv ?? TAVILY_DEFAULT_API_KEY_ENV, baseURL: config.baseURL ?? "" },
+		serper: { literal: config.serperApiKey, envName: config.serperApiKeyEnv ?? "SERPER_API_KEY", baseURL: config.serperBaseURL ?? "" },
+		serpapi: { literal: config.serpapiApiKey, envName: config.serpapiApiKeyEnv ?? "SERPAPI_API_KEY", baseURL: config.serpapiBaseURL ?? "" },
+		exa: { literal: config.exaApiKey, envName: config.exaApiKeyEnv ?? "EXA_API_KEY", baseURL: config.exaBaseURL ?? "" },
+		searxng: { baseURL: config.searxngBaseURL ?? "" },
+		scavio: { literal: config.scavioApiKey, envName: config.scavioApiKeyEnv ?? "SCAVIO_API_KEY", baseURL: config.scavioBaseURL ?? "" },
+		firecrawl: { literal: config.firecrawlApiKey, envName: config.firecrawlApiKeyEnv ?? "FIRECRAWL_API_KEY", baseURL: config.firecrawlBaseURL ?? "" }
+	};
+	const settings = {
+		country: config.country,
+		searchLang: config.searchLang,
+		freshness: config.freshness,
+		searchDepth: config.searchDepth,
+		topic: config.topic,
+		includeAnswer: config.includeAnswer
+	};
+	const rest = {};
+	for (const providerId of BUILTIN_REST_IDS) {
+		const row = findProvider(providerId);
+		if (row === void 0) continue;
+		const source = keySourceByProvider[providerId] ?? {};
+		const resolved = resolveRestProvider(ctx, row, {
+			config,
+			settings,
+			keyed: providerId === "tavily" ? config.mode === "keyed" : false,
+			keySource: source
+		});
+		// Endpoint override: an explicit config baseURL wins over the row default.
+		if (source.baseURL != null && source.baseURL.length > 0) resolved.baseURL = source.baseURL;
+		if (providerId === "brave" && (source.proxy ?? "").length > 0) resolved.proxy = source.proxy;
+		rest[providerId] = resolved;
+	}
 	return {
 		provider,
 		deepseek: resolveDeepseekOptions(ctx, config),
 		tavily: resolveTavilyOptions(ctx, config),
-		brave: resolveBraveOptions(ctx, config)
+		brave: resolveBraveOptions(ctx, config),
+		rest
 	};
 }
 
@@ -236,11 +320,14 @@ function apply(ctx, config) {
 export {
 	BRAVE_DEFAULT_API_KEY_ENV,
 	BRAVE_DEFAULT_BASE_URL,
+	BUILTIN_REST_IDS,
 	BraveSearchProvider,
 	Config,
 	DEEPSEEK_DEFAULT_API_KEY_ENV,
 	DEEPSEEK_DEFAULT_BASE_URL,
 	DeepSeekSearchProvider,
+	REST_PROVIDERS,
+	RestSearchProvider,
 	SEARCH_PROVIDER_ID,
 	TAVILY_DEFAULT_API_KEY_ENV,
 	TAVILY_DEFAULT_BASE_URL,
@@ -250,6 +337,8 @@ export {
 	WEB_SEARCH_SETTINGS_NAMESPACE,
 	WEB_SEARCH_TAVILY_SETTINGS_NAMESPACE,
 	apply,
+	findProvider,
 	inject,
-	name
+	name,
+	resolveRestProvider
 };

package/lib/providers.js

@@ -0,0 +1,212 @@
+/**
+ * Static metadata table for every built-in REST web-search provider.
+ *
+ * This module is intentionally **data, not code**: adding a new REST provider
+ * means adding one row here (plus an optional official "get a key" link in the
+ * settings card), with zero per-provider execution code. The generic backend in
+ * `lib/rest.js` reads a row and performs `method` / `path` / `queryParam` /
+ * `countParam` / `params` / `auth` / `response` / `hooks`.
+ *
+ * Only **pure REST providers** ("one query → one HTTP request → one result
+ * array") belong here. Model-native search tool (DeepSeek's Anthropic Messages
+ * `web_search_20250305`, Perplexity sonar tools, OpenAI Responses) is **not**
+ * REST and stays on its dedicated backend (`lib/deepseek.js`).
+ *
+ * @module dsh-web-search-plugin/providers
+ */
+
+/**
+ * Built-in REST providers.
+ * `auth.type` ∈ { `bearer`, `header`, `none`, `query` }:
+ *   - `bearer` — `Authorization: Bearer <key>`
+ *   - `header` — key in a named header (`auth.header`)
+ *   - `none`   — no key (SearXNG, keyless)
+ *   - `query`  — key as a query-string parameter (`auth.param`, e.g. SerpApi)
+ * `params[].when` ∈ { `always`, `nonEmpty`, `keyed`, `keyless` }.
+ */
+export const REST_PROVIDERS = [
+	{
+		id: "brave",
+		name: "Brave Search",
+		kind: "rest",
+		method: "GET",
+		baseURL: "https://api.search.brave.com/res/v1/web/search",
+		path: "",
+		queryIn: "query",
+		queryParam: "q",
+		countParam: "count",
+		auth: { type: "header", header: "X-Subscription-Token" },
+		keyRequired: true,
+		apiKeyRef: "BRAVE_API_KEY",
+		response: "brave",
+		params: [
+			{ key: "country", setting: "country", when: "nonEmpty" },
+			{ key: "search_lang", setting: "searchLang", when: "nonEmpty" },
+			{ key: "freshness", setting: "freshness", when: "nonEmpty" }
+		],
+		officialUrl: "https://api-dashboard.search.brave.com/",
+		hooks: ["rate-limit", "proxy"]
+	},
+	{
+		id: "tavily",
+		name: "Tavily",
+		kind: "rest",
+		method: "POST",
+		baseURL: "https://api.tavily.com",
+		path: "/search",
+		queryIn: "body",
+		queryParam: "query",
+		countParam: "max_results",
+		auth: { type: "bearer" },
+		keyRequired: false, // keyless by default; keyed only when a key is configured
+		apiKeyRef: "TAVILY_API_KEY",
+		response: "tavily",
+		params: [
+			{ key: "search_depth", setting: "searchDepth", when: "nonEmpty" },
+			{ key: "topic", setting: "topic", when: "nonEmpty" },
+			{ key: "include_answer", setting: "includeAnswer", when: "nonEmpty" },
+			{ key: "include_images", value: false, when: "always" },
+			{ key: "include_usage", value: true, when: "keyed" }
+		],
+		keylessHeaders: { "x-tavily-access-mode": "keyless" },
+		keylessHeaderWhen: "keyless",
+		officialUrl: "https://app.tavily.com/",
+		hooks: ["usage"]
+	},
+	{
+		id: "serper",
+		name: "Serper",
+		kind: "rest",
+		method: "POST",
+		baseURL: "https://google.serper.dev",
+		path: "/search",
+		queryIn: "body",
+		queryParam: "q",
+		countParam: "num",
+		auth: { type: "header", header: "X-API-KEY" },
+		keyRequired: true,
+		apiKeyRef: "SERPER_API_KEY",
+		response: "serper",
+		params: [],
+		officialUrl: "https://serper.dev/",
+		hooks: []
+	},
+	{
+		id: "serpapi",
+		name: "SerpApi",
+		kind: "rest",
+		method: "GET",
+		baseURL: "https://serpapi.com",
+		path: "/search.json",
+		queryIn: "query",
+		queryParam: "q",
+		countParam: "num",
+		auth: { type: "query", param: "api_key" },
+		keyRequired: true,
+		apiKeyRef: "SERPAPI_API_KEY",
+		response: "serper",
+		params: [],
+		officialUrl: "https://serpapi.com/",
+		hooks: []
+	},
+	{
+		id: "exa",
+		name: "Exa",
+		kind: "rest",
+		method: "POST",
+		baseURL: "https://api.exa.ai",
+		path: "/search",
+		queryIn: "body",
+		queryParam: "query",
+		countParam: "num_results",
+		auth: { type: "bearer" },
+		keyRequired: true,
+		apiKeyRef: "EXA_API_KEY",
+		response: "exa",
+		params: [
+			{ key: "contents", value: "text", when: "always" }
+		],
+		officialUrl: "https://dashboard.exa.ai/",
+		hooks: []
+	},
+	{
+		id: "searxng",
+		name: "SearXNG",
+		kind: "rest",
+		method: "GET",
+		baseURL: "https://searx.be",
+		path: "/search",
+		queryIn: "query",
+		queryParam: "q",
+		countParam: "count",
+		auth: { type: "none" },
+		keyRequired: false,
+		apiKeyRef: "",
+		response: "searxng",
+		params: [
+			{ key: "format", value: "json", when: "always" }
+		],
+		officialUrl: "https://docs.searxng.org/",
+		hooks: []
+	},
+	{
+		id: "scavio",
+		name: "Scavio",
+		kind: "rest",
+		method: "POST",
+		baseURL: "https://api.scavio.dev",
+		path: "/api/v2/google",
+		queryIn: "body",
+		queryParam: "query",
+		countParam: "",
+		auth: { type: "bearer" },
+		keyRequired: true,
+		apiKeyRef: "SCAVIO_API_KEY",
+		response: "serper",
+		params: [
+			{ key: "hl", setting: "searchLang", when: "nonEmpty" },
+			{ key: "gl", setting: "country", when: "nonEmpty" }
+		],
+		officialUrl: "https://scavio.dev/",
+		hooks: []
+	},
+	{
+		id: "firecrawl",
+		name: "Firecrawl",
+		kind: "rest",
+		method: "POST",
+		baseURL: "https://api.firecrawl.dev",
+		path: "/v2/search",
+		queryIn: "body",
+		queryParam: "query",
+		countParam: "limit",
+		auth: { type: "bearer" },
+		keyRequired: true,
+		apiKeyRef: "FIRECRAWL_API_KEY",
+		response: "firecrawl",
+		params: [
+			{ key: "sources", value: "web", when: "always" }
+		],
+		officialUrl: "https://firecrawl.dev/",
+		hooks: []
+	}
+];
+
+/** Location of one provider by id (the id doubles as the `provider` value). */
+export function findProvider(id) {
+	for (const entry of REST_PROVIDERS) if (entry.id === id) return entry;
+	return void 0;
+}
+
+/** Set of ids that live in the metadata table. */
+export function restProviderIds() {
+	const ids = /* @__PURE__ */ new Set();
+	for (const entry of REST_PROVIDERS) ids.add(entry.id);
+	return ids;
+}
+
+/** The built-in REST provider ids (seam-facing `provider` values). */
+export const BUILTIN_REST_IDS = REST_PROVIDERS.map((entry) => entry.id);
+
+/** The model-native (non-REST) provider id, kept on its own backend. */
+export const TOOL_PROVIDER_IDS = ["deepseek-official"];