@dbx-tools/appkit-web-search 0.3.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +234 -0
- package/index.ts +21 -0
- package/package.json +54 -0
- package/src/allowlist.ts +143 -0
- package/src/config.ts +282 -0
- package/src/fetch.ts +118 -0
- package/src/plugin.ts +87 -0
- package/src/provider.ts +90 -0
- package/src/runtime.ts +40 -0
- package/src/schema.ts +116 -0
- package/src/scrape.ts +116 -0
- package/src/search.ts +257 -0
- package/src/tool.ts +144 -0
- package/tsconfig.json +41 -0
package/src/config.ts
ADDED
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Configuration for the web-search plugin: the typed
|
|
3
|
+
* {@link WebSearchPluginConfig} (the plugin's slice of AppKit config), the
|
|
4
|
+
* JSON Schema the manifest publishes for it, and {@link resolveWebSearchConfig}
|
|
5
|
+
* which layers that config over environment defaults into the concrete
|
|
6
|
+
* {@link ResolvedWebSearchConfig} the runtime + tools read.
|
|
7
|
+
*
|
|
8
|
+
* `web_search` runs on the Databricks Model Serving native web-search tool
|
|
9
|
+
* (see `provider.ts` / `search.ts`), so the key knob is which web-search-
|
|
10
|
+
* capable model to use. It resolves independently of the calling agent's chat
|
|
11
|
+
* model: `model` (a name, loose name, or capability class) is fuzzy-matched
|
|
12
|
+
* against the workspace catalogue, and when nothing is pinned the
|
|
13
|
+
* {@link WebSearchPluginConfig.modelFallbacks} order (Gemini, then GPT, then a
|
|
14
|
+
* repo floor) picks the first web-search-capable endpoint that exists.
|
|
15
|
+
*
|
|
16
|
+
* Resolution never throws here; it just fills defaults. Precedence per field:
|
|
17
|
+
* explicit plugin config wins, then the matching environment variable, then a
|
|
18
|
+
* built-in default.
|
|
19
|
+
*
|
|
20
|
+
* Env fallbacks: `WEB_SEARCH_MODEL`, `WEB_SEARCH_MODEL_FALLBACKS`,
|
|
21
|
+
* `WEB_SEARCH_TOOLS` (JSON), `WEB_SEARCH_ALLOWED_URLS`,
|
|
22
|
+
* `WEB_SEARCH_MAX_CITATIONS`, `WEB_SEARCH_FETCH_MAX_LENGTH`,
|
|
23
|
+
* `WEB_SEARCH_TIMEOUT_MS`, `WEB_SEARCH_FUZZY`, `WEB_SEARCH_FUZZY_THRESHOLD`.
|
|
24
|
+
*
|
|
25
|
+
* @module
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
import type { BasePluginConfig } from "@databricks/appkit";
|
|
29
|
+
import { object, type OneOrMany } from "@dbx-tools/shared-core";
|
|
30
|
+
import type { JSONSchema7 } from "json-schema";
|
|
31
|
+
import { parseAllowedUrls, toUrlAllowList, type UrlAllowList } from "./allowlist";
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* A URL-pattern gate for per-tool approval. `true` gates every call; a
|
|
35
|
+
* pattern (or list of patterns, in the {@link OneOrMany} shape used across
|
|
36
|
+
* the repo) gates only calls whose URL matches. Patterns use the same glob
|
|
37
|
+
* syntax as the allow-list (see `allowlist.ts`). Omit / `false` for no
|
|
38
|
+
* approval.
|
|
39
|
+
*/
|
|
40
|
+
export type ApprovalGate = boolean | OneOrMany<string> | string;
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Default web-search model preference, tried in order when no model is
|
|
44
|
+
* pinned. Gemini first, then GPT - both support the native web-search tool;
|
|
45
|
+
* a workspace typically has at least one. Each is fuzzy-matched against the
|
|
46
|
+
* live catalogue, so a close variant (e.g. `databricks-gemini-3-1-pro`) is
|
|
47
|
+
* picked when the exact id isn't present.
|
|
48
|
+
*/
|
|
49
|
+
export const DEFAULT_MODEL_FALLBACKS: readonly string[] = [
|
|
50
|
+
"databricks-gemini-3-pro",
|
|
51
|
+
"databricks-gemini-2-5-pro",
|
|
52
|
+
"databricks-gpt-5",
|
|
53
|
+
"databricks-gpt-5-mini",
|
|
54
|
+
];
|
|
55
|
+
|
|
56
|
+
/** Default cap on the number of citations returned from a single search. */
|
|
57
|
+
export const DEFAULT_MAX_CITATIONS = 10;
|
|
58
|
+
|
|
59
|
+
/** Default cap on characters returned from a single `web_fetch`. */
|
|
60
|
+
export const DEFAULT_FETCH_MAX_LENGTH = 50_000;
|
|
61
|
+
|
|
62
|
+
/** Default per-request network timeout (ms) for search + fetch. */
|
|
63
|
+
export const DEFAULT_TIMEOUT_MS = 30_000;
|
|
64
|
+
|
|
65
|
+
/** AppKit config accepted by the web-search plugin. */
|
|
66
|
+
export interface WebSearchPluginConfig extends BasePluginConfig {
|
|
67
|
+
/**
|
|
68
|
+
* The web-search model to use by default: a Databricks serving endpoint
|
|
69
|
+
* name (`"databricks-gemini-3-pro"`), a loose name (`"gemini"`, `"gpt"`),
|
|
70
|
+
* or a capability class. Fuzzy-matched against the live catalogue. Falls
|
|
71
|
+
* back to `WEB_SEARCH_MODEL`, then the {@link modelFallbacks} order. Chosen
|
|
72
|
+
* independently of the calling agent's chat model.
|
|
73
|
+
*/
|
|
74
|
+
model?: string;
|
|
75
|
+
/**
|
|
76
|
+
* Priority-ordered web-search model candidates tried when {@link model} is
|
|
77
|
+
* unset, each fuzzy-matched and checked for web-search support. Falls back
|
|
78
|
+
* to `WEB_SEARCH_MODEL_FALLBACKS` (comma/space-separated), then
|
|
79
|
+
* {@link DEFAULT_MODEL_FALLBACKS} (Gemini, then GPT).
|
|
80
|
+
*/
|
|
81
|
+
modelFallbacks?: string | string[];
|
|
82
|
+
/**
|
|
83
|
+
* Provider -> tool-spec override map, merged over the built-in
|
|
84
|
+
* {@link WEB_SEARCH_PROVIDERS} defaults. Keyed by provider family
|
|
85
|
+
* (`"openai"`, `"gemini"`); each value may override the `tool` entry
|
|
86
|
+
* and/or the `api` surface. Use to change the tool shape as the platform
|
|
87
|
+
* evolves without a code change. Falls back to `WEB_SEARCH_TOOLS` parsed as
|
|
88
|
+
* JSON. This is the `WEB_SEARCH_TOOLS` setting.
|
|
89
|
+
*/
|
|
90
|
+
webSearchTools?: Record<string, unknown>;
|
|
91
|
+
/**
|
|
92
|
+
* Enable fuzzy matching of loose model names against the catalogue.
|
|
93
|
+
* Defaults to `true`; falls back to `WEB_SEARCH_FUZZY`.
|
|
94
|
+
*/
|
|
95
|
+
modelFuzzyMatch?: boolean;
|
|
96
|
+
/** Fuse.js fuzzy threshold. Falls back to `WEB_SEARCH_FUZZY_THRESHOLD`, then 0.4. */
|
|
97
|
+
modelFuzzyThreshold?: number;
|
|
98
|
+
/**
|
|
99
|
+
* Hard cap on the number of citations a single search returns. Falls back
|
|
100
|
+
* to `WEB_SEARCH_MAX_CITATIONS`, then {@link DEFAULT_MAX_CITATIONS}.
|
|
101
|
+
*/
|
|
102
|
+
maxCitations?: number;
|
|
103
|
+
/**
|
|
104
|
+
* Hard cap on the character length of a single `web_fetch` result. Falls
|
|
105
|
+
* back to `WEB_SEARCH_FETCH_MAX_LENGTH`, then {@link DEFAULT_FETCH_MAX_LENGTH}.
|
|
106
|
+
*/
|
|
107
|
+
fetchMaxLength?: number;
|
|
108
|
+
/**
|
|
109
|
+
* Per-request network timeout in ms for search + fetch. Falls back to
|
|
110
|
+
* `WEB_SEARCH_TIMEOUT_MS`, then {@link DEFAULT_TIMEOUT_MS}.
|
|
111
|
+
*/
|
|
112
|
+
timeoutMs?: number;
|
|
113
|
+
/**
|
|
114
|
+
* Fall back to a DuckDuckGo scrape when the workspace has NO deployed
|
|
115
|
+
* web-search-capable model (no GPT / Gemini serving endpoint). The native
|
|
116
|
+
* Databricks tool is always preferred; this only kicks in when there is no
|
|
117
|
+
* native option, so the tool still returns results instead of erroring.
|
|
118
|
+
* Defaults to `true`; set `false` (or `WEB_SEARCH_SCRAPE_FALLBACK=0`) to
|
|
119
|
+
* require a native model and error otherwise.
|
|
120
|
+
*/
|
|
121
|
+
scrapeFallback?: boolean;
|
|
122
|
+
/**
|
|
123
|
+
* Optional URL allow-list. Each entry is a glob (or bare host) tested
|
|
124
|
+
* against a URL's full `href`. When set, `web_search` silently filters
|
|
125
|
+
* citations to the permitted set and `web_fetch` refuses a disallowed URL.
|
|
126
|
+
* Accepts a `string[]` or a comma-/whitespace-separated string; falls back
|
|
127
|
+
* to `WEB_SEARCH_ALLOWED_URLS`. Omit (or leave empty) for no restriction.
|
|
128
|
+
* See `allowlist.ts`.
|
|
129
|
+
*/
|
|
130
|
+
allowedUrls?: string | string[];
|
|
131
|
+
/**
|
|
132
|
+
* Approval gate applied to BOTH tools (per-tool overrides via
|
|
133
|
+
* {@link WebSearchToolOptions.approval} win). `true` gates every call;
|
|
134
|
+
* a URL-pattern (or list) gates only matching calls. Omit for no approval.
|
|
135
|
+
*/
|
|
136
|
+
approval?: ApprovalGate;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** Concrete, validated config the runtime + tools read. */
|
|
140
|
+
export interface ResolvedWebSearchConfig {
|
|
141
|
+
/** Pinned web-search model, when configured (else undefined - use fallbacks). */
|
|
142
|
+
model?: string;
|
|
143
|
+
/** Ordered fallback model candidates (Gemini, then GPT, then a floor). */
|
|
144
|
+
modelFallbacks: readonly string[];
|
|
145
|
+
/** Provider -> tool-spec override map, merged over the built-in defaults. */
|
|
146
|
+
webSearchTools: Record<string, unknown>;
|
|
147
|
+
/** Whether to fuzzy-match loose model names. */
|
|
148
|
+
fuzzy: boolean;
|
|
149
|
+
/** Fuse.js fuzzy threshold. */
|
|
150
|
+
fuzzyThreshold: number;
|
|
151
|
+
maxCitations: number;
|
|
152
|
+
fetchMaxLength: number;
|
|
153
|
+
timeoutMs: number;
|
|
154
|
+
/** Whether to scrape-fallback when no native web-search model is deployed. */
|
|
155
|
+
scrapeFallback: boolean;
|
|
156
|
+
/** Compiled allow-list (permit-all when unconfigured). */
|
|
157
|
+
allowList: UrlAllowList;
|
|
158
|
+
/** Default per-tool approval gate. */
|
|
159
|
+
approval: ApprovalGate;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** JSON Schema published on the manifest's `config.schema`. */
|
|
163
|
+
export const WEB_SEARCH_CONFIG_SCHEMA: JSONSchema7 = {
|
|
164
|
+
type: "object",
|
|
165
|
+
properties: {
|
|
166
|
+
model: {
|
|
167
|
+
type: "string",
|
|
168
|
+
description:
|
|
169
|
+
"Default web-search model (endpoint name, loose name, or class). Fuzzy-matched. Env: WEB_SEARCH_MODEL.",
|
|
170
|
+
},
|
|
171
|
+
modelFallbacks: {
|
|
172
|
+
type: "array",
|
|
173
|
+
items: { type: "string" },
|
|
174
|
+
description:
|
|
175
|
+
"Ordered web-search model candidates when `model` is unset (Gemini, then GPT). Env: WEB_SEARCH_MODEL_FALLBACKS.",
|
|
176
|
+
},
|
|
177
|
+
webSearchTools: {
|
|
178
|
+
type: "object",
|
|
179
|
+
description:
|
|
180
|
+
'Provider -> tool-spec override map merged over the built-in defaults (openai -> {"type":"web_search"}, gemini -> {"google_search":{}}). Env: WEB_SEARCH_TOOLS (JSON).',
|
|
181
|
+
},
|
|
182
|
+
maxCitations: {
|
|
183
|
+
type: "number",
|
|
184
|
+
description: "Hard cap on citations returned (env: WEB_SEARCH_MAX_CITATIONS).",
|
|
185
|
+
},
|
|
186
|
+
fetchMaxLength: {
|
|
187
|
+
type: "number",
|
|
188
|
+
description: "Hard cap on web_fetch content length (env: WEB_SEARCH_FETCH_MAX_LENGTH).",
|
|
189
|
+
},
|
|
190
|
+
timeoutMs: {
|
|
191
|
+
type: "number",
|
|
192
|
+
description: "Per-request network timeout in ms (env: WEB_SEARCH_TIMEOUT_MS).",
|
|
193
|
+
},
|
|
194
|
+
allowedUrls: {
|
|
195
|
+
type: "array",
|
|
196
|
+
items: { type: "string" },
|
|
197
|
+
description:
|
|
198
|
+
'URL allow-list of globs / bare hosts (e.g. "*.databricks.com", "docs.example.com"). Also accepts a comma/space-separated string. Falls back to WEB_SEARCH_ALLOWED_URLS. Empty = unrestricted.',
|
|
199
|
+
},
|
|
200
|
+
},
|
|
201
|
+
};
|
|
202
|
+
|
|
203
|
+
/** Split a CSV / whitespace / array value into a trimmed, non-empty string list. */
|
|
204
|
+
function toStringList(raw: string | string[] | undefined): string[] {
|
|
205
|
+
const entries = typeof raw === "string" ? raw.split(/[\s,]+/) : Array.isArray(raw) ? raw : [];
|
|
206
|
+
return [
|
|
207
|
+
...object
|
|
208
|
+
.sequence(entries)
|
|
209
|
+
.map((e) => e.trim())
|
|
210
|
+
.filter((e) => e.length > 0)
|
|
211
|
+
.distinct()
|
|
212
|
+
.toArray(),
|
|
213
|
+
];
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/** Parse a positive integer env/config value, else the fallback. */
|
|
217
|
+
function resolvePositiveInt(value: number | undefined, envKey: string, fallback: number): number {
|
|
218
|
+
if (typeof value === "number" && Number.isFinite(value) && value > 0) return Math.floor(value);
|
|
219
|
+
const env = Number(process.env[envKey]);
|
|
220
|
+
return Number.isFinite(env) && env > 0 ? Math.floor(env) : fallback;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/** Parse the `WEB_SEARCH_TOOLS` env var (JSON), else `{}`. Bad JSON is ignored. */
|
|
224
|
+
function parseToolsEnv(): Record<string, unknown> {
|
|
225
|
+
const raw = process.env["WEB_SEARCH_TOOLS"];
|
|
226
|
+
if (!raw) return {};
|
|
227
|
+
try {
|
|
228
|
+
const parsed = JSON.parse(raw) as unknown;
|
|
229
|
+
return parsed && typeof parsed === "object" ? (parsed as Record<string, unknown>) : {};
|
|
230
|
+
} catch {
|
|
231
|
+
return {};
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/**
|
|
236
|
+
* Resolve plugin config over environment defaults into the concrete
|
|
237
|
+
* {@link ResolvedWebSearchConfig}. Never throws - the model is resolved
|
|
238
|
+
* lazily at call time (against the live catalogue), so an unconfigured plugin
|
|
239
|
+
* resolves to sensible defaults with no restrictions.
|
|
240
|
+
*/
|
|
241
|
+
export function resolveWebSearchConfig(
|
|
242
|
+
config: WebSearchPluginConfig = {},
|
|
243
|
+
): ResolvedWebSearchConfig {
|
|
244
|
+
const patterns = parseAllowedUrls(config.allowedUrls ?? process.env["WEB_SEARCH_ALLOWED_URLS"]);
|
|
245
|
+
const model = config.model ?? process.env["WEB_SEARCH_MODEL"];
|
|
246
|
+
const fallbacks = toStringList(config.modelFallbacks ?? process.env["WEB_SEARCH_MODEL_FALLBACKS"]);
|
|
247
|
+
const fuzzyThresholdRaw = config.modelFuzzyThreshold ?? Number(process.env["WEB_SEARCH_FUZZY_THRESHOLD"]);
|
|
248
|
+
return {
|
|
249
|
+
...(model ? { model } : {}),
|
|
250
|
+
modelFallbacks: fallbacks.length > 0 ? fallbacks : DEFAULT_MODEL_FALLBACKS,
|
|
251
|
+
webSearchTools: { ...parseToolsEnv(), ...(config.webSearchTools ?? {}) },
|
|
252
|
+
fuzzy:
|
|
253
|
+
config.modelFuzzyMatch ?? object.toBoolean(process.env["WEB_SEARCH_FUZZY"]) ?? true,
|
|
254
|
+
fuzzyThreshold: Number.isFinite(fuzzyThresholdRaw) && fuzzyThresholdRaw ? Number(fuzzyThresholdRaw) : 0.4,
|
|
255
|
+
maxCitations: resolvePositiveInt(config.maxCitations, "WEB_SEARCH_MAX_CITATIONS", DEFAULT_MAX_CITATIONS),
|
|
256
|
+
fetchMaxLength: resolvePositiveInt(
|
|
257
|
+
config.fetchMaxLength,
|
|
258
|
+
"WEB_SEARCH_FETCH_MAX_LENGTH",
|
|
259
|
+
DEFAULT_FETCH_MAX_LENGTH,
|
|
260
|
+
),
|
|
261
|
+
timeoutMs: resolvePositiveInt(config.timeoutMs, "WEB_SEARCH_TIMEOUT_MS", DEFAULT_TIMEOUT_MS),
|
|
262
|
+
scrapeFallback:
|
|
263
|
+
config.scrapeFallback ?? object.toBoolean(process.env["WEB_SEARCH_SCRAPE_FALLBACK"]) ?? true,
|
|
264
|
+
allowList: toUrlAllowList(patterns),
|
|
265
|
+
approval: config.approval ?? false,
|
|
266
|
+
};
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/**
|
|
270
|
+
* Resolve an {@link ApprovalGate} against a set of candidate URLs into a
|
|
271
|
+
* concrete boolean: `true`/`false` pass through; a pattern (or list) gates
|
|
272
|
+
* when ANY candidate matches. Empty candidates with a pattern gate never
|
|
273
|
+
* match (nothing to approve). Reuses the allow-list matcher so approval
|
|
274
|
+
* globs read exactly like allow-list globs.
|
|
275
|
+
*/
|
|
276
|
+
export function approvalMatches(gate: ApprovalGate, urls: readonly string[]): boolean {
|
|
277
|
+
if (typeof gate === "boolean") return gate;
|
|
278
|
+
const patterns = parseAllowedUrls(typeof gate === "string" ? gate : [...gate]);
|
|
279
|
+
if (patterns.length === 0) return false;
|
|
280
|
+
const list = toUrlAllowList(patterns);
|
|
281
|
+
return urls.some((url) => list.allows(url));
|
|
282
|
+
}
|
package/src/fetch.ts
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Page fetching over {@link https://www.npmjs.com/package/got-scraping | got-scraping}
|
|
3
|
+
* - a `got` wrapper that generates browser-like TLS + header fingerprints, so
|
|
4
|
+
* fetches survive the bot walls a plain `fetch` trips. {@link runWebFetch}
|
|
5
|
+
* enforces the URL allow-list (an explicit fetch of a disallowed URL is
|
|
6
|
+
* refused, not silently emptied), fetches with the plugin's timeout, and
|
|
7
|
+
* returns either the raw HTML or a readable plain-text reduction, capped at
|
|
8
|
+
* the configured length.
|
|
9
|
+
*
|
|
10
|
+
* The HTML-to-text reduction is deliberately dependency-free (strip
|
|
11
|
+
* script/style, unwrap tags, decode a handful of entities, collapse
|
|
12
|
+
* whitespace): good enough to feed a model, and it keeps the add-on's
|
|
13
|
+
* dependency surface to the two libraries the task called for.
|
|
14
|
+
*
|
|
15
|
+
* @module
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import { log } from "@dbx-tools/shared-core";
|
|
19
|
+
import { gotScraping } from "got-scraping";
|
|
20
|
+
import { assertUrlAllowed } from "./allowlist";
|
|
21
|
+
import type { ResolvedWebSearchConfig } from "./config";
|
|
22
|
+
import type { WebFetchRequest, WebFetchResult } from "./schema";
|
|
23
|
+
|
|
24
|
+
const logger = log.logger("web-search/fetch");
|
|
25
|
+
|
|
26
|
+
/** Pull the <title> text out of an HTML document, when present. */
|
|
27
|
+
function extractTitle(html: string): string | undefined {
|
|
28
|
+
const match = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(html);
|
|
29
|
+
const title = match?.[1] ? decodeEntities(match[1]).trim() : "";
|
|
30
|
+
return title.length > 0 ? title : undefined;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Decode the handful of HTML entities that survive tag-stripping. */
|
|
34
|
+
function decodeEntities(text: string): string {
|
|
35
|
+
return text
|
|
36
|
+
.replace(/ /g, " ")
|
|
37
|
+
.replace(/&/g, "&")
|
|
38
|
+
.replace(/</g, "<")
|
|
39
|
+
.replace(/>/g, ">")
|
|
40
|
+
.replace(/"/g, '"')
|
|
41
|
+
.replace(/'/g, "'")
|
|
42
|
+
.replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code)));
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Reduce an HTML document to readable plain text: drop `<script>` /
|
|
47
|
+
* `<style>` / `<noscript>` blocks and HTML comments, turn block-level tags
|
|
48
|
+
* into newlines, strip the remaining tags, decode entities, and collapse
|
|
49
|
+
* runs of blank lines / trailing spaces.
|
|
50
|
+
*/
|
|
51
|
+
export function htmlToText(html: string): string {
|
|
52
|
+
return decodeEntities(
|
|
53
|
+
html
|
|
54
|
+
.replace(/<!--[\s\S]*?-->/g, "")
|
|
55
|
+
.replace(/<(script|style|noscript)[^>]*>[\s\S]*?<\/\1>/gi, "")
|
|
56
|
+
.replace(/<\/(p|div|section|article|li|tr|h[1-6]|header|footer|br)>/gi, "\n")
|
|
57
|
+
.replace(/<br\s*\/?>/gi, "\n")
|
|
58
|
+
.replace(/<[^>]+>/g, ""),
|
|
59
|
+
)
|
|
60
|
+
.replace(/[ \t]+\n/g, "\n")
|
|
61
|
+
.replace(/\n{3,}/g, "\n\n")
|
|
62
|
+
.replace(/[ \t]{2,}/g, " ")
|
|
63
|
+
.trim();
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** Truncate `text` to `max` chars, reporting whether it was cut. */
|
|
67
|
+
function truncate(text: string, max: number): { content: string; truncated: boolean } {
|
|
68
|
+
if (text.length <= max) return { content: text, truncated: false };
|
|
69
|
+
return { content: text.slice(0, max), truncated: true };
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Fetch a single URL and return its content in the requested format.
|
|
74
|
+
*
|
|
75
|
+
* Throws when the URL is not permitted by the allow-list (the visible,
|
|
76
|
+
* correctable failure the design calls for on the fetch path). Network /
|
|
77
|
+
* HTTP errors propagate from got-scraping. The request's `maxLength` narrows
|
|
78
|
+
* (never widens) the plugin's `fetchMaxLength` cap.
|
|
79
|
+
*/
|
|
80
|
+
export async function runWebFetch(
|
|
81
|
+
request: WebFetchRequest,
|
|
82
|
+
config: ResolvedWebSearchConfig,
|
|
83
|
+
): Promise<WebFetchResult> {
|
|
84
|
+
assertUrlAllowed(request.url, config.allowList);
|
|
85
|
+
const cap = Math.min(request.maxLength ?? config.fetchMaxLength, config.fetchMaxLength);
|
|
86
|
+
|
|
87
|
+
const response = await gotScraping({
|
|
88
|
+
url: request.url,
|
|
89
|
+
timeout: { request: config.timeoutMs },
|
|
90
|
+
throwHttpErrors: false,
|
|
91
|
+
followRedirect: true,
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
const body = typeof response.body === "string" ? response.body : String(response.body ?? "");
|
|
95
|
+
const contentType = response.headers["content-type"];
|
|
96
|
+
const isHtml = !contentType || /html|xml/i.test(contentType);
|
|
97
|
+
const rawContent =
|
|
98
|
+
request.format === "html" || !isHtml ? body : htmlToText(body);
|
|
99
|
+
const { content, truncated } = truncate(rawContent, cap);
|
|
100
|
+
const title = isHtml ? extractTitle(body) : undefined;
|
|
101
|
+
|
|
102
|
+
logger.debug("fetched", {
|
|
103
|
+
url: response.url,
|
|
104
|
+
status: response.statusCode,
|
|
105
|
+
bytes: body.length,
|
|
106
|
+
returned: content.length,
|
|
107
|
+
...(truncated ? { truncated: true } : {}),
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
return {
|
|
111
|
+
url: response.url ?? request.url,
|
|
112
|
+
status: response.statusCode,
|
|
113
|
+
...(contentType ? { contentType } : {}),
|
|
114
|
+
...(title ? { title } : {}),
|
|
115
|
+
content,
|
|
116
|
+
truncated,
|
|
117
|
+
};
|
|
118
|
+
}
|
package/src/plugin.ts
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AppKit plugin (registered name: `web-search`) that owns the resolved
|
|
3
|
+
* web-search runtime - the URL allow-list, result / length caps, timeout,
|
|
4
|
+
* and default approval gate the {@link webSearchTool} / {@link webFetchTool}
|
|
5
|
+
* read. Registering it resolves and logs the effective config (whether an
|
|
6
|
+
* allow-list is active, the caps) so a misconfiguration is visible in the
|
|
7
|
+
* boot logs rather than on the first search.
|
|
8
|
+
*
|
|
9
|
+
* The tools do the actual work when spread into an agent; this plugin primes
|
|
10
|
+
* the shared runtime they reuse and exposes direct {@link runWebSearch} /
|
|
11
|
+
* {@link runWebFetch} for non-agent callers.
|
|
12
|
+
*
|
|
13
|
+
* @module
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { getExecutionContext, Plugin, toPlugin, type PluginManifest } from "@databricks/appkit";
|
|
17
|
+
import { log } from "@dbx-tools/shared-core";
|
|
18
|
+
import { WEB_SEARCH_CONFIG_SCHEMA, type WebSearchPluginConfig } from "./config";
|
|
19
|
+
import { runWebFetch } from "./fetch";
|
|
20
|
+
import { getWebSearchRuntime } from "./runtime";
|
|
21
|
+
import type { WebFetchRequest, WebFetchResult, WebSearchRequest, WebSearchResult } from "./schema";
|
|
22
|
+
import { runWebSearch, type WebSearchContext } from "./search";
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* AppKit plugin that resolves and holds the web-search runtime config used
|
|
26
|
+
* by the `web_search` / `web_fetch` tools.
|
|
27
|
+
*/
|
|
28
|
+
export class WebSearchPlugin extends Plugin<WebSearchPluginConfig> {
|
|
29
|
+
static manifest = {
|
|
30
|
+
name: "web-search",
|
|
31
|
+
displayName: "Web Search",
|
|
32
|
+
description:
|
|
33
|
+
"Searches the web (via duck-duck-scrape) and fetches pages (via got-scraping), " +
|
|
34
|
+
"with an optional URL allow-list and per-tool approval gating.",
|
|
35
|
+
stability: "beta",
|
|
36
|
+
resources: {
|
|
37
|
+
required: [],
|
|
38
|
+
optional: [],
|
|
39
|
+
},
|
|
40
|
+
config: { schema: WEB_SEARCH_CONFIG_SCHEMA },
|
|
41
|
+
} satisfies PluginManifest<"web-search">;
|
|
42
|
+
|
|
43
|
+
private logger = log.logger(this);
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Prime the shared runtime from this plugin's config (over env) and log
|
|
47
|
+
* the effective policy so an active allow-list / caps are obvious at boot.
|
|
48
|
+
*/
|
|
49
|
+
override async setup(): Promise<void> {
|
|
50
|
+
const { config } = getWebSearchRuntime(this.config);
|
|
51
|
+
this.logger.info("ready", {
|
|
52
|
+
model: config.model ?? `fallbacks:[${config.modelFallbacks.join(", ")}]`,
|
|
53
|
+
restricted: config.allowList.restricted,
|
|
54
|
+
...(config.allowList.restricted ? { allowedUrls: config.allowList.patterns } : {}),
|
|
55
|
+
maxCitations: config.maxCitations,
|
|
56
|
+
fetchMaxLength: config.fetchMaxLength,
|
|
57
|
+
approval: config.approval === false ? "none" : config.approval,
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** Resolve the OBO client + host from the active execution context. */
|
|
62
|
+
private async searchContext(): Promise<WebSearchContext> {
|
|
63
|
+
const ctx = getExecutionContext();
|
|
64
|
+
const host = (await ctx.client.config.getHost()).toString();
|
|
65
|
+
return { client: ctx.client, host };
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
override exports() {
|
|
69
|
+
return {
|
|
70
|
+
/**
|
|
71
|
+
* Run a web search directly (bypassing the agent tool). Resolves the
|
|
72
|
+
* OBO client from the active execution context and reads the shared
|
|
73
|
+
* runtime config primed at setup.
|
|
74
|
+
*/
|
|
75
|
+
search: async (request: WebSearchRequest): Promise<WebSearchResult> =>
|
|
76
|
+
runWebSearch(request, getWebSearchRuntime().config, await this.searchContext()),
|
|
77
|
+
/**
|
|
78
|
+
* Fetch one URL directly (bypassing the agent tool). Enforces the
|
|
79
|
+
* configured allow-list. Reads the shared runtime config.
|
|
80
|
+
*/
|
|
81
|
+
fetch: (request: WebFetchRequest): Promise<WebFetchResult> =>
|
|
82
|
+
runWebFetch(request, getWebSearchRuntime().config),
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export const webSearch = toPlugin(WebSearchPlugin);
|
package/src/provider.ts
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Provider detection + web-search tool-spec mapping for the Databricks
|
|
3
|
+
* Model Serving native web-search tool.
|
|
4
|
+
*
|
|
5
|
+
* Databricks exposes web search as a first-party tool that runs *inside* a
|
|
6
|
+
* model call: the model searches the web and folds the results into its
|
|
7
|
+
* answer. The tool spec is provider-specific (see the Databricks docs,
|
|
8
|
+
* `machine-learning/model-serving/web-search`):
|
|
9
|
+
*
|
|
10
|
+
* - OpenAI GPT models, via the Responses API (`/serving-endpoints/responses`):
|
|
11
|
+
* `tools: [{ "type": "web_search" }]`
|
|
12
|
+
* - Google Gemini models, via Chat Completions
|
|
13
|
+
* (`/serving-endpoints/chat/completions`): `tools: [{ "google_search": {} }]`
|
|
14
|
+
*
|
|
15
|
+
* (Anthropic exposes it over MCP, which needs a different call shape; only
|
|
16
|
+
* GPT + Gemini are wired here, matching what the platform supports today.)
|
|
17
|
+
*
|
|
18
|
+
* The provider family is detected from the endpoint id the same way
|
|
19
|
+
* `@dbx-tools/shared-model`'s `classifyByFamily` keys off name substrings
|
|
20
|
+
* (`gpt` / `gemini` / `claude`), so a resolved endpoint like
|
|
21
|
+
* `databricks-gpt-5` or `databricks-gemini-3-pro` maps to its API shape.
|
|
22
|
+
*
|
|
23
|
+
* @module
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
/** A web-search-capable model provider family. */
|
|
27
|
+
export type WebSearchProvider = "openai" | "gemini";
|
|
28
|
+
|
|
29
|
+
/** How a provider's native web-search call is shaped. */
|
|
30
|
+
export interface WebSearchProviderSpec {
|
|
31
|
+
/**
|
|
32
|
+
* Which serving REST surface to call. `"responses"` posts to
|
|
33
|
+
* `/serving-endpoints/responses` (OpenAI Responses API); `"chat"` posts to
|
|
34
|
+
* `/serving-endpoints/chat/completions`.
|
|
35
|
+
*/
|
|
36
|
+
api: "responses" | "chat";
|
|
37
|
+
/** The tool entry appended to the request's `tools` array. */
|
|
38
|
+
tool: Record<string, unknown>;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Built-in provider -> tool-spec map. Operators can override or extend this
|
|
43
|
+
* per provider via the plugin's `webSearchTools` config (env
|
|
44
|
+
* `WEB_SEARCH_TOOLS`), which is merged over these defaults.
|
|
45
|
+
*/
|
|
46
|
+
export const WEB_SEARCH_PROVIDERS: Readonly<Record<WebSearchProvider, WebSearchProviderSpec>> = {
|
|
47
|
+
openai: { api: "responses", tool: { type: "web_search" } },
|
|
48
|
+
gemini: { api: "chat", tool: { google_search: {} } },
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Detect the web-search provider family for an endpoint id, or `null` when
|
|
53
|
+
* the model is not one of the web-search-capable families. Anthropic
|
|
54
|
+
* (`opus`/`sonnet`/`haiku`) is intentionally excluded - it needs an MCP call
|
|
55
|
+
* shape this module doesn't implement - so a Claude endpoint returns `null`
|
|
56
|
+
* and is treated as unsupported.
|
|
57
|
+
*/
|
|
58
|
+
export function detectWebSearchProvider(modelId: string): WebSearchProvider | null {
|
|
59
|
+
const n = modelId.toLowerCase();
|
|
60
|
+
// gpt-oss open-weights don't carry the hosted web-search tool; only the
|
|
61
|
+
// hosted GPT family (Responses API) does. Both contain "gpt", so exclude
|
|
62
|
+
// the open-weights explicitly.
|
|
63
|
+
if (n.includes("gpt") && !n.includes("gpt-oss")) return "openai";
|
|
64
|
+
if (n.includes("gemini")) return "gemini";
|
|
65
|
+
return null;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** Whether `modelId` is a web-search-capable model. */
|
|
69
|
+
export function supportsWebSearch(modelId: string): boolean {
|
|
70
|
+
return detectWebSearchProvider(modelId) !== null;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Resolve the effective {@link WebSearchProviderSpec} for a provider: the
|
|
75
|
+
* built-in default, with any operator override (the `webSearchTools` map,
|
|
76
|
+
* keyed by provider) shallow-merged over it. An override may replace just the
|
|
77
|
+
* `tool` (the common case - a new tool type) or also the `api`.
|
|
78
|
+
*/
|
|
79
|
+
export function webSearchToolSpec(
|
|
80
|
+
provider: WebSearchProvider,
|
|
81
|
+
overrides?: Record<string, unknown>,
|
|
82
|
+
): WebSearchProviderSpec {
|
|
83
|
+
const base = WEB_SEARCH_PROVIDERS[provider];
|
|
84
|
+
const override = overrides?.[provider] as Partial<WebSearchProviderSpec> | undefined;
|
|
85
|
+
if (!override) return base;
|
|
86
|
+
return {
|
|
87
|
+
api: override.api ?? base.api,
|
|
88
|
+
tool: override.tool ?? base.tool,
|
|
89
|
+
};
|
|
90
|
+
}
|
package/src/runtime.ts
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The web-search runtime: a lazily-resolved, process-wide config shared by
|
|
3
|
+
* the plugin and the `web_search` / `web_fetch` tools, so both read one
|
|
4
|
+
* resolved allow-list / cap / timeout set. The first caller (normally the
|
|
5
|
+
* plugin at setup) primes it from the plugin's config; later callers (the
|
|
6
|
+
* tools' `execute`) reuse it.
|
|
7
|
+
*
|
|
8
|
+
* Unlike the email runtime there is no connection to pool - the backend is
|
|
9
|
+
* stateless HTTP per call - so the runtime holds only the resolved config.
|
|
10
|
+
*
|
|
11
|
+
* @module
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { resolveWebSearchConfig, type ResolvedWebSearchConfig, type WebSearchPluginConfig } from "./config";
|
|
15
|
+
|
|
16
|
+
/** The shared resolved config. */
|
|
17
|
+
export interface WebSearchRuntime {
|
|
18
|
+
config: ResolvedWebSearchConfig;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
let runtime: WebSearchRuntime | undefined;
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Return the shared runtime, building it on first use from the supplied
|
|
25
|
+
* config layered over environment defaults. Overrides are only read when the
|
|
26
|
+
* runtime is first created, so prime it from the plugin's config at setup;
|
|
27
|
+
* subsequent calls (the tools' `execute`) pass nothing and get the same
|
|
28
|
+
* instance.
|
|
29
|
+
*/
|
|
30
|
+
export function getWebSearchRuntime(overrides?: WebSearchPluginConfig): WebSearchRuntime {
|
|
31
|
+
if (!runtime) {
|
|
32
|
+
runtime = { config: resolveWebSearchConfig(overrides) };
|
|
33
|
+
}
|
|
34
|
+
return runtime;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** Drop the memoized runtime so the next {@link getWebSearchRuntime} rebuilds it. */
|
|
38
|
+
export function resetWebSearchRuntime(): void {
|
|
39
|
+
runtime = undefined;
|
|
40
|
+
}
|