@dbx-tools/appkit-web-search 0.3.28 → 0.3.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +128 -30
- package/index.ts +5 -2
- package/package.json +6 -6
- package/src/allowlist.ts +7 -12
- package/src/config.ts +226 -56
- package/src/defaults.ts +127 -0
- package/src/fetch.ts +49 -54
- package/src/html-text.ts +72 -0
- package/src/plugin.ts +219 -33
- package/src/provider.ts +28 -5
- package/src/runtime.ts +87 -3
- package/src/schema.ts +22 -0
- package/src/scrape.ts +35 -27
- package/src/search.ts +147 -56
- package/src/tool.ts +44 -45
- package/test/allowlist.test.ts +70 -4
package/src/config.ts
CHANGED
|
@@ -13,20 +13,40 @@
|
|
|
13
13
|
* {@link WebSearchPluginConfig.modelFallbacks} order (Gemini, then GPT, then a
|
|
14
14
|
* repo floor) picks the first web-search-capable endpoint that exists.
|
|
15
15
|
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
* built-in default.
|
|
16
|
+
* Which endpoint id is used follows the standard precedence - explicit plugin
|
|
17
|
+
* config, then environment, then a default - with two environment sources:
|
|
19
18
|
*
|
|
20
|
-
*
|
|
21
|
-
* `
|
|
19
|
+
* 1. `model` in plugin config;
|
|
20
|
+
* 2. `WEB_SEARCH_MODEL`, the dedicated override, so a deployment can point
|
|
21
|
+
* web search at a different endpoint than the agent's chat model;
|
|
22
|
+
* 3. `DATABRICKS_SERVING_ENDPOINT_NAME`, AppKit's standard name for a
|
|
23
|
+
* Model Serving binding, honored so the resource declared in the manifest
|
|
24
|
+
* wires this plugin up like any other. Because that binding is shared
|
|
25
|
+
* with whatever else the app serves, it is treated as a preference: an
|
|
26
|
+
* endpoint that cannot run the native web-search tool is skipped in
|
|
27
|
+
* favor of {@link WebSearchPluginConfig.modelFallbacks} rather than
|
|
28
|
+
* failing the call. A pin from (1) or (2) is explicit and DOES fail;
|
|
29
|
+
* 4. otherwise the fallback order, resolved against the live catalogue.
|
|
30
|
+
*
|
|
31
|
+
* Which model wins is decided lazily, at call time, against the live
|
|
32
|
+
* catalogue. Resolution here is eager about everything else and fails loudly
|
|
33
|
+
* on a contradiction: an unparseable `WEB_SEARCH_TOOLS`, an unknown
|
|
34
|
+
* {@link UrlPolicyMode}, or a URL policy that disagrees with the allow-list it
|
|
35
|
+
* was given.
|
|
36
|
+
*
|
|
37
|
+
* Env fallbacks: `WEB_SEARCH_MODEL`, `DATABRICKS_SERVING_ENDPOINT_NAME`,
|
|
38
|
+
* `WEB_SEARCH_MODEL_FALLBACKS`, `WEB_SEARCH_TOOLS` (JSON),
|
|
39
|
+
* `WEB_SEARCH_URL_POLICY`, `WEB_SEARCH_ALLOWED_URLS`,
|
|
22
40
|
* `WEB_SEARCH_MAX_CITATIONS`, `WEB_SEARCH_FETCH_MAX_LENGTH`,
|
|
23
|
-
* `WEB_SEARCH_TIMEOUT_MS`, `
|
|
41
|
+
* `WEB_SEARCH_TIMEOUT_MS`, `WEB_SEARCH_SCRAPE_FALLBACK`, `WEB_SEARCH_FUZZY`,
|
|
42
|
+
* `WEB_SEARCH_FUZZY_THRESHOLD`.
|
|
24
43
|
*
|
|
25
44
|
* @module
|
|
26
45
|
*/
|
|
27
46
|
|
|
28
|
-
import type
|
|
29
|
-
import {
|
|
47
|
+
import { ConfigurationError, type BasePluginConfig } from "@databricks/appkit";
|
|
48
|
+
import { serving } from "@dbx-tools/model";
|
|
49
|
+
import { json, object, type OneOrMany, string } from "@dbx-tools/shared-core";
|
|
30
50
|
import type { JSONSchema7 } from "json-schema";
|
|
31
51
|
import { parseAllowedUrls, toUrlAllowList, type UrlAllowList } from "./allowlist";
|
|
32
52
|
|
|
@@ -35,10 +55,37 @@ import { parseAllowedUrls, toUrlAllowList, type UrlAllowList } from "./allowlist
|
|
|
35
55
|
* pattern (or list of patterns, in the {@link OneOrMany} shape used across
|
|
36
56
|
* the repo) gates only calls whose URL matches. Patterns use the same glob
|
|
37
57
|
* syntax as the allow-list (see `allowlist.ts`). Omit / `false` for no
|
|
38
|
-
* approval.
|
|
58
|
+
* approval. Normalized to an {@link ApprovalPolicy} before use.
|
|
39
59
|
*/
|
|
40
60
|
export type ApprovalGate = boolean | OneOrMany<string> | string;
|
|
41
61
|
|
|
62
|
+
/**
|
|
63
|
+
* The normalized form of an {@link ApprovalGate}: which calls pause for a
|
|
64
|
+
* human. `"none"` runs every call straight through, `"always"` gates all of
|
|
65
|
+
* them, and `"urls"` gates only calls whose URL matches one of `patterns`.
|
|
66
|
+
*/
|
|
67
|
+
export type ApprovalPolicy =
|
|
68
|
+
| { readonly mode: "none" }
|
|
69
|
+
| { readonly mode: "always" }
|
|
70
|
+
| { readonly mode: "urls"; readonly patterns: readonly string[] };
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Which URLs the tools may reach. `"allowlist"` permits only the configured
|
|
74
|
+
* entries; `"unrestricted"` names the permissive mode explicitly, so an
|
|
75
|
+
* unrestricted deployment is a stated choice visible in the boot log rather
|
|
76
|
+
* than an empty list nobody noticed.
|
|
77
|
+
*/
|
|
78
|
+
export type UrlPolicyMode = "unrestricted" | "allowlist";
|
|
79
|
+
|
|
80
|
+
/** Dedicated override for the web-search endpoint id. */
|
|
81
|
+
export const MODEL_ENV = "WEB_SEARCH_MODEL";
|
|
82
|
+
|
|
83
|
+
/** AppKit's standard environment name for a Model Serving endpoint binding. */
|
|
84
|
+
export const SERVING_ENDPOINT_ENV = "DATABRICKS_SERVING_ENDPOINT_NAME";
|
|
85
|
+
|
|
86
|
+
/** Where the pinned web-search endpoint id came from, or `"none"` when unpinned. */
|
|
87
|
+
export type ModelSource = "config" | typeof MODEL_ENV | typeof SERVING_ENDPOINT_ENV | "none";
|
|
88
|
+
|
|
42
89
|
/**
|
|
43
90
|
* Default web-search model preference, tried in order when no model is
|
|
44
91
|
* pinned. Gemini first, then GPT - both support the native web-search tool;
|
|
@@ -68,8 +115,9 @@ export interface WebSearchPluginConfig extends BasePluginConfig {
|
|
|
68
115
|
* The web-search model to use by default: a Databricks serving endpoint
|
|
69
116
|
* name (`"databricks-gemini-3-pro"`), a loose name (`"gemini"`, `"gpt"`),
|
|
70
117
|
* or a capability class. Fuzzy-matched against the live catalogue. Falls
|
|
71
|
-
* back to `WEB_SEARCH_MODEL`, then
|
|
72
|
-
* independently of the calling
|
|
118
|
+
* back to `WEB_SEARCH_MODEL`, then `DATABRICKS_SERVING_ENDPOINT_NAME`, then
|
|
119
|
+
* the {@link modelFallbacks} order. Chosen independently of the calling
|
|
120
|
+
* agent's chat model.
|
|
73
121
|
*/
|
|
74
122
|
model?: string;
|
|
75
123
|
/**
|
|
@@ -93,7 +141,7 @@ export interface WebSearchPluginConfig extends BasePluginConfig {
|
|
|
93
141
|
* Defaults to `true`; falls back to `WEB_SEARCH_FUZZY`.
|
|
94
142
|
*/
|
|
95
143
|
modelFuzzyMatch?: boolean;
|
|
96
|
-
/** Fuse.js fuzzy threshold. Falls back to `WEB_SEARCH_FUZZY_THRESHOLD`, then
|
|
144
|
+
/** Fuse.js fuzzy threshold. Falls back to `WEB_SEARCH_FUZZY_THRESHOLD`, then the shared default. */
|
|
97
145
|
modelFuzzyThreshold?: number;
|
|
98
146
|
/**
|
|
99
147
|
* Hard cap on the number of citations a single search returns. Falls back
|
|
@@ -119,6 +167,15 @@ export interface WebSearchPluginConfig extends BasePluginConfig {
|
|
|
119
167
|
* require a native model and error otherwise.
|
|
120
168
|
*/
|
|
121
169
|
scrapeFallback?: boolean;
|
|
170
|
+
/**
|
|
171
|
+
* Which URLs the tools may reach ({@link UrlPolicyMode}). Falls back to
|
|
172
|
+
* `WEB_SEARCH_URL_POLICY`, then to `"allowlist"` when {@link allowedUrls}
|
|
173
|
+
* has entries and `"unrestricted"` when it does not. Naming the mode
|
|
174
|
+
* explicitly is the way to state that an open deployment is intended;
|
|
175
|
+
* `"allowlist"` with no entries, or `"unrestricted"` alongside entries, is
|
|
176
|
+
* a contradiction and fails at resolution.
|
|
177
|
+
*/
|
|
178
|
+
urlPolicy?: UrlPolicyMode;
|
|
122
179
|
/**
|
|
123
180
|
* Optional URL allow-list. Each entry is a glob (or bare host) tested
|
|
124
181
|
* against a URL's full `href`. When set, `web_search` silently filters
|
|
@@ -131,15 +188,18 @@ export interface WebSearchPluginConfig extends BasePluginConfig {
|
|
|
131
188
|
/**
|
|
132
189
|
* Approval gate applied to BOTH tools (per-tool overrides via
|
|
133
190
|
* {@link WebSearchToolOptions.approval} win). `true` gates every call;
|
|
134
|
-
* a URL-pattern (or list) gates only matching calls
|
|
191
|
+
* a URL-pattern (or list) gates only matching calls; an
|
|
192
|
+
* {@link ApprovalPolicy} states the mode directly. Omit for no approval.
|
|
135
193
|
*/
|
|
136
|
-
approval?: ApprovalGate;
|
|
194
|
+
approval?: ApprovalGate | ApprovalPolicy;
|
|
137
195
|
}
|
|
138
196
|
|
|
139
197
|
/** Concrete, validated config the runtime + tools read. */
|
|
140
198
|
export interface ResolvedWebSearchConfig {
|
|
141
199
|
/** Pinned web-search model, when configured (else undefined - use fallbacks). */
|
|
142
200
|
model?: string;
|
|
201
|
+
/** Where {@link model} came from, which decides whether an unsupported pin is fatal. */
|
|
202
|
+
modelSource: ModelSource;
|
|
143
203
|
/** Ordered fallback model candidates (Gemini, then GPT, then a floor). */
|
|
144
204
|
modelFallbacks: readonly string[];
|
|
145
205
|
/** Provider -> tool-spec override map, merged over the built-in defaults. */
|
|
@@ -153,10 +213,12 @@ export interface ResolvedWebSearchConfig {
|
|
|
153
213
|
timeoutMs: number;
|
|
154
214
|
/** Whether to scrape-fallback when no native web-search model is deployed. */
|
|
155
215
|
scrapeFallback: boolean;
|
|
156
|
-
/**
|
|
216
|
+
/** The named URL policy in force. */
|
|
217
|
+
urlPolicy: UrlPolicyMode;
|
|
218
|
+
/** Compiled allow-list (permit-all under the `unrestricted` policy). */
|
|
157
219
|
allowList: UrlAllowList;
|
|
158
|
-
/** Default per-tool approval
|
|
159
|
-
approval:
|
|
220
|
+
/** Default per-tool approval policy. */
|
|
221
|
+
approval: ApprovalPolicy;
|
|
160
222
|
}
|
|
161
223
|
|
|
162
224
|
/** JSON Schema published on the manifest's `config.schema`. */
|
|
@@ -166,7 +228,7 @@ export const WEB_SEARCH_CONFIG_SCHEMA: JSONSchema7 = {
|
|
|
166
228
|
model: {
|
|
167
229
|
type: "string",
|
|
168
230
|
description:
|
|
169
|
-
"Default web-search model (endpoint name, loose name, or class). Fuzzy-matched. Env: WEB_SEARCH_MODEL.",
|
|
231
|
+
"Default web-search model (endpoint name, loose name, or class). Fuzzy-matched. Env: WEB_SEARCH_MODEL, then DATABRICKS_SERVING_ENDPOINT_NAME.",
|
|
170
232
|
},
|
|
171
233
|
modelFallbacks: {
|
|
172
234
|
type: "array",
|
|
@@ -179,6 +241,16 @@ export const WEB_SEARCH_CONFIG_SCHEMA: JSONSchema7 = {
|
|
|
179
241
|
description:
|
|
180
242
|
'Provider -> tool-spec override map merged over the built-in defaults (openai -> {"type":"web_search"}, gemini -> {"google_search":{}}). Env: WEB_SEARCH_TOOLS (JSON).',
|
|
181
243
|
},
|
|
244
|
+
modelFuzzyMatch: {
|
|
245
|
+
type: "boolean",
|
|
246
|
+
description:
|
|
247
|
+
"Fuzzy-match loose model names against the live catalogue. Default true (env: WEB_SEARCH_FUZZY).",
|
|
248
|
+
},
|
|
249
|
+
modelFuzzyThreshold: {
|
|
250
|
+
type: "number",
|
|
251
|
+
description:
|
|
252
|
+
"Fuse.js score threshold below which a fuzzy model match is accepted (env: WEB_SEARCH_FUZZY_THRESHOLD).",
|
|
253
|
+
},
|
|
182
254
|
maxCitations: {
|
|
183
255
|
type: "number",
|
|
184
256
|
description: "Hard cap on citations returned (env: WEB_SEARCH_MAX_CITATIONS).",
|
|
@@ -191,28 +263,43 @@ export const WEB_SEARCH_CONFIG_SCHEMA: JSONSchema7 = {
|
|
|
191
263
|
type: "number",
|
|
192
264
|
description: "Per-request network timeout in ms (env: WEB_SEARCH_TIMEOUT_MS).",
|
|
193
265
|
},
|
|
266
|
+
scrapeFallback: {
|
|
267
|
+
type: "boolean",
|
|
268
|
+
description:
|
|
269
|
+
"Fall back to a DuckDuckGo scrape when no web-search-capable model is deployed. Default true (env: WEB_SEARCH_SCRAPE_FALLBACK).",
|
|
270
|
+
},
|
|
271
|
+
urlPolicy: {
|
|
272
|
+
type: "string",
|
|
273
|
+
enum: ["unrestricted", "allowlist"],
|
|
274
|
+
description:
|
|
275
|
+
"Which URLs the tools may reach. Defaults to allowlist when allowedUrls has entries, else unrestricted (env: WEB_SEARCH_URL_POLICY).",
|
|
276
|
+
},
|
|
194
277
|
allowedUrls: {
|
|
195
278
|
type: "array",
|
|
196
279
|
items: { type: "string" },
|
|
197
280
|
description:
|
|
198
281
|
'URL allow-list of globs / bare hosts (e.g. "*.databricks.com", "docs.example.com"). Also accepts a comma/space-separated string. Falls back to WEB_SEARCH_ALLOWED_URLS. Empty = unrestricted.',
|
|
199
282
|
},
|
|
283
|
+
approval: {
|
|
284
|
+
description:
|
|
285
|
+
'Approval gate for both tools: true gates every call, a glob (or list of globs) gates only matching URLs, or state the mode directly as {"mode":"none"|"always"|"urls"}. Default none.',
|
|
286
|
+
oneOf: [
|
|
287
|
+
{ type: "boolean" },
|
|
288
|
+
{ type: "string" },
|
|
289
|
+
{ type: "array", items: { type: "string" } },
|
|
290
|
+
{
|
|
291
|
+
type: "object",
|
|
292
|
+
properties: {
|
|
293
|
+
mode: { type: "string", enum: ["none", "always", "urls"] },
|
|
294
|
+
patterns: { type: "array", items: { type: "string" } },
|
|
295
|
+
},
|
|
296
|
+
required: ["mode"],
|
|
297
|
+
},
|
|
298
|
+
],
|
|
299
|
+
},
|
|
200
300
|
},
|
|
201
301
|
};
|
|
202
302
|
|
|
203
|
-
/** Split a CSV / whitespace / array value into a trimmed, non-empty string list. */
|
|
204
|
-
function toStringList(raw: string | string[] | undefined): string[] {
|
|
205
|
-
const entries = typeof raw === "string" ? raw.split(/[\s,]+/) : Array.isArray(raw) ? raw : [];
|
|
206
|
-
return [
|
|
207
|
-
...object
|
|
208
|
-
.sequence(entries)
|
|
209
|
-
.map((e) => e.trim())
|
|
210
|
-
.filter((e) => e.length > 0)
|
|
211
|
-
.distinct()
|
|
212
|
-
.toArray(),
|
|
213
|
-
];
|
|
214
|
-
}
|
|
215
|
-
|
|
216
303
|
/** Parse a positive integer env/config value, else the fallback. */
|
|
217
304
|
function resolvePositiveInt(value: number | undefined, envKey: string, fallback: number): number {
|
|
218
305
|
if (typeof value === "number" && Number.isFinite(value) && value > 0) return Math.floor(value);
|
|
@@ -220,39 +307,118 @@ function resolvePositiveInt(value: number | undefined, envKey: string, fallback:
|
|
|
220
307
|
return Number.isFinite(env) && env > 0 ? Math.floor(env) : fallback;
|
|
221
308
|
}
|
|
222
309
|
|
|
223
|
-
/**
|
|
310
|
+
/**
|
|
311
|
+
* Parse the `WEB_SEARCH_TOOLS` env var (JSON object), else `{}`. A value that
|
|
312
|
+
* is set but not a JSON object is a deployment mistake that would otherwise
|
|
313
|
+
* silently leave the built-in tool specs in place, so it throws.
|
|
314
|
+
*/
|
|
224
315
|
function parseToolsEnv(): Record<string, unknown> {
|
|
225
|
-
const raw = process.env.WEB_SEARCH_TOOLS;
|
|
226
|
-
if (
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
316
|
+
const raw = string.trimToNull(process.env.WEB_SEARCH_TOOLS);
|
|
317
|
+
if (raw === null) return {};
|
|
318
|
+
const parsed = json.parseRecord(raw);
|
|
319
|
+
if (!parsed) {
|
|
320
|
+
throw new ConfigurationError(
|
|
321
|
+
"WEB_SEARCH_TOOLS must be a JSON object mapping a provider family to its tool spec",
|
|
322
|
+
{ context: { envVar: "WEB_SEARCH_TOOLS" } },
|
|
323
|
+
);
|
|
324
|
+
}
|
|
325
|
+
return parsed;
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/** Read the pinned endpoint id and record which source supplied it. */
|
|
329
|
+
function resolveModelPin(config: WebSearchPluginConfig): { model?: string; source: ModelSource } {
|
|
330
|
+
const fromConfig = string.trimToNull(config.model);
|
|
331
|
+
if (fromConfig !== null) return { model: fromConfig, source: "config" };
|
|
332
|
+
const fromModelEnv = string.trimToNull(process.env[MODEL_ENV]);
|
|
333
|
+
if (fromModelEnv !== null) return { model: fromModelEnv, source: MODEL_ENV };
|
|
334
|
+
const fromResourceEnv = string.trimToNull(process.env[SERVING_ENDPOINT_ENV]);
|
|
335
|
+
if (fromResourceEnv !== null) return { model: fromResourceEnv, source: SERVING_ENDPOINT_ENV };
|
|
336
|
+
return { source: "none" };
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
/**
|
|
340
|
+
* Resolve the named {@link UrlPolicyMode}, failing on a stated mode that
|
|
341
|
+
* contradicts the allow-list entries it was given.
|
|
342
|
+
*/
|
|
343
|
+
function resolveUrlPolicy(
|
|
344
|
+
configured: UrlPolicyMode | undefined,
|
|
345
|
+
patterns: readonly string[],
|
|
346
|
+
): UrlPolicyMode {
|
|
347
|
+
const raw = configured ?? string.trimToNull(process.env.WEB_SEARCH_URL_POLICY) ?? undefined;
|
|
348
|
+
if (raw === undefined) return patterns.length > 0 ? "allowlist" : "unrestricted";
|
|
349
|
+
if (raw !== "allowlist" && raw !== "unrestricted") {
|
|
350
|
+
throw new ConfigurationError(
|
|
351
|
+
'urlPolicy must be "allowlist" or "unrestricted" (env: WEB_SEARCH_URL_POLICY)',
|
|
352
|
+
{ context: { field: "urlPolicy", envVar: "WEB_SEARCH_URL_POLICY" } },
|
|
353
|
+
);
|
|
354
|
+
}
|
|
355
|
+
if (raw === "allowlist" && patterns.length === 0) {
|
|
356
|
+
throw new ConfigurationError(
|
|
357
|
+
'urlPolicy "allowlist" needs at least one entry in allowedUrls (env: WEB_SEARCH_ALLOWED_URLS)',
|
|
358
|
+
{ context: { field: "allowedUrls", envVar: "WEB_SEARCH_ALLOWED_URLS" } },
|
|
359
|
+
);
|
|
232
360
|
}
|
|
361
|
+
if (raw === "unrestricted" && patterns.length > 0) {
|
|
362
|
+
throw new ConfigurationError(
|
|
363
|
+
'urlPolicy "unrestricted" cannot be combined with allowedUrls entries; drop the entries or set urlPolicy to "allowlist"',
|
|
364
|
+
{ context: { field: "allowedUrls", envVar: "WEB_SEARCH_ALLOWED_URLS" } },
|
|
365
|
+
);
|
|
366
|
+
}
|
|
367
|
+
return raw;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
/**
|
|
371
|
+
* Normalize any accepted approval spelling into an {@link ApprovalPolicy}.
|
|
372
|
+
* A pattern gate with no usable patterns collapses to `"none"` rather than
|
|
373
|
+
* gating everything, so a blank env var can never wedge the tools behind an
|
|
374
|
+
* approval nobody configured.
|
|
375
|
+
*/
|
|
376
|
+
export function toApprovalPolicy(gate: ApprovalGate | ApprovalPolicy | undefined): ApprovalPolicy {
|
|
377
|
+
if (gate === undefined || gate === false) return { mode: "none" };
|
|
378
|
+
if (gate === true) return { mode: "always" };
|
|
379
|
+
if (object.isRecord(gate) && "mode" in gate) {
|
|
380
|
+
if (gate.mode === "urls") {
|
|
381
|
+
const patterns = parseAllowedUrls([...gate.patterns]);
|
|
382
|
+
return patterns.length > 0 ? { mode: "urls", patterns } : { mode: "none" };
|
|
383
|
+
}
|
|
384
|
+
if (gate.mode === "always" || gate.mode === "none") return { mode: gate.mode };
|
|
385
|
+
throw new ConfigurationError('approval mode must be "none", "always" or "urls"', {
|
|
386
|
+
context: { field: "approval" },
|
|
387
|
+
});
|
|
388
|
+
}
|
|
389
|
+
const patterns = parseAllowedUrls(typeof gate === "string" ? gate : [...gate]);
|
|
390
|
+
return patterns.length > 0 ? { mode: "urls", patterns } : { mode: "none" };
|
|
233
391
|
}
|
|
234
392
|
|
|
235
393
|
/**
|
|
236
394
|
* Resolve plugin config over environment defaults into the concrete
|
|
237
|
-
* {@link ResolvedWebSearchConfig}.
|
|
238
|
-
*
|
|
239
|
-
*
|
|
395
|
+
* {@link ResolvedWebSearchConfig}. Which model is used stays lazy - it is
|
|
396
|
+
* picked at call time against the live catalogue - but everything else is
|
|
397
|
+
* settled here, and a contradiction (unparseable `WEB_SEARCH_TOOLS`, an
|
|
398
|
+
* unknown URL policy, a policy that disagrees with its allow-list) throws a
|
|
399
|
+
* {@link ConfigurationError} naming the field and the environment variable.
|
|
240
400
|
*/
|
|
241
401
|
export function resolveWebSearchConfig(
|
|
242
402
|
config: WebSearchPluginConfig = {},
|
|
243
403
|
): ResolvedWebSearchConfig {
|
|
244
404
|
const patterns = parseAllowedUrls(config.allowedUrls ?? process.env.WEB_SEARCH_ALLOWED_URLS);
|
|
245
|
-
const
|
|
246
|
-
const
|
|
405
|
+
const urlPolicy = resolveUrlPolicy(config.urlPolicy, patterns);
|
|
406
|
+
const { model, source } = resolveModelPin(config);
|
|
407
|
+
const fallbacks = string.parseList(
|
|
408
|
+
config.modelFallbacks ?? process.env.WEB_SEARCH_MODEL_FALLBACKS,
|
|
409
|
+
);
|
|
247
410
|
const fuzzyThresholdRaw =
|
|
248
411
|
config.modelFuzzyThreshold ?? Number(process.env.WEB_SEARCH_FUZZY_THRESHOLD);
|
|
249
412
|
return {
|
|
250
413
|
...(model ? { model } : {}),
|
|
414
|
+
modelSource: source,
|
|
251
415
|
modelFallbacks: fallbacks.length > 0 ? fallbacks : DEFAULT_MODEL_FALLBACKS,
|
|
252
416
|
webSearchTools: { ...parseToolsEnv(), ...(config.webSearchTools ?? {}) },
|
|
253
417
|
fuzzy: config.modelFuzzyMatch ?? object.toBoolean(process.env.WEB_SEARCH_FUZZY) ?? true,
|
|
254
418
|
fuzzyThreshold:
|
|
255
|
-
Number.isFinite(fuzzyThresholdRaw) && fuzzyThresholdRaw
|
|
419
|
+
Number.isFinite(fuzzyThresholdRaw) && fuzzyThresholdRaw
|
|
420
|
+
? Number(fuzzyThresholdRaw)
|
|
421
|
+
: serving.DEFAULT_FUZZY_THRESHOLD,
|
|
256
422
|
maxCitations: resolvePositiveInt(
|
|
257
423
|
config.maxCitations,
|
|
258
424
|
"WEB_SEARCH_MAX_CITATIONS",
|
|
@@ -266,22 +432,26 @@ export function resolveWebSearchConfig(
|
|
|
266
432
|
timeoutMs: resolvePositiveInt(config.timeoutMs, "WEB_SEARCH_TIMEOUT_MS", DEFAULT_TIMEOUT_MS),
|
|
267
433
|
scrapeFallback:
|
|
268
434
|
config.scrapeFallback ?? object.toBoolean(process.env.WEB_SEARCH_SCRAPE_FALLBACK) ?? true,
|
|
269
|
-
|
|
270
|
-
|
|
435
|
+
urlPolicy,
|
|
436
|
+
allowList: toUrlAllowList(urlPolicy === "allowlist" ? patterns : []),
|
|
437
|
+
approval: toApprovalPolicy(config.approval),
|
|
271
438
|
};
|
|
272
439
|
}
|
|
273
440
|
|
|
274
441
|
/**
|
|
275
|
-
* Resolve an
|
|
276
|
-
*
|
|
277
|
-
* when ANY candidate matches. Empty candidates with a pattern gate
|
|
278
|
-
* match (nothing to approve). Reuses the allow-list matcher so approval
|
|
442
|
+
* Resolve an approval gate against a set of candidate URLs into a concrete
|
|
443
|
+
* boolean: `"none"` / `"always"` pass straight through; a `"urls"` policy
|
|
444
|
+
* gates when ANY candidate matches. Empty candidates with a pattern gate
|
|
445
|
+
* never match (nothing to approve). Reuses the allow-list matcher so approval
|
|
279
446
|
* globs read exactly like allow-list globs.
|
|
280
447
|
*/
|
|
281
|
-
export function approvalMatches(
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
const
|
|
448
|
+
export function approvalMatches(
|
|
449
|
+
gate: ApprovalGate | ApprovalPolicy,
|
|
450
|
+
urls: readonly string[],
|
|
451
|
+
): boolean {
|
|
452
|
+
const policy = toApprovalPolicy(gate);
|
|
453
|
+
if (policy.mode === "none") return false;
|
|
454
|
+
if (policy.mode === "always") return true;
|
|
455
|
+
const list = toUrlAllowList(policy.patterns);
|
|
286
456
|
return urls.some((url) => list.allows(url));
|
|
287
457
|
}
|
package/src/defaults.ts
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Interceptor settings for the add-on's three outbound calls: the native
|
|
3
|
+
* web-search POST to Model Serving, the `web_fetch` page read, and the
|
|
4
|
+
* DuckDuckGo scrape fallback. They live here, in the sibling `defaults.ts`
|
|
5
|
+
* AppKit plugins keep, so a TTL or retry budget is tuned in one place instead
|
|
6
|
+
* of at a call site.
|
|
7
|
+
*
|
|
8
|
+
* All three are idempotent READS: none of them create or mutate anything, so
|
|
9
|
+
* both caching and retrying are safe. Each constant below records why its
|
|
10
|
+
* numbers were chosen.
|
|
11
|
+
*
|
|
12
|
+
* @module
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* The `PluginExecuteConfig` slice this package sets. Mirrored structurally
|
|
17
|
+
* because AppKit's `PluginExecuteConfig` lives behind a subpath its `exports`
|
|
18
|
+
* map does not publish, so the nominal type cannot be imported.
|
|
19
|
+
*/
|
|
20
|
+
export type WebSearchExecuteConfig = {
|
|
21
|
+
cache?: { enabled?: boolean; ttl?: number; cacheKey?: (string | number | object)[] };
|
|
22
|
+
retry?: { enabled?: boolean; attempts?: number; initialDelay?: number; maxDelay?: number };
|
|
23
|
+
timeout?: number;
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* The `PluginExecutionSettings` shape accepted by AppKit's `Plugin.execute()`.
|
|
28
|
+
* Mirrored structurally for the same reason as {@link WebSearchExecuteConfig}.
|
|
29
|
+
*/
|
|
30
|
+
export type WebSearchExecutionSettings = {
|
|
31
|
+
default: WebSearchExecuteConfig;
|
|
32
|
+
user?: WebSearchExecuteConfig;
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Cache lifetime for a native web-search answer. Long enough that a model
|
|
37
|
+
* re-asking the same question inside one conversation does not pay for a
|
|
38
|
+
* second serving call, short enough that "current information" stays current.
|
|
39
|
+
*/
|
|
40
|
+
export const SEARCH_CACHE_TTL_SECONDS = 300;
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Cache lifetime for a fetched page. Pages move more slowly than search
|
|
44
|
+
* results, and an agent commonly re-reads the same source across turns.
|
|
45
|
+
*/
|
|
46
|
+
export const FETCH_CACHE_TTL_SECONDS = 900;
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Attempts for a serving call. Model Serving sheds load with 429 / 503 under
|
|
50
|
+
* contention; three attempts with AppKit's exponential backoff and full jitter
|
|
51
|
+
* clears that without turning a genuine outage into a long stall.
|
|
52
|
+
*/
|
|
53
|
+
export const SERVING_RETRY_ATTEMPTS = 3;
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Attempts for an open-web request. Kept lower than the serving budget: a
|
|
57
|
+
* remote site that refuses once usually refuses again, and the caller is
|
|
58
|
+
* waiting on an interactive tool call.
|
|
59
|
+
*/
|
|
60
|
+
export const WEB_RETRY_ATTEMPTS = 2;
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Settings for the native web-search POST to Model Serving.
|
|
64
|
+
*
|
|
65
|
+
* Cache enabled: the POST is a pure read of the model's answer for one query,
|
|
66
|
+
* and repeat asks of the same question are common in a multi-turn agent.
|
|
67
|
+
* Retry enabled: the POST creates no conversation or state, so replaying it
|
|
68
|
+
* after a transient 429 / 5xx is safe.
|
|
69
|
+
*/
|
|
70
|
+
export const webSearchExecuteDefaults: WebSearchExecutionSettings = {
|
|
71
|
+
default: {
|
|
72
|
+
cache: { enabled: true, ttl: SEARCH_CACHE_TTL_SECONDS },
|
|
73
|
+
retry: { enabled: true, attempts: SERVING_RETRY_ATTEMPTS },
|
|
74
|
+
},
|
|
75
|
+
};
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Settings for a `web_fetch` page read.
|
|
79
|
+
*
|
|
80
|
+
* Cache enabled: fetching a URL is a read, and an agent that searches then
|
|
81
|
+
* reads its own citations hits the same page repeatedly.
|
|
82
|
+
* Retry enabled: a GET is idempotent, so a dropped connection is worth one
|
|
83
|
+
* more attempt.
|
|
84
|
+
*/
|
|
85
|
+
export const webFetchExecuteDefaults: WebSearchExecutionSettings = {
|
|
86
|
+
default: {
|
|
87
|
+
cache: { enabled: true, ttl: FETCH_CACHE_TTL_SECONDS },
|
|
88
|
+
retry: { enabled: true, attempts: WEB_RETRY_ATTEMPTS },
|
|
89
|
+
},
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Settings for the DuckDuckGo scrape fallback. Same read semantics as the
|
|
94
|
+
* native search, and the shorter cache lifetime applies for the same reason,
|
|
95
|
+
* but the retry budget follows the open-web one: DuckDuckGo answers a repeat
|
|
96
|
+
* request with a bot challenge rather than results.
|
|
97
|
+
*/
|
|
98
|
+
export const scrapeSearchExecuteDefaults: WebSearchExecutionSettings = {
|
|
99
|
+
default: {
|
|
100
|
+
cache: { enabled: true, ttl: SEARCH_CACHE_TTL_SECONDS },
|
|
101
|
+
retry: { enabled: true, attempts: WEB_RETRY_ATTEMPTS },
|
|
102
|
+
},
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Bind one of the named defaults to a single call: the configured network
|
|
107
|
+
* timeout and the cache key for this request.
|
|
108
|
+
*
|
|
109
|
+
* AppKit only installs the cache interceptor when a non-empty `cacheKey` is
|
|
110
|
+
* present, so every cached call has to name its own key parts. The per-user
|
|
111
|
+
* half of the key is added by `execute()` from the active execution context,
|
|
112
|
+
* which is what keeps an OBO result from leaking across identities.
|
|
113
|
+
*/
|
|
114
|
+
export function toCallSettings(
|
|
115
|
+
base: WebSearchExecutionSettings,
|
|
116
|
+
timeoutMs: number,
|
|
117
|
+
cacheKey: readonly (string | number)[],
|
|
118
|
+
): WebSearchExecutionSettings {
|
|
119
|
+
return {
|
|
120
|
+
default: {
|
|
121
|
+
...base.default,
|
|
122
|
+
cache: { ...base.default.cache, cacheKey: [...cacheKey] },
|
|
123
|
+
timeout: timeoutMs,
|
|
124
|
+
},
|
|
125
|
+
...(base.user ? { user: base.user } : {}),
|
|
126
|
+
};
|
|
127
|
+
}
|