@dbx-tools/appkit-web-search 0.3.21 → 0.3.23

package/package.json

@@ -17,17 +17,17 @@
     "@mastra/core": "^1.47.0",
     "got-scraping": "^4.2.1",
     "zod": "^4.3.6",
-    "@dbx-tools/shared-core": "0.3.21",
-    "@dbx-tools/model": "0.3.21",
-    "@dbx-tools/path": "0.3.21",
-    "@dbx-tools/shared-model": "0.3.21"
+    "@dbx-tools/path": "0.3.23",
+    "@dbx-tools/model": "0.3.23",
+    "@dbx-tools/shared-core": "0.3.23",
+    "@dbx-tools/shared-model": "0.3.23"
   },
   "main": "index.ts",
   "license": "UNLICENSED",
   "publishConfig": {
     "access": "public"
   },
-  "version": "0.3.21",
+  "version": "0.3.23",
   "types": "index.ts",
   "type": "module",
   "exports": {

package/src/allowlist.ts

@@ -29,8 +29,8 @@
  * @module
  */
 
-import { object } from "@dbx-tools/shared-core";
 import { match, type PathMatcher } from "@dbx-tools/path";
+import { object } from "@dbx-tools/shared-core";
 
 /** A compiled URL allow-list. Build one with {@link toUrlAllowList}. */
 export interface UrlAllowList {

package/src/config.ts

@@ -222,7 +222,7 @@ function resolvePositiveInt(value: number | undefined, envKey: string, fallback:
 
 /** Parse the `WEB_SEARCH_TOOLS` env var (JSON), else `{}`. Bad JSON is ignored. */
 function parseToolsEnv(): Record<string, unknown> {
-  const raw = process.env["WEB_SEARCH_TOOLS"];
+  const raw = process.env.WEB_SEARCH_TOOLS;
   if (!raw) return {};
   try {
     const parsed = JSON.parse(raw) as unknown;
@@ -241,18 +241,23 @@ function parseToolsEnv(): Record<string, unknown> {
 export function resolveWebSearchConfig(
   config: WebSearchPluginConfig = {},
 ): ResolvedWebSearchConfig {
-  const patterns = parseAllowedUrls(config.allowedUrls ?? process.env["WEB_SEARCH_ALLOWED_URLS"]);
-  const model = config.model ?? process.env["WEB_SEARCH_MODEL"];
-  const fallbacks = toStringList(config.modelFallbacks ?? process.env["WEB_SEARCH_MODEL_FALLBACKS"]);
-  const fuzzyThresholdRaw = config.modelFuzzyThreshold ?? Number(process.env["WEB_SEARCH_FUZZY_THRESHOLD"]);
+  const patterns = parseAllowedUrls(config.allowedUrls ?? process.env.WEB_SEARCH_ALLOWED_URLS);
+  const model = config.model ?? process.env.WEB_SEARCH_MODEL;
+  const fallbacks = toStringList(config.modelFallbacks ?? process.env.WEB_SEARCH_MODEL_FALLBACKS);
+  const fuzzyThresholdRaw =
+    config.modelFuzzyThreshold ?? Number(process.env.WEB_SEARCH_FUZZY_THRESHOLD);
   return {
     ...(model ? { model } : {}),
     modelFallbacks: fallbacks.length > 0 ? fallbacks : DEFAULT_MODEL_FALLBACKS,
     webSearchTools: { ...parseToolsEnv(), ...(config.webSearchTools ?? {}) },
-    fuzzy:
-      config.modelFuzzyMatch ?? object.toBoolean(process.env["WEB_SEARCH_FUZZY"]) ?? true,
-    fuzzyThreshold: Number.isFinite(fuzzyThresholdRaw) && fuzzyThresholdRaw ? Number(fuzzyThresholdRaw) : 0.4,
-    maxCitations: resolvePositiveInt(config.maxCitations, "WEB_SEARCH_MAX_CITATIONS", DEFAULT_MAX_CITATIONS),
+    fuzzy: config.modelFuzzyMatch ?? object.toBoolean(process.env.WEB_SEARCH_FUZZY) ?? true,
+    fuzzyThreshold:
+      Number.isFinite(fuzzyThresholdRaw) && fuzzyThresholdRaw ? Number(fuzzyThresholdRaw) : 0.4,
+    maxCitations: resolvePositiveInt(
+      config.maxCitations,
+      "WEB_SEARCH_MAX_CITATIONS",
+      DEFAULT_MAX_CITATIONS,
+    ),
     fetchMaxLength: resolvePositiveInt(
       config.fetchMaxLength,
       "WEB_SEARCH_FETCH_MAX_LENGTH",
@@ -260,7 +265,7 @@ export function resolveWebSearchConfig(
     ),
     timeoutMs: resolvePositiveInt(config.timeoutMs, "WEB_SEARCH_TIMEOUT_MS", DEFAULT_TIMEOUT_MS),
     scrapeFallback:
-      config.scrapeFallback ?? object.toBoolean(process.env["WEB_SEARCH_SCRAPE_FALLBACK"]) ?? true,
+      config.scrapeFallback ?? object.toBoolean(process.env.WEB_SEARCH_SCRAPE_FALLBACK) ?? true,
     allowList: toUrlAllowList(patterns),
     approval: config.approval ?? false,
   };

package/src/fetch.ts

@@ -94,8 +94,7 @@ export async function runWebFetch(
   const body = typeof response.body === "string" ? response.body : String(response.body ?? "");
   const contentType = response.headers["content-type"];
   const isHtml = !contentType || /html|xml/i.test(contentType);
-  const rawContent =
-    request.format === "html" || !isHtml ? body : htmlToText(body);
+  const rawContent = request.format === "html" || !isHtml ? body : htmlToText(body);
   const { content, truncated } = truncate(rawContent, cap);
   const title = isHtml ? extractTitle(body) : undefined;
 

package/src/runtime.ts

@@ -11,7 +11,11 @@
  * @module
  */
 
-import { resolveWebSearchConfig, type ResolvedWebSearchConfig, type WebSearchPluginConfig } from "./config";
+import {
+  resolveWebSearchConfig,
+  type ResolvedWebSearchConfig,
+  type WebSearchPluginConfig,
+} from "./config";
 
 /** The shared resolved config. */
 export interface WebSearchRuntime {

package/src/schema.ts

@@ -58,9 +58,7 @@ export type WebSearchCitation = z.infer<typeof webSearchCitationSchema>;
 /** Schema for the `web_search` tool output. */
 export const webSearchResultSchema = z.object({
   query: z.string().describe("Echo of the query that was searched."),
-  answer: z
-    .string()
-    .describe("The model's answer, synthesized from live web results."),
+  answer: z.string().describe("The model's answer, synthesized from live web results."),
   citations: z
     .array(webSearchCitationSchema)
     .describe(

package/src/scrape.ts

@@ -104,7 +104,9 @@ export async function runScrapeSearch(
   const answer =
     citations.length > 0
       ? `Web results for "${request.query}" (no Databricks web-search model is deployed in this workspace, so these are unsynthesized search results - read the citations):\n\n` +
-        citations.map((c, n) => `${n + 1}. ${c.title}${c.snippet ? ` - ${c.snippet}` : ""} (${c.url})`).join("\n")
+        citations
+          .map((c, n) => `${n + 1}. ${c.title}${c.snippet ? ` - ${c.snippet}` : ""} (${c.url})`)
+          .join("\n")
       : `No web results found for "${request.query}".`;
   logger.debug("scraped", {
     query: request.query,

package/src/search.ts

@@ -21,8 +21,9 @@
  * @module
  */
 
-import { error, log } from "@dbx-tools/shared-core";
 import { resolve, serving } from "@dbx-tools/model";
+import { error, log } from "@dbx-tools/shared-core";
+import { openaiResponses } from "@dbx-tools/shared-model";
 import type { ResolvedWebSearchConfig } from "./config";
 import { detectWebSearchProvider, supportsWebSearch, webSearchToolSpec } from "./provider";
 import type { WebSearchCitation, WebSearchRequest, WebSearchResult } from "./schema";
@@ -118,36 +119,16 @@ function str(v: unknown): string {
 }
 
 /**
- * Extract answer text + citations from an OpenAI Responses API payload. The
- * Responses API returns an `output` array of items; message items carry
- * `content` parts with `text` and optional `annotations` (url_citation).
- * `output_text` is the convenience aggregate when present.
+ * Extract answer text + citations from an OpenAI Responses API payload, via the
+ * shared reader in `@dbx-tools/shared-model` (the same module the model-proxy
+ * uses to translate the Responses wire format in the other direction).
  */
 function fromResponsesPayload(payload: Record<string, unknown>): {
   answer: string;
   citations: WebSearchCitation[];
 } {
-  const citations: WebSearchCitation[] = [];
-  const texts: string[] = [];
-  const output = Array.isArray(payload["output"]) ? (payload["output"] as unknown[]) : [];
-  for (const item of output) {
-    const content = (item as { content?: unknown }).content;
-    if (!Array.isArray(content)) continue;
-    for (const part of content) {
-      const p = part as { text?: unknown; annotations?: unknown };
-      const text = str(p.text);
-      if (text) texts.push(text);
-      if (Array.isArray(p.annotations)) {
-        for (const ann of p.annotations) {
-          const a = ann as { url?: unknown; title?: unknown };
-          const url = str(a.url);
-          if (url) citations.push({ url, ...(str(a.title) ? { title: str(a.title) } : {}) });
-        }
-      }
-    }
-  }
-  const answer = str(payload["output_text"]) || texts.join("\n").trim();
-  return { answer, citations };
+  const { text, citations } = openaiResponses.readResponsesOutput(payload);
+  return { answer: text, citations };
 }
 
 /**
@@ -160,26 +141,26 @@ function fromChatPayload(payload: Record<string, unknown>): {
   answer: string;
   citations: WebSearchCitation[];
 } {
-  const choices = Array.isArray(payload["choices"]) ? (payload["choices"] as unknown[]) : [];
+  const choices = Array.isArray(payload.choices) ? (payload.choices as unknown[]) : [];
   const message = (choices[0] as { message?: Record<string, unknown> })?.message ?? {};
-  const answer = str(message["content"]);
+  const answer = str(message.content);
   const citations: WebSearchCitation[] = [];
   // Best-effort grounding extraction: walk any nested object for {uri|url,title}.
   const seen = new Set<string>();
   const visit = (v: unknown, depth: number): void => {
     if (depth > 6 || v === null || typeof v !== "object") return;
     const o = v as Record<string, unknown>;
-    const url = str(o["url"]) || str(o["uri"]);
+    const url = str(o.url) || str(o.uri);
     if (url && !seen.has(url)) {
       seen.add(url);
-      citations.push({ url, ...(str(o["title"]) ? { title: str(o["title"]) } : {}) });
+      citations.push({ url, ...(str(o.title) ? { title: str(o.title) } : {}) });
     }
     for (const val of Object.values(o)) {
       if (Array.isArray(val)) val.forEach((x) => visit(x, depth + 1));
       else if (val && typeof val === "object") visit(val, depth + 1);
     }
   };
-  visit(message["grounding_metadata"] ?? message["groundingMetadata"], 0);
+  visit(message.grounding_metadata ?? message.groundingMetadata, 0);
   return { answer, citations };
 }
 

package/test/allowlist.test.ts

@@ -0,0 +1,150 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import {
+  assertUrlAllowed,
+  normalizeUrlPattern,
+  parseAllowedUrls,
+  toUrlAllowList,
+} from "../src/allowlist";
+import { approvalMatches, resolveWebSearchConfig } from "../src/config";
+import { htmlToText } from "../src/fetch";
+import {
+  detectWebSearchProvider,
+  supportsWebSearch,
+  webSearchToolSpec,
+  WEB_SEARCH_PROVIDERS,
+} from "../src/provider";
+
+describe("web-search allow-list", () => {
+  it("strips the scheme from entries, leaving host / path verbatim", () => {
+    assert.equal(normalizeUrlPattern("https://*.databricks.com"), "*.databricks.com");
+    assert.equal(normalizeUrlPattern("docs.example.com/api/**"), "docs.example.com/api/**");
+    assert.equal(normalizeUrlPattern("databricks.com"), "databricks.com");
+  });
+
+  it("parses CSV and array forms, trimming + de-duping", () => {
+    assert.deepEqual(parseAllowedUrls("https://databricks.com, docs.example.com"), [
+      "databricks.com",
+      "docs.example.com",
+    ]);
+    assert.deepEqual(parseAllowedUrls(["databricks.com", "databricks.com"]), ["databricks.com"]);
+  });
+
+  it("permits everything when unconfigured", () => {
+    const list = toUrlAllowList(parseAllowedUrls(undefined));
+    assert.equal(list.restricted, false);
+    assert.equal(list.allows("https://anything.example.com/x"), true);
+  });
+
+  it("a bare host permits the host and its subdomains, blocks others", () => {
+    const list = toUrlAllowList(parseAllowedUrls(["databricks.com", "*.trusted.io"]));
+    assert.equal(list.restricted, true);
+    assert.equal(list.allows("https://databricks.com/"), true);
+    assert.equal(list.allows("https://docs.databricks.com/aws/en/index.html"), true);
+    assert.equal(list.allows("https://api.trusted.io/v1"), true);
+    assert.equal(list.allows("https://evil.example.com/"), false);
+    assert.equal(list.allows("https://notdatabricks.com/"), false);
+  });
+
+  it("a path entry constrains the pathname", () => {
+    const list = toUrlAllowList(parseAllowedUrls(["docs.example.com/api/**"]));
+    assert.equal(list.allows("https://docs.example.com/api/v1/x"), true);
+    assert.equal(list.allows("https://docs.example.com/blog/x"), false);
+  });
+
+  it("assertUrlAllowed throws for a disallowed URL, passes an allowed one", () => {
+    const list = toUrlAllowList(parseAllowedUrls(["databricks.com"]));
+    assert.throws(() => assertUrlAllowed("https://evil.example.com/", list), /not permitted/);
+    assert.doesNotThrow(() => assertUrlAllowed("https://databricks.com/", list));
+  });
+});
+
+describe("web-search provider detection", () => {
+  it("maps GPT ids to openai and Gemini ids to gemini", () => {
+    assert.equal(detectWebSearchProvider("databricks-gpt-5"), "openai");
+    assert.equal(detectWebSearchProvider("databricks-gpt-5-mini"), "openai");
+    assert.equal(detectWebSearchProvider("databricks-gemini-3-pro"), "gemini");
+    assert.equal(detectWebSearchProvider("databricks-gemini-2-5-flash"), "gemini");
+  });
+
+  it("treats gpt-oss and non-web families as unsupported", () => {
+    assert.equal(detectWebSearchProvider("databricks-gpt-oss-120b"), null);
+    assert.equal(detectWebSearchProvider("databricks-claude-sonnet-4-6"), null);
+    assert.equal(detectWebSearchProvider("databricks-llama-3-70b"), null);
+    assert.equal(supportsWebSearch("databricks-claude-sonnet-4-6"), false);
+    assert.equal(supportsWebSearch("databricks-gpt-5"), true);
+  });
+
+  it("uses the built-in tool spec per provider", () => {
+    assert.deepEqual(webSearchToolSpec("openai").tool, { type: "web_search" });
+    assert.equal(webSearchToolSpec("openai").api, "responses");
+    assert.deepEqual(webSearchToolSpec("gemini").tool, { google_search: {} });
+    assert.equal(webSearchToolSpec("gemini").api, "chat");
+    assert.deepEqual(WEB_SEARCH_PROVIDERS.openai.tool, { type: "web_search" });
+  });
+
+  it("merges an operator override over the built-in map", () => {
+    const overrides = { gemini: { tool: { google_search_retrieval: {} } } };
+    const spec = webSearchToolSpec("gemini", overrides);
+    assert.deepEqual(spec.tool, { google_search_retrieval: {} });
+    // api falls through to the built-in when not overridden
+    assert.equal(spec.api, "chat");
+    // other providers keep their defaults
+    assert.deepEqual(webSearchToolSpec("openai", overrides).tool, { type: "web_search" });
+  });
+});
+
+describe("web-search config", () => {
+  it("defaults model fallbacks to Gemini-then-GPT and approval to none", () => {
+    const c = resolveWebSearchConfig();
+    assert.equal(c.approval, false);
+    assert.equal(c.model, undefined);
+    assert.match(c.modelFallbacks[0]!, /gemini/);
+    assert.ok(c.modelFallbacks.some((m) => m.includes("gpt")));
+  });
+
+  it("merges webSearchTools override into resolved config", () => {
+    const c = resolveWebSearchConfig({ webSearchTools: { gemini: { tool: { x: {} } } } });
+    assert.deepEqual((c.webSearchTools as { gemini: unknown }).gemini, { tool: { x: {} } });
+  });
+
+  it("enables the scrape fallback by default, honors an explicit off", () => {
+    assert.equal(resolveWebSearchConfig().scrapeFallback, true);
+    assert.equal(resolveWebSearchConfig({ scrapeFallback: false }).scrapeFallback, false);
+  });
+});
+
+describe("web-search approval gate", () => {
+  it("boolean gates pass through", () => {
+    assert.equal(approvalMatches(true, ["https://x.com"]), true);
+    assert.equal(approvalMatches(false, ["https://x.com"]), false);
+  });
+
+  it("pattern gate matches only candidate URLs it covers", () => {
+    assert.equal(
+      approvalMatches("*.internal.example.com", ["https://api.internal.example.com/x"]),
+      true,
+    );
+    assert.equal(
+      approvalMatches("*.internal.example.com", ["https://public.example.com/x"]),
+      false,
+    );
+    assert.equal(approvalMatches(["a.com", "b.com"], ["https://b.com/y"]), true);
+  });
+});
+
+describe("web-search html-to-text", () => {
+  it("strips script/style and unwraps tags", () => {
+    const html =
+      "<html><head><title>Hi</title><style>.x{}</style></head><body><p>One</p><script>bad()</script><p>Two</p></body></html>";
+    const text = htmlToText(html);
+    assert.match(text, /One/);
+    assert.match(text, /Two/);
+    assert.doesNotMatch(text, /bad\(\)/);
+    assert.doesNotMatch(text, /\.x\{\}/);
+  });
+
+  it("decodes common entities", () => {
+    assert.equal(htmlToText("<p>a &amp; b &lt;c&gt;</p>"), "a & b <c>");
+  });
+});

package/test/tsconfig.json

@@ -0,0 +1,14 @@
+// ~~ Generated by projen. To modify, edit .projenrc.js and run "pnpm exec projen".
+{
+  "extends": "../tsconfig.json",
+  "compilerOptions": {
+    "noEmit": true,
+    "rootDir": ".."
+  },
+  "include": [
+    "**/*.ts"
+  ],
+  "exclude": [
+    "node_modules"
+  ]
+}