@dbx-tools/appkit-web-search 0.3.28 → 0.3.29

package/README.md

@@ -223,7 +223,9 @@ the [Databricks docs](https://docs.databricks.com/aws/en/machine-learning/model-
 - `provider` - provider detection + the provider -> tool-spec map.
 - `scrape` - `runScrapeSearch()` DuckDuckGo fallback for workspaces with no
   native web-search model.
-- `fetch` - `runWebFetch()` over got-scraping, plus `htmlToText()`.
+- `fetch` - `runWebFetch()` over got-scraping.
+- `html-text` - `htmlToText()` / `htmlFragmentToText()` / `decodeHtmlEntities()`, shared by
+  `fetch` and the DuckDuckGo scrape fallback.
 - `runtime` - shared runtime, `getWebSearchRuntime()`, `resetWebSearchRuntime()`.
 - `config` - config types, JSON schema, `resolveWebSearchConfig()`, approval helpers.
 - `allowlist` - URL allow-list parsing/compiling on top of `@dbx-tools/path`.

package/index.ts

@@ -5,6 +5,7 @@
 export * as allowlist from "./src/allowlist";
 export * as config from "./src/config";
 export * as fetch from "./src/fetch";
+export * as htmlText from "./src/html-text";
 export * as plugin from "./src/plugin";
 export * as provider from "./src/provider";
 export * as runtime from "./src/runtime";

package/package.json

@@ -17,17 +17,17 @@
     "@mastra/core": "^1.47.0",
     "got-scraping": "^4.2.1",
     "zod": "^4.3.6",
-    "@dbx-tools/path": "0.3.28",
-    "@dbx-tools/model": "0.3.28",
-    "@dbx-tools/shared-core": "0.3.28",
-    "@dbx-tools/shared-model": "0.3.28"
+    "@dbx-tools/path": "0.3.29",
+    "@dbx-tools/model": "0.3.29",
+    "@dbx-tools/shared-core": "0.3.29",
+    "@dbx-tools/shared-model": "0.3.29"
   },
   "main": "index.ts",
   "license": "UNLICENSED",
   "publishConfig": {
     "access": "public"
   },
-  "version": "0.3.28",
+  "version": "0.3.29",
   "types": "index.ts",
   "type": "module",
   "exports": {

package/src/allowlist.ts

@@ -30,7 +30,7 @@
  */
 
 import { match, type PathMatcher } from "@dbx-tools/path";
-import { object } from "@dbx-tools/shared-core";
+import { string } from "@dbx-tools/shared-core";
 
 /** A compiled URL allow-list. Build one with {@link toUrlAllowList}. */
 export interface UrlAllowList {
@@ -64,15 +64,7 @@ export function normalizeUrlPattern(pattern: string): string {
  * identically.
  */
 export function parseAllowedUrls(raw: string | string[] | undefined): string[] {
-  const entries = typeof raw === "string" ? raw.split(/[\s,]+/) : Array.isArray(raw) ? raw : [];
-  return [
-    ...object
-      .sequence(entries)
-      .map((entry) => normalizeUrlPattern(entry))
-      .filter((entry) => entry.length > 0)
-      .distinct()
-      .toArray(),
-  ];
+  return string.parseList(raw, normalizeUrlPattern);
 }
 
 /** One compiled entry: which URL slice it tests, and the matcher for it. */

package/src/config.ts

@@ -26,7 +26,8 @@
  */
 
 import type { BasePluginConfig } from "@databricks/appkit";
-import { object, type OneOrMany } from "@dbx-tools/shared-core";
+import { serving } from "@dbx-tools/model";
+import { json, object, type OneOrMany, string } from "@dbx-tools/shared-core";
 import type { JSONSchema7 } from "json-schema";
 import { parseAllowedUrls, toUrlAllowList, type UrlAllowList } from "./allowlist";
 
@@ -93,7 +94,7 @@ export interface WebSearchPluginConfig extends BasePluginConfig {
    * Defaults to `true`; falls back to `WEB_SEARCH_FUZZY`.
    */
   modelFuzzyMatch?: boolean;
-  /** Fuse.js fuzzy threshold. Falls back to `WEB_SEARCH_FUZZY_THRESHOLD`, then 0.4. */
+  /** Fuse.js fuzzy threshold. Falls back to `WEB_SEARCH_FUZZY_THRESHOLD`, then the shared default. */
   modelFuzzyThreshold?: number;
   /**
    * Hard cap on the number of citations a single search returns. Falls back
@@ -200,19 +201,6 @@ export const WEB_SEARCH_CONFIG_SCHEMA: JSONSchema7 = {
   },
 };
 
-/** Split a CSV / whitespace / array value into a trimmed, non-empty string list. */
-function toStringList(raw: string | string[] | undefined): string[] {
-  const entries = typeof raw === "string" ? raw.split(/[\s,]+/) : Array.isArray(raw) ? raw : [];
-  return [
-    ...object
-      .sequence(entries)
-      .map((e) => e.trim())
-      .filter((e) => e.length > 0)
-      .distinct()
-      .toArray(),
-  ];
-}
-
 /** Parse a positive integer env/config value, else the fallback. */
 function resolvePositiveInt(value: number | undefined, envKey: string, fallback: number): number {
   if (typeof value === "number" && Number.isFinite(value) && value > 0) return Math.floor(value);
@@ -222,14 +210,7 @@ function resolvePositiveInt(value: number | undefined, envKey: string, fallback:
 
 /** Parse the `WEB_SEARCH_TOOLS` env var (JSON), else `{}`. Bad JSON is ignored. */
 function parseToolsEnv(): Record<string, unknown> {
-  const raw = process.env.WEB_SEARCH_TOOLS;
-  if (!raw) return {};
-  try {
-    const parsed = JSON.parse(raw) as unknown;
-    return parsed && typeof parsed === "object" ? (parsed as Record<string, unknown>) : {};
-  } catch {
-    return {};
-  }
+  return json.parseRecord(process.env.WEB_SEARCH_TOOLS) ?? {};
 }
 
 /**
@@ -243,7 +224,9 @@ export function resolveWebSearchConfig(
 ): ResolvedWebSearchConfig {
   const patterns = parseAllowedUrls(config.allowedUrls ?? process.env.WEB_SEARCH_ALLOWED_URLS);
   const model = config.model ?? process.env.WEB_SEARCH_MODEL;
-  const fallbacks = toStringList(config.modelFallbacks ?? process.env.WEB_SEARCH_MODEL_FALLBACKS);
+  const fallbacks = string.parseList(
+    config.modelFallbacks ?? process.env.WEB_SEARCH_MODEL_FALLBACKS,
+  );
   const fuzzyThresholdRaw =
     config.modelFuzzyThreshold ?? Number(process.env.WEB_SEARCH_FUZZY_THRESHOLD);
   return {
@@ -252,7 +235,9 @@ export function resolveWebSearchConfig(
     webSearchTools: { ...parseToolsEnv(), ...(config.webSearchTools ?? {}) },
     fuzzy: config.modelFuzzyMatch ?? object.toBoolean(process.env.WEB_SEARCH_FUZZY) ?? true,
     fuzzyThreshold:
-      Number.isFinite(fuzzyThresholdRaw) && fuzzyThresholdRaw ? Number(fuzzyThresholdRaw) : 0.4,
+      Number.isFinite(fuzzyThresholdRaw) && fuzzyThresholdRaw
+        ? Number(fuzzyThresholdRaw)
+        : serving.DEFAULT_FUZZY_THRESHOLD,
     maxCitations: resolvePositiveInt(
       config.maxCitations,
       "WEB_SEARCH_MAX_CITATIONS",

package/src/fetch.ts

@@ -18,6 +18,7 @@
 import { log } from "@dbx-tools/shared-core";
 import { gotScraping } from "got-scraping";
 import { assertUrlAllowed } from "./allowlist";
+import { decodeHtmlEntities, htmlToText } from "./html-text";
 import type { ResolvedWebSearchConfig } from "./config";
 import type { WebFetchRequest, WebFetchResult } from "./schema";
 
@@ -26,43 +27,10 @@ const logger = log.logger("web-search/fetch");
 /** Pull the <title> text out of an HTML document, when present. */
 function extractTitle(html: string): string | undefined {
   const match = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(html);
-  const title = match?.[1] ? decodeEntities(match[1]).trim() : "";
+  const title = match?.[1] ? decodeHtmlEntities(match[1]).trim() : "";
   return title.length > 0 ? title : undefined;
 }
 
-/** Decode the handful of HTML entities that survive tag-stripping. */
-function decodeEntities(text: string): string {
-  return text
-    .replace(/&nbsp;/g, " ")
-    .replace(/&amp;/g, "&")
-    .replace(/&lt;/g, "<")
-    .replace(/&gt;/g, ">")
-    .replace(/&quot;/g, '"')
-    .replace(/&#39;/g, "'")
-    .replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code)));
-}
-
-/**
- * Reduce an HTML document to readable plain text: drop `<script>` /
- * `<style>` / `<noscript>` blocks and HTML comments, turn block-level tags
- * into newlines, strip the remaining tags, decode entities, and collapse
- * runs of blank lines / trailing spaces.
- */
-export function htmlToText(html: string): string {
-  return decodeEntities(
-    html
-      .replace(/<!--[\s\S]*?-->/g, "")
-      .replace(/<(script|style|noscript)[^>]*>[\s\S]*?<\/\1>/gi, "")
-      .replace(/<\/(p|div|section|article|li|tr|h[1-6]|header|footer|br)>/gi, "\n")
-      .replace(/<br\s*\/?>/gi, "\n")
-      .replace(/<[^>]+>/g, ""),
-  )
-    .replace(/[ \t]+\n/g, "\n")
-    .replace(/\n{3,}/g, "\n\n")
-    .replace(/[ \t]{2,}/g, " ")
-    .trim();
-}
-
 /** Truncate `text` to `max` chars, reporting whether it was cut. */
 function truncate(text: string, max: number): { content: string; truncated: boolean } {
   if (text.length <= max) return { content: text, truncated: false };

package/src/html-text.ts

@@ -0,0 +1,72 @@
+/**
+ * Turning fetched HTML into the plain text a model can read.
+ *
+ * Both scraping paths need this: {@link fetchUrl}'s full-page read and the
+ * DuckDuckGo fallback's title/snippet extraction. They previously carried
+ * separate entity tables that drifted, so the decode lives here once.
+ *
+ * This is deliberately regex-based rather than a DOM parse. The inputs are
+ * whole pages fetched for summarization, not documents to be queried, so the
+ * cost of a real parser buys nothing - and a malformed page still degrades to
+ * readable text instead of throwing.
+ *
+ * @module
+ */
+
+/**
+ * The named entities that survive tag-stripping in practice, plus numeric
+ * escapes. Not a complete HTML entity table - anything rarer passes through
+ * as-is, which reads acceptably in model input.
+ */
+const NAMED_ENTITIES: Record<string, string> = {
+  "&nbsp;": " ",
+  "&amp;": "&",
+  "&lt;": "<",
+  "&gt;": ">",
+  "&quot;": '"',
+  "&#39;": "'",
+  "&#x27;": "'",
+  "&apos;": "'",
+};
+
+const NAMED_ENTITY_REGEXP = new RegExp(Object.keys(NAMED_ENTITIES).join("|"), "g");
+const NUMERIC_ENTITY_REGEXP = /&#(\d+);/g;
+const TAG_REGEXP = /<[^>]+>/g;
+
+/** Decode the HTML entities that survive tag-stripping. */
+export function decodeHtmlEntities(text: string): string {
+  return text
+    .replace(NAMED_ENTITY_REGEXP, (entity) => NAMED_ENTITIES[entity] ?? entity)
+    .replace(NUMERIC_ENTITY_REGEXP, (_, code) => String.fromCodePoint(Number(code)));
+}
+
+/**
+ * Strip tags and decode entities from a short HTML fragment, collapsing all
+ * whitespace to single spaces. For inline snippets (a search result title or
+ * summary), where layout carries no meaning.
+ */
+export function htmlFragmentToText(html: string): string {
+  return decodeHtmlEntities(html.replace(TAG_REGEXP, "")).replace(/\s+/g, " ").trim();
+}
+
+/**
+ * Reduce a full HTML document to readable plain text: drop `<script>` /
+ * `<style>` / `<noscript>` blocks and comments, turn block-level tags into
+ * newlines, strip the remaining tags, decode entities, and collapse runs of
+ * blank lines / trailing spaces. Unlike {@link htmlFragmentToText}, this
+ * preserves line structure because paragraph breaks carry meaning in a page.
+ */
+export function htmlToText(html: string): string {
+  return decodeHtmlEntities(
+    html
+      .replace(/<!--[\s\S]*?-->/g, "")
+      .replace(/<(script|style|noscript)[^>]*>[\s\S]*?<\/\1>/gi, "")
+      .replace(/<\/(p|div|section|article|li|tr|h[1-6]|header|footer|br)>/gi, "\n")
+      .replace(/<br\s*\/?>/gi, "\n")
+      .replace(TAG_REGEXP, ""),
+  )
+    .replace(/[ \t]+\n/g, "\n")
+    .replace(/\n{3,}/g, "\n\n")
+    .replace(/[ \t]{2,}/g, " ")
+    .trim();
+}

package/src/plugin.ts

@@ -13,13 +13,13 @@
  * @module
  */
 
-import { getExecutionContext, Plugin, toPlugin, type PluginManifest } from "@databricks/appkit";
+import { Plugin, toPlugin, type PluginManifest } from "@databricks/appkit";
 import { log } from "@dbx-tools/shared-core";
 import { WEB_SEARCH_CONFIG_SCHEMA, type WebSearchPluginConfig } from "./config";
 import { runWebFetch } from "./fetch";
 import { getWebSearchRuntime } from "./runtime";
 import type { WebFetchRequest, WebFetchResult, WebSearchRequest, WebSearchResult } from "./schema";
-import { runWebSearch, type WebSearchContext } from "./search";
+import { resolveWebSearchContext, runWebSearch } from "./search";
 
 /**
  * AppKit plugin that resolves and holds the web-search runtime config used
@@ -30,8 +30,9 @@ export class WebSearchPlugin extends Plugin<WebSearchPluginConfig> {
     name: "web-search",
     displayName: "Web Search",
     description:
-      "Searches the web (via duck-duck-scrape) and fetches pages (via got-scraping), " +
-      "with an optional URL allow-list and per-tool approval gating.",
+      "Searches the web through the Databricks native web-search tool on its own " +
+      "web-capable serving endpoint, and fetches pages via got-scraping, behind an " +
+      "optional URL allow-list and per-tool approval gating.",
     stability: "beta",
     resources: {
       required: [],
@@ -58,13 +59,6 @@ export class WebSearchPlugin extends Plugin<WebSearchPluginConfig> {
     });
   }
 
-  /** Resolve the OBO client + host from the active execution context. */
-  private async searchContext(): Promise<WebSearchContext> {
-    const ctx = getExecutionContext();
-    const host = (await ctx.client.config.getHost()).toString();
-    return { client: ctx.client, host };
-  }
-
   override exports() {
     return {
       /**
@@ -73,7 +67,7 @@ export class WebSearchPlugin extends Plugin<WebSearchPluginConfig> {
        * runtime config primed at setup.
        */
       search: async (request: WebSearchRequest): Promise<WebSearchResult> =>
-        runWebSearch(request, getWebSearchRuntime().config, await this.searchContext()),
+        runWebSearch(request, getWebSearchRuntime().config, await resolveWebSearchContext()),
       /**
        * Fetch one URL directly (bypassing the agent tool). Enforces the
        * configured allow-list. Reads the shared runtime config.

package/src/scrape.ts

@@ -21,6 +21,7 @@
 import { log } from "@dbx-tools/shared-core";
 import { gotScraping } from "got-scraping";
 import type { ResolvedWebSearchConfig } from "./config";
+import { htmlFragmentToText } from "./html-text";
 import type { WebSearchCitation, WebSearchRequest, WebSearchResult } from "./schema";
 
 const logger = log.logger("web-search/scrape");
@@ -28,20 +29,6 @@ const logger = log.logger("web-search/scrape");
 /** DuckDuckGo's no-JS HTML results endpoint (queried via GET). */
 const DDG_HTML_URL = "https://html.duckduckgo.com/html/";
 
-/** Strip HTML tags and decode the few entities DDG emits in titles/snippets. */
-function stripTags(html: string): string {
-  return html
-    .replace(/<[^>]+>/g, "")
-    .replace(/&amp;/g, "&")
-    .replace(/&lt;/g, "<")
-    .replace(/&gt;/g, ">")
-    .replace(/&quot;/g, '"')
-    .replace(/&#x27;|&#39;/g, "'")
-    .replace(/&nbsp;/g, " ")
-    .replace(/\s+/g, " ")
-    .trim();
-}
-
 /**
  * DDG wraps result URLs in a redirect (`//duckduckgo.com/l/?uddg=<encoded>`).
  * Unwrap to the real destination; leave already-absolute URLs untouched.
@@ -66,12 +53,12 @@ function parseDdgHtml(html: string): WebSearchCitation[] {
   const snippetRe = /<a[^>]*class="[^"]*result__snippet[^"]*"[^>]*>([\s\S]*?)<\/a>/g;
   const snippets: string[] = [];
   let sm: RegExpExecArray | null;
-  while ((sm = snippetRe.exec(html)) !== null) snippets.push(stripTags(sm[1] ?? ""));
+  while ((sm = snippetRe.exec(html)) !== null) snippets.push(htmlFragmentToText(sm[1] ?? ""));
   let am: RegExpExecArray | null;
   let i = 0;
   while ((am = anchorRe.exec(html)) !== null) {
     const url = unwrapDdgUrl(am[1] ?? "");
-    const title = stripTags(am[2] ?? "");
+    const title = htmlFragmentToText(am[2] ?? "");
     if (!url || !title) continue;
     const snippet = snippets[i] ?? "";
     citations.push({ url, title, ...(snippet ? { snippet } : {}) });

package/src/search.ts

@@ -21,9 +21,10 @@
  * @module
  */
 
-import { resolve, serving } from "@dbx-tools/model";
-import { error, log } from "@dbx-tools/shared-core";
-import { openaiResponses } from "@dbx-tools/shared-model";
+import { getExecutionContext } from "@databricks/appkit";
+import { invoke, resolve, serving } from "@dbx-tools/model";
+import { error, log, string } from "@dbx-tools/shared-core";
+import { openaiChat, openaiResponses } from "@dbx-tools/shared-model";
 import type { ResolvedWebSearchConfig } from "./config";
 import { detectWebSearchProvider, supportsWebSearch, webSearchToolSpec } from "./provider";
 import type { WebSearchCitation, WebSearchRequest, WebSearchResult } from "./schema";
@@ -40,6 +41,18 @@ export interface WebSearchContext {
   host: string;
 }
 
+/**
+ * Resolve the OBO workspace client + host from the active AppKit execution
+ * context. Inside `agent.stream`'s `asUser(req)` scope this hits the serving
+ * endpoint as the requesting user; outside a user context AppKit falls back to
+ * the service principal.
+ */
+export async function resolveWebSearchContext(): Promise<WebSearchContext> {
+  const ctx = getExecutionContext();
+  const host = (await ctx.client.config.getHost()).toString();
+  return { client: ctx.client, host };
+}
+
 /**
  * Resolve a web-search-capable model against the LIVE workspace catalogue - so
  * we never return an endpoint id that isn't actually deployed (the "endpoint
@@ -113,11 +126,6 @@ async function postServing(
 
 /* --------------------------- response extraction --------------------------- */
 
-/** Coerce an unknown to a trimmed string, or "" . */
-function str(v: unknown): string {
-  return typeof v === "string" ? v.trim() : "";
-}
-
 /**
  * Extract answer text + citations from an OpenAI Responses API payload, via the
  * shared reader in `@dbx-tools/shared-model` (the same module the model-proxy
@@ -143,17 +151,18 @@ function fromChatPayload(payload: Record<string, unknown>): {
 } {
   const choices = Array.isArray(payload.choices) ? (payload.choices as unknown[]) : [];
   const message = (choices[0] as { message?: Record<string, unknown> })?.message ?? {};
-  const answer = str(message.content);
+  const answer = openaiChat.chatContentToText(message.content);
   const citations: WebSearchCitation[] = [];
   // Best-effort grounding extraction: walk any nested object for {uri|url,title}.
   const seen = new Set<string>();
   const visit = (v: unknown, depth: number): void => {
     if (depth > 6 || v === null || typeof v !== "object") return;
     const o = v as Record<string, unknown>;
-    const url = str(o.url) || str(o.uri);
+    const url = string.trimToEmpty(o.url) || string.trimToEmpty(o.uri);
     if (url && !seen.has(url)) {
       seen.add(url);
-      citations.push({ url, ...(str(o.title) ? { title: str(o.title) } : {}) });
+      const title = string.trimToEmpty(o.title);
+      citations.push({ url, ...(title ? { title } : {}) });
     }
     for (const val of Object.values(o)) {
       if (Array.isArray(val)) val.forEach((x) => visit(x, depth + 1));
@@ -195,9 +204,7 @@ export async function runWebSearch(
   const spec = webSearchToolSpec(provider, config.webSearchTools);
 
   const path =
-    spec.api === "responses"
-      ? "/serving-endpoints/responses"
-      : "/serving-endpoints/chat/completions";
+    spec.api === "responses" ? `/${invoke.RESPONSES_PATH}` : `/${invoke.CHAT_COMPLETIONS_PATH}`;
   const body =
     spec.api === "responses"
       ? {

package/src/tool.ts

@@ -18,7 +18,6 @@
  * @module
  */
 
-import { getExecutionContext } from "@databricks/appkit";
 import { string } from "@dbx-tools/shared-core";
 import { createTool } from "@mastra/core/tools";
 import { approvalMatches, type ApprovalGate } from "./config";
@@ -32,7 +31,7 @@ import {
   type WebFetchRequest,
   type WebSearchRequest,
 } from "./schema";
-import { runWebSearch, type WebSearchContext } from "./search";
+import { resolveWebSearchContext, runWebSearch } from "./search";
 
 /** Options shared by both web tools. */
 export interface WebSearchToolOptions {
@@ -51,18 +50,6 @@ function effectiveGate(opts: WebSearchToolOptions): ApprovalGate {
   return opts.approval ?? getWebSearchRuntime().config.approval;
 }
 
-/**
- * Resolve the OBO workspace client + host from the active AppKit execution
- * context. Runs inside `agent.stream`'s `asUser(req)` scope, so the search
- * hits the serving endpoint as the requesting user; outside a user context it
- * falls back to the service principal.
- */
-async function webSearchContext(): Promise<WebSearchContext> {
-  const ctx = getExecutionContext();
-  const host = (await ctx.client.config.getHost()).toString();
-  return { client: ctx.client, host };
-}
-
 /**
  * Build the `web_search` tool. Spread it into the agents that should be able
  * to search the web.
@@ -99,7 +86,7 @@ export function webSearchTool(opts: WebSearchToolOptions = {}) {
       : { requireApproval: () => (typeof gate === "boolean" ? gate : true) }),
     execute: async (input) => {
       const { config } = getWebSearchRuntime();
-      return runWebSearch(input as WebSearchRequest, config, await webSearchContext());
+      return runWebSearch(input as WebSearchRequest, config, await resolveWebSearchContext());
     },
   });
 }

package/test/allowlist.test.ts

@@ -7,7 +7,7 @@ import {
   toUrlAllowList,
 } from "../src/allowlist";
 import { approvalMatches, resolveWebSearchConfig } from "../src/config";
-import { htmlToText } from "../src/fetch";
+import { htmlToText } from "../src/html-text";
 import {
   detectWebSearchProvider,
   supportsWebSearch,