@vellumai/assistant 0.12.2 → 0.12.3-staging.2

package/src/tools/network/web-search.ts

@@ -1,8 +1,8 @@
-import { getConfig } from "../../config/loader.js";
 import type {
   WebSearchMetadata,
   WebSearchResultItem,
-} from "../../daemon/message-types/web-activity.js";
+} from "../../api/events/tool-result.js";
+import { getConfig } from "../../config/loader.js";
 import { RiskLevel } from "../../permissions/types.js";
 import { getProviderKeyAsync } from "../../security/secure-keys.js";
 import { wrapUntrustedContent } from "../../security/untrusted-content.js";
@@ -44,6 +44,7 @@ const PERPLEXITY_API_URL = "https://api.perplexity.ai/chat/completions";
 const TAVILY_API_URL = "https://api.tavily.com/search";
 const FIRECRAWL_API_URL = "https://api.firecrawl.dev/v2/search";
 const FASTCRW_SEARCH_PATH = "/v1/search";
+const TINYFISH_DEFAULT_SEARCH_API_BASE = "https://api.search.tinyfish.ai";
 // Keenable is keyless by default: the public path needs no key (rate-limited);
 // a key switches to the authenticated path and lifts the cap.
 const KEENABLE_API_BASE_URL = "https://api.keenable.ai";
@@ -56,7 +57,9 @@ type WebSearchProvider =
   | "tavily"
   | "firecrawl"
   | "keenable"
-  | "fastcrw";
+  | "fastcrw"
+  | "searxng"
+  | "tinyfish";
 
 /**
  * Arguments passed to every {@link WebSearchAdapter}. The full superset is
@@ -167,6 +170,38 @@ interface KeenableSearchResponse {
   results?: KeenableSearchResult[];
 }
 
+interface SearxngSearchResult {
+  title?: string;
+  url?: string;
+  content?: string;
+}
+
+interface SearxngSearchResponse {
+  query?: string;
+  results?: SearxngSearchResult[];
+}
+
+interface TinyfishSearchResult {
+  position?: number;
+  site_name?: string;
+  title?: string;
+  snippet?: string;
+  url?: string;
+  date?: string;
+}
+
+interface TinyfishSearchResponse {
+  query?: string;
+  results?: TinyfishSearchResult[];
+  total_results?: number;
+  page?: number;
+}
+
+const SEARXNG_MISSING_INSTANCE_MESSAGE =
+  "SearXNG needs an instance URL. Set API Base in Settings under Web Search.";
+const SEARXNG_JSON_DISABLED_MESSAGE =
+  "This SearXNG instance did not return JSON. Enable the JSON format on the instance, or point API Base at an instance that allows it.";
+
 function getWebSearchProvider(): WebSearchProvider {
   const config = getConfig();
   const configured = config.services["web-search"].provider ?? "perplexity";
@@ -573,6 +608,159 @@ function buildKeenableMetadata(
   };
 }
 
+function formatSearxngResults(
+  data: SearxngSearchResponse,
+  query: string,
+): string {
+  const results = data.results ?? [];
+  if (results.length === 0) {
+    return `No results found for "${query}".`;
+  }
+
+  const lines: string[] = [`Web search results for "${query}":\n`];
+  for (let i = 0; i < results.length; i++) {
+    const r = results[i];
+    const title = r.title?.trim() || r.url?.trim() || "Untitled result";
+    lines.push(`${i + 1}. ${title}`);
+    if (r.url) {
+      lines.push(`   URL: ${r.url}`);
+    }
+    if (r.content) {
+      lines.push(`   ${r.content}`);
+    }
+    lines.push("");
+  }
+  return lines.join("\n");
+}
+
+function buildSearxngMetadata(
+  data: SearxngSearchResponse,
+  query: string,
+  durationMs: number,
+): WebSearchMetadata {
+  const results = data.results ?? [];
+  const items: WebSearchResultItem[] = results.map((r, i) => {
+    const url = r.url ?? "";
+    const domain = extractDomain(url);
+    return {
+      rank: i + 1,
+      title: r.title?.trim() || url.trim() || "Untitled result",
+      url,
+      domain,
+      faviconUrl: faviconUrlForDomain(domain),
+      snippet: r.content?.trim() || undefined,
+    };
+  });
+  return {
+    query,
+    provider: "searxng",
+    resultCount: items.length,
+    durationMs,
+    results: items,
+  };
+}
+
+function formatTinyfishResults(
+  data: TinyfishSearchResponse,
+  query: string,
+): string {
+  const results = data.results ?? [];
+  if (results.length === 0) {
+    return `No results found for "${query}".`;
+  }
+
+  const lines: string[] = [`Web search results for "${query}":\n`];
+  for (let i = 0; i < results.length; i++) {
+    const result = results[i];
+    const title =
+      result.title?.trim() || result.url?.trim() || "Untitled result";
+    lines.push(`${i + 1}. ${title}`);
+    if (result.url) {
+      lines.push(`   URL: ${result.url}`);
+    }
+    if (result.snippet) {
+      lines.push(`   ${result.snippet}`);
+    }
+    if (result.date) {
+      lines.push(`   Published: ${result.date}`);
+    }
+    lines.push("");
+  }
+  return lines.join("\n");
+}
+
+function buildTinyfishMetadata(
+  data: TinyfishSearchResponse,
+  query: string,
+  durationMs: number,
+): WebSearchMetadata {
+  const results = data.results ?? [];
+  const items: WebSearchResultItem[] = results.map((result, index) => {
+    const url = result.url ?? "";
+    const domain = extractDomain(url);
+    return {
+      rank: index + 1,
+      title: result.title?.trim() || url.trim() || "Untitled result",
+      url,
+      domain,
+      faviconUrl: faviconUrlForDomain(domain),
+      snippet: result.snippet?.trim() || undefined,
+    };
+  });
+  return {
+    query,
+    provider: "tinyfish",
+    resultCount: items.length,
+    durationMs,
+    results: items,
+  };
+}
+
+function tinyfishRecencyMinutesForFreshness(
+  freshness: string | undefined,
+): number | undefined {
+  switch (freshness) {
+    case "pd":
+      return 24 * 60;
+    case "pw":
+      return 7 * 24 * 60;
+    case "pm":
+      return 30 * 24 * 60;
+    case "py":
+      return 365 * 24 * 60;
+    default:
+      return undefined;
+  }
+}
+
+/**
+ * Map the tool's `freshness` window onto SearXNG's `time_range`
+ * (`day` / `month` / `year`). SearXNG has no week filter, so `pw` is omitted.
+ */
+function searxngTimeRangeForFreshness(
+  freshness: string | undefined,
+): "day" | "month" | "year" | undefined {
+  switch (freshness) {
+    case "pd":
+      return "day";
+    case "pm":
+      return "month";
+    case "py":
+      return "year";
+    default:
+      return undefined;
+  }
+}
+
+function isNonJsonSearchBody(contentType: string, bodyText: string): boolean {
+  const type = contentType.toLowerCase();
+  if (type.includes("text/html") || type.includes("application/xhtml")) {
+    return true;
+  }
+  const trimmed = bodyText.trimStart();
+  return trimmed.startsWith("<");
+}
+
 /**
  * Map the tool's `freshness` window onto a Keenable `published_after` date
  * (YYYY-MM-DD). Keenable filters by absolute publication date rather than a
@@ -1358,6 +1546,261 @@ async function executeKeenableSearch(
   );
 }
 
+async function executeSearxngSearch(
+  query: string,
+  count: number,
+  freshness: string | undefined,
+  apiKey: string,
+  signal?: AbortSignal,
+): Promise<ToolExecutionResult> {
+  const startedAt = Date.now();
+  const apiBase = getConfig().services["web-search"]?.apiBase?.trim() ?? "";
+  if (apiBase.length === 0) {
+    return errorResult(
+      query,
+      "searxng",
+      startedAt,
+      SEARXNG_MISSING_INSTANCE_MESSAGE,
+    );
+  }
+
+  const params = new URLSearchParams({
+    q: query,
+    format: "json",
+  });
+  const timeRange = searxngTimeRangeForFreshness(freshness);
+  if (timeRange) {
+    params.set("time_range", timeRange);
+  }
+  const url = `${resolveProviderApiUrl(apiBase, "/search", apiBase)}?${params.toString()}`;
+
+  const headers: Record<string, string> = {
+    Accept: "application/json",
+  };
+  const trimmedKey = apiKey.trim();
+  if (trimmedKey.length > 0) {
+    headers.Authorization = `Bearer ${trimmedKey}`;
+  }
+
+  for (let attempt = 0; attempt <= DEFAULT_MAX_RETRIES; attempt++) {
+    let response: Response;
+    try {
+      response = await fetch(url, {
+        headers,
+        signal,
+      });
+    } catch (err) {
+      return networkFailureResult(query, "searxng", startedAt, err, signal);
+    }
+
+    const bodyText = await response.text();
+    const contentType = response.headers.get("content-type") ?? "";
+    if (isNonJsonSearchBody(contentType, bodyText)) {
+      return errorResult(
+        query,
+        "searxng",
+        startedAt,
+        SEARXNG_JSON_DISABLED_MESSAGE,
+      );
+    }
+
+    if (response.ok) {
+      let data: SearxngSearchResponse;
+      try {
+        data = JSON.parse(bodyText) as SearxngSearchResponse;
+      } catch {
+        return errorResult(
+          query,
+          "searxng",
+          startedAt,
+          SEARXNG_JSON_DISABLED_MESSAGE,
+        );
+      }
+      if (data.results && data.results.length > count) {
+        data.results = data.results.slice(0, count);
+      }
+      const durationMs = Date.now() - startedAt;
+      return {
+        content:
+          wrapUntrustedContent(formatSearxngResults(data, query), {
+            source: "search",
+            sourceDetail: "searxng",
+          }) + CITATION_INSTRUCTION,
+        isError: false,
+        activityMetadata: {
+          webSearch: buildSearxngMetadata(data, query, durationMs),
+        },
+      };
+    }
+
+    if (response.status === 401 || response.status === 403) {
+      return errorResult(
+        query,
+        "searxng",
+        startedAt,
+        "SearXNG rejected the request. Check the instance URL, optional token, and that JSON search is enabled.",
+      );
+    }
+
+    if (response.status === 429 && attempt < DEFAULT_MAX_RETRIES) {
+      const delayMs = getHttpRetryDelay(
+        response,
+        attempt,
+        DEFAULT_BASE_DELAY_MS,
+      );
+      log.warn(
+        { attempt: attempt + 1, delayMs },
+        "SearXNG Search rate limited, retrying",
+      );
+      await sleep(delayMs);
+      continue;
+    }
+
+    log.warn({ status: response.status }, "SearXNG Search API error");
+    return backendFailureResult(
+      query,
+      "searxng",
+      startedAt,
+      { statusCode: response.status, error: rawBodyDetail(bodyText) },
+      response.status === 429
+        ? "SearXNG Search rate limit exceeded after retries. Try again shortly."
+        : `SearXNG Search API returned status ${response.status}`,
+    );
+  }
+
+  return backendFailureResult(
+    query,
+    "searxng",
+    startedAt,
+    { statusCode: 429 },
+    "SearXNG Search rate limit exceeded after retries. Try again shortly.",
+  );
+}
+
+async function executeTinyfishSearch(
+  query: string,
+  count: number,
+  freshness: string | undefined,
+  apiKey: string,
+  signal?: AbortSignal,
+): Promise<ToolExecutionResult> {
+  const startedAt = Date.now();
+  const apiBase = getConfig().services["web-search"]?.apiBase;
+  const endpoint = resolveProviderApiUrl(
+    apiBase,
+    "/",
+    TINYFISH_DEFAULT_SEARCH_API_BASE,
+  );
+  const url = new URL(endpoint);
+  url.searchParams.set("query", query);
+  const recencyMinutes = tinyfishRecencyMinutesForFreshness(freshness);
+  if (recencyMinutes !== undefined) {
+    url.searchParams.set("recency_minutes", String(recencyMinutes));
+  }
+
+  const headers = {
+    Accept: "application/json",
+    "X-API-Key": apiKey.trim(),
+  };
+
+  for (let attempt = 0; attempt <= DEFAULT_MAX_RETRIES; attempt++) {
+    let response: Response;
+    try {
+      response = await fetch(url.toString(), { headers, signal });
+    } catch (err) {
+      return networkFailureResult(query, "tinyfish", startedAt, err, signal);
+    }
+
+    const bodyText = await response.text();
+    if (response.ok) {
+      let data: TinyfishSearchResponse;
+      try {
+        data = JSON.parse(bodyText) as TinyfishSearchResponse;
+      } catch {
+        return errorResult(
+          query,
+          "tinyfish",
+          startedAt,
+          "TinyFish Search returned an invalid JSON payload.",
+        );
+      }
+      if (data.results && data.results.length > count) {
+        data.results = data.results.slice(0, count);
+      }
+      const durationMs = Date.now() - startedAt;
+      return {
+        content:
+          wrapUntrustedContent(formatTinyfishResults(data, query), {
+            source: "search",
+            sourceDetail: "tinyfish",
+          }) + CITATION_INSTRUCTION,
+        isError: false,
+        activityMetadata: {
+          webSearch: buildTinyfishMetadata(data, query, durationMs),
+        },
+      };
+    }
+
+    if (response.status === 401) {
+      return errorResult(
+        query,
+        "tinyfish",
+        startedAt,
+        "Invalid or expired TinyFish API key",
+      );
+    }
+    if (response.status === 402) {
+      return errorResult(
+        query,
+        "tinyfish",
+        startedAt,
+        "TinyFish Search API access is not enabled for this account.",
+      );
+    }
+    if (response.status === 403) {
+      return errorResult(
+        query,
+        "tinyfish",
+        startedAt,
+        "TinyFish Search request was forbidden by the upstream service.",
+      );
+    }
+
+    if (response.status === 429 && attempt < DEFAULT_MAX_RETRIES) {
+      const delayMs = getHttpRetryDelay(
+        response,
+        attempt,
+        DEFAULT_BASE_DELAY_MS,
+      );
+      log.warn(
+        { attempt: attempt + 1, delayMs },
+        "TinyFish Search rate limited, retrying",
+      );
+      await sleep(delayMs);
+      continue;
+    }
+
+    log.warn({ status: response.status }, "TinyFish Search API error");
+    return backendFailureResult(
+      query,
+      "tinyfish",
+      startedAt,
+      { statusCode: response.status, error: rawBodyDetail(bodyText) },
+      response.status === 429
+        ? "TinyFish Search rate limit exceeded after retries. Try again shortly."
+        : `TinyFish Search API returned status ${response.status}`,
+    );
+  }
+
+  return backendFailureResult(
+    query,
+    "tinyfish",
+    startedAt,
+    { statusCode: 429 },
+    "TinyFish Search rate limit exceeded after retries. Try again shortly.",
+  );
+}
+
 // ----------------------------------------------------------------------------
 // Adapter registry
 //
@@ -1424,6 +1867,23 @@ const fastcrwSearchAdapter: WebSearchAdapter = {
     executeFastcrwSearch(query, count, freshness, apiKey, signal),
 };
 
+const searxngSearchAdapter: WebSearchAdapter = {
+  id: "searxng",
+  providerKeyName: "searxng",
+  fallbackOrder: 7,
+  keyless: true,
+  execute: ({ query, count, freshness, apiKey, signal }) =>
+    executeSearxngSearch(query, count, freshness, apiKey, signal),
+};
+
+const tinyfishSearchAdapter: WebSearchAdapter = {
+  id: "tinyfish",
+  providerKeyName: "tinyfish",
+  fallbackOrder: 8,
+  execute: ({ query, count, freshness, apiKey, signal }) =>
+    executeTinyfishSearch(query, count, freshness, apiKey, signal),
+};
+
 /**
  * All built-in web-search adapters keyed by provider id. The
  * `Record<WebSearchProvider, ...>` shape forces TypeScript to flag any
@@ -1436,6 +1896,8 @@ const WEB_SEARCH_ADAPTERS: Record<WebSearchProvider, WebSearchAdapter> = {
   firecrawl: firecrawlSearchAdapter,
   keenable: keenableSearchAdapter,
   fastcrw: fastcrwSearchAdapter,
+  searxng: searxngSearchAdapter,
+  tinyfish: tinyfishSearchAdapter,
 };
 
 /**
@@ -1467,7 +1929,7 @@ export const webSearchTool = {
       count: {
         type: "number",
         description:
-          "Number of results to return (1-20, default 10). Used with Brave, Tavily, Firecrawl, Keenable, and fastCRW providers.",
+          "Number of results to return (1-20, default 10). Used with Brave, Tavily, Firecrawl, Keenable, fastCRW, SearXNG, and TinyFish providers.",
       },
       offset: {
         type: "number",
@@ -1477,7 +1939,7 @@ export const webSearchTool = {
       freshness: {
         type: "string",
         description:
-          'Filter by recency: "pd" (past day), "pw" (past week), "pm" (past month), "py" (past year). Used with Brave, Tavily, Firecrawl, Keenable, and fastCRW providers.',
+          'Filter by recency: "pd" (past day), "pw" (past week), "pm" (past month), "py" (past year). Used with Brave, Tavily, Firecrawl, Keenable, fastCRW, SearXNG, and TinyFish providers. SearXNG maps day/month/year and omits week.',
       },
     },
     required: ["query"],
@@ -1593,7 +2055,7 @@ export const webSearchTool = {
           query,
           provider,
           startedAt,
-          "No web search API key configured. Set it via `keys set perplexity <key>`, `keys set brave <key>`, `keys set tavily <key>`, `keys set firecrawl <key>`, or `keys set fastcrw <key>`, or configure it from the Settings page under API Keys. Or switch the web-search provider to Keenable, which works without a key.",
+          "No web search API key configured. Set it via `keys set perplexity <key>`, `keys set brave <key>`, `keys set tavily <key>`, `keys set firecrawl <key>`, or `keys set fastcrw <key>`, or `keys set tinyfish <key>`, or configure it from the Settings page under API Keys. Or switch the web-search provider to Keenable, which works without a key, or SearXNG with your instance URL.",
         );
       }
     }

package/src/tools/permission-checker.ts

@@ -119,8 +119,8 @@ export class PermissionChecker {
     >,
     classification?: RiskClassificationWithMeta,
   ): Promise<PermissionDecision> {
-    // Sandbox/host routing for the prompt comes from the tool's manifest.
-    const executionTarget = resolveExecutionTarget(tool);
+    // Policy and prompts use the invocation's execution target.
+    const executionTarget = resolveExecutionTarget(tool, input, context);
     classification ??= await classifyRisk(
       name,
       input,
@@ -149,7 +149,7 @@ export class PermissionChecker {
     // this, the executor's catch block would fall back to the default
     // low risk, degrading audit/alert accuracy for high-risk attempts.
     try {
-      const policyContext = buildPolicyContext(tool, context);
+      const policyContext = buildPolicyContext(tool, context, input);
       const result = await check(
         name,
         input,

package/src/tools/policy-context.ts

@@ -3,6 +3,7 @@ import { isSkillImprovementActive } from "../config/memory-v3-gate.js";
 import type { ExecutionContext } from "../permissions/approval-policy.js";
 import type { ChannelPermissionCoordinates } from "../permissions/channel-permission-query.js";
 import type { PolicyContext } from "../permissions/types.js";
+import { resolveExecutionTarget } from "./execution-target.js";
 import { getToolOwner } from "./registry.js";
 import type { Tool, ToolContext } from "./types.js";
 
@@ -48,6 +49,7 @@ function deriveExecutionContext(context?: ToolContext): ExecutionContext {
 export function buildPolicyContext(
   tool: Tool,
   context?: ToolContext,
+  input?: Record<string, unknown>,
 ): PolicyContext {
   const executionContext = deriveExecutionContext(context);
 
@@ -73,7 +75,7 @@ export function buildPolicyContext(
   const ownerKind = getToolOwner(tool.name)?.kind;
   if (ownerKind === "skill" || ownerKind === "plugin") {
     return {
-      executionTarget: tool.executionTarget,
+      executionTarget: resolveExecutionTarget(tool, input, context),
       executionContext,
       conversationId,
       ...originSignals,

package/src/tools/shared/filesystem/file-ops-service.ts

@@ -4,6 +4,7 @@ import { dirname, join } from "node:path";
 import { minimatch } from "minimatch";
 
 import { ensureDir, pathExists } from "../../../util/fs.js";
+import { surrogateSafeWindow } from "../../../util/unicode.js";
 import { isAbortLikeError } from "../abort.js";
 import { applyEdit } from "./edit-engine.js";
 import * as Err from "./errors.js";
@@ -102,37 +103,6 @@ function truncationNotice(
   return `\n\n[Truncated: characters ${start}-${end} of ${totalChars}. Read on with start_index=${end}.]`;
 }
 
-const isHighSurrogate = (code: number): boolean =>
-  code >= 0xd800 && code <= 0xdbff;
-const isLowSurrogate = (code: number): boolean =>
-  code >= 0xdc00 && code <= 0xdfff;
-
-/**
- * Character window that never splits a surrogate pair. A split leaves a lone
- * half at each edge, and each encodes to U+FFFD, so the character is lost from
- * both this window and the next one paged in after it.
- */
-export function surrogateSafeWindow(
-  total: number,
-  charCodeAt: (index: number) => number,
-  requestedStart: number,
-  maxChars: number,
-): { start: number; end: number } {
-  let start = Math.max(0, Math.min(requestedStart, total));
-  if (start > 0 && start < total && isLowSurrogate(charCodeAt(start))) {
-    start -= 1;
-  }
-
-  let end = Math.min(total, start + maxChars);
-  if (end > start && end < total && isHighSurrogate(charCodeAt(end - 1))) {
-    // Backing off would empty a one-character window, which stalls paging on
-    // the same offset, so take the whole pair instead.
-    end = end - 1 > start ? end - 1 : Math.min(total, end + 1);
-  }
-
-  return { start, end };
-}
-
 export class FileSystemOps {
   private policy: PathPolicy;
   private sizeLimit: number | undefined;

package/src/tools/shared/shell-output.test.ts

@@ -7,6 +7,7 @@ import {
   formatShellOutput,
   MAX_OUTPUT_LENGTH,
   OUTPUT_TRUNCATED_TAG,
+  SHELL_DID_NOT_START_MESSAGE,
 } from "./shell-output.js";
 
 describe("BoundedStdioCollector", () => {
@@ -85,6 +86,15 @@ describe("attachBoundedStdio", () => {
   });
 });
 
+describe("formatShellOutput launch start", () => {
+  test("does not report command_completed when the process never started", () => {
+    const result = formatShellOutput("", "", 0, false, 120, { started: false });
+    expect(result.isError).toBe(true);
+    expect(result.content).toBe(SHELL_DID_NOT_START_MESSAGE);
+    expect(result.content).not.toContain("<command_completed />");
+  });
+});
+
 describe("formatShellOutput truncation", () => {
   test("truncates an already-materialized oversized string without a file path", () => {
     const longOutput = "x".repeat(30_000);

package/src/tools/shared/shell-output.ts

@@ -4,6 +4,9 @@ export const MAX_OUTPUT_LENGTH = 20_000;
 
 export const OUTPUT_TRUNCATED_TAG = `<output_truncated limit="20K" />`;
 
+export const SHELL_DID_NOT_START_MESSAGE =
+  "Error: the shell command did not start. No process was created, so the command did not run.";
+
 export interface ShellOutputResult {
   content: string;
   status: string | undefined;
@@ -31,8 +34,16 @@ export function formatShellOutput(
   code: number | null,
   timedOut: boolean,
   timeoutSec: number,
-  options?: { truncated?: boolean },
+  options?: { truncated?: boolean; started?: boolean },
 ): ShellOutputResult {
+  if (options?.started === false) {
+    return {
+      content: SHELL_DID_NOT_START_MESSAGE,
+      status: undefined,
+      isError: true,
+    };
+  }
+
   let output = stdout;
   if (stderr) {
     output += (output ? "\n" : "") + stderr;
@@ -137,6 +148,7 @@ export class BoundedStdioCollector {
     code: number | null,
     timedOut: boolean,
     timeoutSec: number,
+    options?: { started?: boolean },
   ): ShellOutputResult {
     return formatShellOutput(
       Buffer.concat(this.stdoutParts).toString(),
@@ -144,7 +156,7 @@ export class BoundedStdioCollector {
       code,
       timedOut,
       timeoutSec,
-      { truncated: this.truncated },
+      { truncated: this.truncated, started: options?.started },
     );
   }
 }