@vellumai/assistant 0.9.1-staging.1 → 0.10.0-staging.2

package/src/tools/network/__tests__/web-fetch-firecrawl.test.ts

@@ -0,0 +1,360 @@
+import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
+
+// --- Mutable mock state (per test) -----------------------------------------
+let mockWebFetchProvider: string | undefined = "default";
+let mockFirecrawlSecureKey: string | undefined;
+
+// Capture the registered tool so we can exercise provider dispatch.
+let capturedTool: any = null;
+
+mock.module("../../registry.js", () => ({
+  registerTool: (tool: any) => {
+    capturedTool = tool;
+  },
+}));
+
+mock.module("../../../config/loader.js", () => ({
+  getConfig: () => ({
+    services: {
+      "web-fetch": { mode: "your-own", provider: mockWebFetchProvider },
+    },
+  }),
+}));
+
+mock.module("../../../security/secure-keys.js", () => ({
+  getProviderKeyAsync: async (provider: string) =>
+    provider === "firecrawl" ? mockFirecrawlSecureKey : undefined,
+}));
+
+const realLogger = await import("../../../util/logger.js");
+mock.module("../../../util/logger.js", () => ({
+  ...realLogger,
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, { get: () => () => {} }),
+}));
+
+mock.module("../../../permissions/types.js", () => ({
+  RiskLevel: { Low: "low", Medium: "medium", High: "high" },
+}));
+
+// Keep real url-safety helpers (parseUrl, sanitize*, isPrivateOrLocalHost,
+// resolveRequestAddress) but stub DNS resolution. The returned address list is
+// mutable per test: an empty list makes the built-in fallback path
+// short-circuit ("Unable to resolve host") before any socket is opened, while a
+// public address lets a Firecrawl-routed request through the dispatcher's DNS
+// safety gate. A private address exercises the "don't leak to Firecrawl" guard.
+let mockResolveAddresses: string[] = [];
+const realUrlSafety = await import("../url-safety.js");
+mock.module("../url-safety.js", () => ({
+  ...realUrlSafety,
+  resolveHostAddresses: async () => mockResolveAddresses,
+}));
+
+const { executeFirecrawlScrape } = await import("../web-fetch.js");
+
+const SCRAPE_URL = "api.firecrawl.dev/v2/scrape";
+
+function scrapeResponse(body: unknown, status = 200): Response {
+  return new Response(JSON.stringify(body), {
+    status,
+    headers: { "content-type": "application/json" },
+  });
+}
+
+describe("executeFirecrawlScrape", () => {
+  let originalFetch: typeof globalThis.fetch;
+
+  beforeEach(() => {
+    originalFetch = globalThis.fetch;
+  });
+  afterEach(() => {
+    globalThis.fetch = originalFetch;
+  });
+
+  test("returns markdown content + metadata on success", async () => {
+    globalThis.fetch = (async () =>
+      scrapeResponse({
+        success: true,
+        data: {
+          markdown: "# Example\n\nHello from Firecrawl.",
+          metadata: {
+            title: "Example Domain",
+            description: "An example page",
+            url: "https://example.com/",
+            statusCode: 200,
+            contentType: "text/html",
+          },
+        },
+      })) as any;
+
+    const result = await executeFirecrawlScrape(
+      { url: "https://example.com" },
+      { apiKey: "fc-test-key" },
+    );
+
+    expect(result.isError).toBe(false);
+    expect(result.content).toContain("Hello from Firecrawl.");
+    expect(result.content).toContain("Mode: markdown");
+    expect(result.content).toContain("Title: Example Domain");
+    const meta = result.activityMetadata?.webFetch;
+    expect(meta?.provider).toBe("firecrawl");
+    expect(meta?.status).toBe(200);
+    expect(meta?.title).toBe("Example Domain");
+    expect(meta?.finalUrl).toContain("example.com");
+  });
+
+  test("sends the correct request shape", async () => {
+    let capturedUrl = "";
+    let capturedBody: any = null;
+    let capturedHeaders: Headers | null = null;
+    globalThis.fetch = (async (url: string, init?: RequestInit) => {
+      capturedUrl = url;
+      capturedBody = JSON.parse(init?.body as string);
+      capturedHeaders = new Headers(init?.headers);
+      return scrapeResponse({ success: true, data: { markdown: "ok" } });
+    }) as any;
+
+    await executeFirecrawlScrape(
+      { url: "https://example.com/docs" },
+      { apiKey: "fc-test-key" },
+    );
+
+    expect(capturedUrl).toContain(SCRAPE_URL);
+    expect(capturedBody.url).toBe("https://example.com/docs");
+    expect(capturedBody.formats).toEqual(["markdown"]);
+    expect(capturedBody.onlyMainContent).toBe(true);
+    expect(capturedHeaders!.get("authorization")).toBe("Bearer fc-test-key");
+    expect(capturedHeaders!.get("x-client-source")).toBe("vellum-assistant");
+  });
+
+  test("honors max_chars windowing and emits a truncation notice", async () => {
+    globalThis.fetch = (async () =>
+      scrapeResponse({
+        success: true,
+        data: { markdown: "abcdefghij", metadata: { statusCode: 200 } },
+      })) as any;
+
+    const result = await executeFirecrawlScrape(
+      { url: "https://example.com", max_chars: 4 },
+      { apiKey: "fc-key" },
+    );
+    expect(result.isError).toBe(false);
+    expect(result.activityMetadata?.webFetch?.truncated).toBe(true);
+    expect(result.status).toContain("truncated");
+  });
+
+  test("empty markdown yields a no-content marker, not an error", async () => {
+    globalThis.fetch = (async () =>
+      scrapeResponse({ success: true, data: { markdown: "" } })) as any;
+
+    const result = await executeFirecrawlScrape(
+      { url: "https://example.com" },
+      { apiKey: "fc-key" },
+    );
+    expect(result.isError).toBe(false);
+    expect(result.content).toContain("<no_content />");
+  });
+
+  test("rejects URLs with embedded credentials instead of forwarding them", async () => {
+    let hit = false;
+    globalThis.fetch = (async () => {
+      hit = true;
+      return scrapeResponse({ success: true, data: { markdown: "x" } });
+    }) as any;
+
+    const result = await executeFirecrawlScrape(
+      { url: "https://user:secret@example.com/page" },
+      { apiKey: "fc-key" },
+    );
+    expect(hit).toBe(false); // never sent to Firecrawl
+    expect(result.isError).toBe(true);
+    expect(result.content).toContain("embedded credentials");
+    // The secret must not leak into the surfaced url/metadata.
+    expect(result.content).not.toContain("secret");
+  });
+
+  test("surfaces a payload-level failure on a 200 (success:false / error)", async () => {
+    globalThis.fetch = (async () =>
+      scrapeResponse({
+        success: false,
+        error: "This website is no longer supported",
+        data: {},
+      })) as any;
+
+    const result = await executeFirecrawlScrape(
+      { url: "https://example.com" },
+      { apiKey: "fc-key" },
+    );
+    expect(result.isError).toBe(true);
+    expect(result.content).toContain("no longer supported");
+  });
+
+  test("surfaces invalid JSON as a clean error", async () => {
+    globalThis.fetch = (async () =>
+      new Response("<html>not json</html>", {
+        status: 200,
+        headers: { "content-type": "application/json" },
+      })) as any;
+
+    const result = await executeFirecrawlScrape(
+      { url: "https://example.com" },
+      { apiKey: "fc-key" },
+    );
+    expect(result.isError).toBe(true);
+    expect(result.content).toContain("invalid JSON");
+  });
+
+  test.each([401, 403])("surfaces %d as an invalid-key error", async (status) => {
+    globalThis.fetch = (async () =>
+      new Response("Unauthorized", { status })) as any;
+
+    const result = await executeFirecrawlScrape(
+      { url: "https://example.com" },
+      { apiKey: "bad-key" },
+    );
+    expect(result.isError).toBe(true);
+    expect(result.content).toContain("Invalid or expired Firecrawl API key");
+    expect(result.activityMetadata?.webFetch?.provider).toBe("firecrawl");
+  });
+
+  test("retries 429 then surfaces a rate-limit error", async () => {
+    let callCount = 0;
+    globalThis.fetch = (async () => {
+      callCount++;
+      return new Response("Too Many Requests", {
+        status: 429,
+        headers: { "retry-after": "0" },
+      });
+    }) as any;
+
+    const result = await executeFirecrawlScrape(
+      { url: "https://example.com" },
+      { apiKey: "fc-key" },
+    );
+    expect(result.isError).toBe(true);
+    expect(result.content).toContain("rate limit");
+    expect(callCount).toBe(4); // 1 + DEFAULT_MAX_RETRIES
+  });
+});
+
+describe("web_fetch provider dispatch", () => {
+  let originalFetch: typeof globalThis.fetch;
+
+  const execute = (input: Record<string, unknown>, ctx: any = {}) =>
+    capturedTool.execute(input, ctx);
+
+  beforeEach(() => {
+    originalFetch = globalThis.fetch;
+    mockWebFetchProvider = "default";
+    mockFirecrawlSecureKey = undefined;
+    mockResolveAddresses = [];
+  });
+  afterEach(() => {
+    globalThis.fetch = originalFetch;
+  });
+
+  test("routes to Firecrawl when provider=firecrawl and a key is set", async () => {
+    mockWebFetchProvider = "firecrawl";
+    mockFirecrawlSecureKey = "fc-key";
+    mockResolveAddresses = ["93.184.216.34"]; // public → passes the DNS safety gate
+    let hitUrl = "";
+    globalThis.fetch = (async (url: string) => {
+      hitUrl = url;
+      return scrapeResponse({
+        success: true,
+        data: { markdown: "routed", metadata: { statusCode: 200 } },
+      });
+    }) as any;
+
+    const result = await execute({ url: "https://example.com" });
+    expect(hitUrl).toContain(SCRAPE_URL);
+    expect(result.activityMetadata?.webFetch?.provider).toBe("firecrawl");
+  });
+
+  test("falls back to the built-in fetcher when provider=firecrawl but no key", async () => {
+    mockWebFetchProvider = "firecrawl";
+    mockFirecrawlSecureKey = undefined;
+    let firecrawlHit = false;
+    globalThis.fetch = (async (url: string) => {
+      if (typeof url === "string" && url.includes(SCRAPE_URL)) {
+        firecrawlHit = true;
+      }
+      return new Response("", { status: 200 });
+    }) as any;
+
+    const result = await execute({ url: "https://example.com" });
+    // Built-in path runs (DNS stubbed to empty → resolve error), Firecrawl not hit.
+    expect(firecrawlHit).toBe(false);
+    expect(result.activityMetadata?.webFetch?.provider).toBe("default");
+  });
+
+  test("provider=default never touches Firecrawl", async () => {
+    mockWebFetchProvider = "default";
+    mockFirecrawlSecureKey = "fc-key";
+    let firecrawlHit = false;
+    globalThis.fetch = (async (url: string) => {
+      if (typeof url === "string" && url.includes(SCRAPE_URL)) {
+        firecrawlHit = true;
+      }
+      return new Response("", { status: 200 });
+    }) as any;
+
+    const result = await execute({ url: "https://example.com" });
+    expect(firecrawlHit).toBe(false);
+    expect(result.activityMetadata?.webFetch?.provider).toBe("default");
+  });
+
+  test("private/local targets bypass Firecrawl and use the built-in fetcher", async () => {
+    mockWebFetchProvider = "firecrawl";
+    mockFirecrawlSecureKey = "fc-key";
+    let firecrawlHit = false;
+    globalThis.fetch = (async (url: string) => {
+      if (typeof url === "string" && url.includes(SCRAPE_URL)) {
+        firecrawlHit = true;
+      }
+      return new Response("", { status: 200 });
+    }) as any;
+
+    const result = await execute({ url: "http://localhost:8080/admin" });
+    expect(firecrawlHit).toBe(false);
+    expect(result.isError).toBe(true);
+    expect(result.content.toLowerCase()).toContain("private");
+    expect(result.activityMetadata?.webFetch?.provider).toBe("default");
+  });
+
+  test("a public host that DNS-resolves to a private IP is not sent to Firecrawl", async () => {
+    mockWebFetchProvider = "firecrawl";
+    mockFirecrawlSecureKey = "fc-key";
+    mockResolveAddresses = ["10.0.0.5"]; // public name, private address → blocked
+    let firecrawlHit = false;
+    globalThis.fetch = (async (url: string) => {
+      if (typeof url === "string" && url.includes(SCRAPE_URL)) {
+        firecrawlHit = true;
+      }
+      return new Response("", { status: 200 });
+    }) as any;
+
+    const result = await execute({ url: "https://internal.example/secret?token=abc" });
+    expect(firecrawlHit).toBe(false); // internal URL never leaked to Firecrawl
+    expect(result.isError).toBe(true);
+    expect(result.activityMetadata?.webFetch?.provider).toBe("default");
+  });
+
+  test("non-http(s) schemes are not sent to Firecrawl", async () => {
+    mockWebFetchProvider = "firecrawl";
+    mockFirecrawlSecureKey = "fc-key";
+    mockResolveAddresses = ["93.184.216.34"];
+    let firecrawlHit = false;
+    globalThis.fetch = (async (url: string) => {
+      if (typeof url === "string" && url.includes(SCRAPE_URL)) {
+        firecrawlHit = true;
+      }
+      return new Response("", { status: 200 });
+    }) as any;
+
+    const result = await execute({ url: "ftp://example.com/file" });
+    expect(firecrawlHit).toBe(false);
+    expect(result.isError).toBe(true);
+    expect(result.activityMetadata?.webFetch?.provider).toBe("default");
+  });
+});

package/src/tools/network/__tests__/web-search.test.ts

@@ -8,6 +8,7 @@ let mockWebSearchMode: string | undefined = "your-own";
 let mockBraveSecureKey: string | undefined;
 let mockPerplexitySecureKey: string | undefined;
 let mockTavilySecureKey: string | undefined;
+let mockFirecrawlSecureKey: string | undefined;
 let mockManagedSearchProxyResult: any;
 let mockManagedSearchProxyCalls: Array<{
   provider: string;
@@ -40,6 +41,7 @@ mock.module("../../../security/secure-keys.js", () => ({
     if (provider === "brave") return mockBraveSecureKey;
     if (provider === "perplexity") return mockPerplexitySecureKey;
     if (provider === "tavily") return mockTavilySecureKey;
+    if (provider === "firecrawl") return mockFirecrawlSecureKey;
     return undefined;
   },
 }));
@@ -81,6 +83,7 @@ describe("web_search tool", () => {
     mockBraveSecureKey = undefined;
     mockPerplexitySecureKey = undefined;
     mockTavilySecureKey = undefined;
+    mockFirecrawlSecureKey = undefined;
     mockManagedSearchProxyCalls = [];
     mockManagedSearchProxyResult = {
       ok: true,
@@ -685,6 +688,129 @@ describe("web_search tool", () => {
     expect(callCount).toBe(4);
   });
 
+  // ---- Firecrawl provider -------------------------------------------------
+
+  test("executes Firecrawl search successfully", async () => {
+    mockWebSearchProvider = "firecrawl";
+    mockFirecrawlSecureKey = "fc-test-key";
+    globalThis.fetch = (async () => {
+      return new Response(
+        JSON.stringify({
+          success: true,
+          data: {
+            web: [
+              {
+                title: "Firecrawl Result 1",
+                url: "https://example.com/fc-1",
+                description: "First Firecrawl result",
+              },
+              {
+                title: "Firecrawl Result 2",
+                url: "https://example.com/fc-2",
+                description: "Second Firecrawl result",
+              },
+            ],
+          },
+        }),
+        { status: 200, headers: { "content-type": "application/json" } },
+      );
+    }) as any;
+
+    const result = await execute({ query: "what is TypeScript" });
+    expect(result.isError).toBe(false);
+    expect(result.content).toContain("Firecrawl Result 1");
+    expect(result.content).toContain("https://example.com/fc-1");
+    expect(result.content).toContain("First Firecrawl result");
+  });
+
+  test("Firecrawl sends correct request format", async () => {
+    mockWebSearchProvider = "firecrawl";
+    mockFirecrawlSecureKey = "fc-test-key";
+    let capturedUrl = "";
+    let capturedBody: any = null;
+    let capturedHeaders: any = null;
+    globalThis.fetch = (async (url: string, init?: RequestInit) => {
+      capturedUrl = url;
+      capturedBody = JSON.parse(init?.body as string);
+      capturedHeaders = new Headers(init?.headers);
+      return new Response(JSON.stringify({ success: true, data: { web: [] } }), {
+        status: 200,
+        headers: { "content-type": "application/json" },
+      });
+    }) as any;
+
+    await execute({ query: "test query", count: 50, freshness: "pm" });
+    expect(capturedUrl).toContain("api.firecrawl.dev/v2/search");
+    expect(capturedBody.query).toBe("test query");
+    expect(capturedBody.limit).toBe(20);
+    expect(capturedBody.sources).toEqual(["web"]);
+    expect(capturedBody.tbs).toBe("qdr:m");
+    expect(capturedHeaders.get("authorization")).toBe("Bearer fc-test-key");
+    expect(capturedHeaders.get("x-client-source")).toBe("vellum-assistant");
+  });
+
+  test("Firecrawl skips invalid freshness values", async () => {
+    mockWebSearchProvider = "firecrawl";
+    mockFirecrawlSecureKey = "fc-key";
+    let capturedBody: any = null;
+    globalThis.fetch = (async (_url: string, init?: RequestInit) => {
+      capturedBody = JSON.parse(init?.body as string);
+      return new Response(JSON.stringify({ success: true, data: { web: [] } }), {
+        status: 200,
+        headers: { "content-type": "application/json" },
+      });
+    }) as any;
+
+    await execute({ query: "test", freshness: "invalid" });
+    expect(capturedBody.tbs).toBeUndefined();
+  });
+
+  test("Firecrawl returns no results message when response is empty", async () => {
+    mockWebSearchProvider = "firecrawl";
+    mockFirecrawlSecureKey = "fc-key";
+    globalThis.fetch = (async () => {
+      return new Response(JSON.stringify({ success: true, data: { web: [] } }), {
+        status: 200,
+        headers: { "content-type": "application/json" },
+      });
+    }) as any;
+
+    const result = await execute({ query: "obscure query" });
+    expect(result.isError).toBe(false);
+    expect(result.content).toContain("No results found");
+  });
+
+  test.each([401, 403])("Firecrawl handles %d auth error", async (status) => {
+    mockWebSearchProvider = "firecrawl";
+    mockFirecrawlSecureKey = "bad-key";
+    globalThis.fetch = (async () => {
+      return new Response("Auth error", { status });
+    }) as any;
+
+    const result = await execute({ query: "test" });
+    expect(result.isError).toBe(true);
+    expect(result.content).toContain("Invalid or expired Firecrawl API key");
+  });
+
+  test("Firecrawl handles 429 rate limit after max retries", async () => {
+    mockWebSearchProvider = "firecrawl";
+    mockFirecrawlSecureKey = "fc-key";
+    let callCount = 0;
+    globalThis.fetch = (async () => {
+      callCount++;
+      return new Response("Too Many Requests", {
+        status: 429,
+        headers: { "retry-after": "0" },
+      });
+    }) as any;
+
+    const result = await execute({ query: "test" });
+    expect(result.isError).toBe(true);
+    // Post-retry rate limits surface the friendly recoverable copy (ATL-727).
+    expect(result.content).toBe(WEB_SEARCH_BACKEND_FAILURE_MESSAGE);
+    expect(callCount).toBe(4);
+  });
+
   // ---- Provider fallback --------------------------------------------------
 
   test("falls back from perplexity to brave when perplexity has no key", async () => {
@@ -740,6 +866,23 @@ describe("web_search tool", () => {
     expect(capturedUrl).toContain("tavily");
   });
 
+  test("falls back to firecrawl when all earlier providers have no key", async () => {
+    mockWebSearchProvider = "perplexity";
+    mockFirecrawlSecureKey = "fc-fallback-key";
+    let capturedUrl = "";
+    globalThis.fetch = (async (url: string) => {
+      capturedUrl = url;
+      return new Response(JSON.stringify({ success: true, data: { web: [] } }), {
+        status: 200,
+        headers: { "content-type": "application/json" },
+      });
+    }) as any;
+
+    const result = await execute({ query: "fallback test" });
+    expect(result.isError).toBe(false);
+    expect(capturedUrl).toContain("firecrawl");
+  });
+
   test("falls back from tavily to perplexity when tavily has no key", async () => {
     mockWebSearchProvider = "tavily";
     mockPerplexitySecureKey = "pplx-fallback-key";