@k2b/cloud 0.6.0-rc.0

package/src/ai/files-store.ts

@@ -0,0 +1,390 @@
+import { sql } from "bun";
+
+export { guessAiMediaType } from "./file-media-type";
+
+/** Per-file and per-conversation caps — read once per operation from settings by the caller layer if needed. */
+export const AI_FILES_MAX_FILE_BYTES_DEFAULT = 50 * 1024 * 1024;
+export const AI_FILES_MAX_CONVERSATION_BYTES_DEFAULT = 250 * 1024 * 1024;
+
+export type AiFileStat = {
+  path: string;
+  size: number;
+  mediaType: string;
+  origin: "user" | "assistant";
+  updatedAt: string;
+  version: number;
+};
+
+type FileRow = {
+  path: string;
+  size: number;
+  media_type: string;
+  origin: "user" | "assistant";
+  updated_at: Date | string;
+  version: number | string;
+};
+
+type FileContentRow = FileRow & { bytes: Uint8Array };
+export type AiFileContent = AiFileStat & { bytes: Uint8Array };
+
+const iso = (value: Date | string): string => (value instanceof Date ? value.toISOString() : new Date(value).toISOString());
+
+const toStat = (row: FileRow): AiFileStat => ({
+  path: row.path,
+  size: Number(row.size),
+  mediaType: row.media_type,
+  origin: row.origin,
+  updatedAt: iso(row.updated_at),
+  version: Number(row.version),
+});
+const toContent = (row: FileContentRow): AiFileContent => ({ ...toStat(row), bytes: row.bytes });
+
+const numberedAiFilePath = (path: string, number: number): string => {
+  if (number === 1) return path;
+  const slash = path.lastIndexOf("/");
+  const directory = path.slice(0, slash + 1);
+  const name = path.slice(slash + 1);
+  const dot = name.lastIndexOf(".");
+  return dot > 0 ? `${directory}${name.slice(0, dot)}-${number}${name.slice(dot)}` : `${directory}${name}-${number}`;
+};
+
+const createUniqueAiFile = async (input: {
+  conversationId: string;
+  path: string;
+  bytes: Uint8Array;
+  mediaType?: string;
+  origin: "user" | "assistant";
+  maxFileBytes?: number;
+  maxConversationBytes?: number;
+}): Promise<AiFileStat> => {
+  const maxFile = input.maxFileBytes ?? AI_FILES_MAX_FILE_BYTES_DEFAULT;
+  const maxConversation = input.maxConversationBytes ?? AI_FILES_MAX_CONVERSATION_BYTES_DEFAULT;
+  if (input.bytes.byteLength > maxFile) {
+    throw new Error(`File exceeds the per-file limit of ${Math.floor(maxFile / (1024 * 1024))} MB.`);
+  }
+
+  return sql.begin(async (tx) => {
+    await tx`SELECT id FROM ai.conversations WHERE id = ${input.conversationId} FOR UPDATE`;
+    const totals = await tx<{ total: number | string }[]>`
+      SELECT COALESCE(SUM(size), 0) AS total FROM ai.files WHERE conversation_id = ${input.conversationId}
+    `;
+    if (Number(totals[0]?.total ?? 0) + input.bytes.byteLength > maxConversation) {
+      throw new Error(`Conversation storage limit of ${Math.floor(maxConversation / (1024 * 1024))} MB exceeded.`);
+    }
+
+    for (let number = 1; number <= 100; number++) {
+      const path = numberedAiFilePath(input.path, number);
+      const rows = await tx<FileRow[]>`
+        INSERT INTO ai.files (conversation_id, path, bytes, media_type, size, origin, updated_at)
+        VALUES (${input.conversationId}, ${path}, ${input.bytes}, ${input.mediaType ?? "application/octet-stream"}, ${input.bytes.byteLength}, ${input.origin}, now())
+        ON CONFLICT (conversation_id, path) DO NOTHING
+        RETURNING path, size, media_type, origin, updated_at, version
+      `;
+      if (rows[0]) return toStat(rows[0]);
+    }
+    throw new Error("Could not allocate a unique file path.");
+  });
+};
+
+/** Normalize a VFS path: absolute, no `.`/`..` segments, no trailing slash. */
+export const normalizeAiFilePath = (path: string): string | null => {
+  if (!path.startsWith("/")) return null;
+  const segments: string[] = [];
+  for (const part of path.split("/")) {
+    if (part === "" || part === ".") continue;
+    if (part === "..") return null;
+    if (part.includes("\0")) return null;
+    if (/[\r\n"<>]/u.test(part)) return null;
+    segments.push(part);
+  }
+  if (segments.length === 0) return null;
+  return `/${segments.join("/")}`;
+};
+
+export const decodeAiFileContent = (content: string, encoding: "utf8" | "base64"): Uint8Array => {
+  if (encoding === "utf8") return new TextEncoder().encode(content);
+  if (content.length % 4 !== 0 || !/^[A-Za-z0-9+/]*={0,2}$/.test(content)) throw new Error("Invalid base64 file content.");
+  const bytes = Buffer.from(content, "base64");
+  if (bytes.toString("base64") !== content) throw new Error("Invalid base64 file content.");
+  return new Uint8Array(bytes);
+};
+
+/**
+ * Conversation-scoped file storage backing Assistant file tools. Every
+ * operation goes straight to Postgres — no rehydration, horizontal-safe,
+ * crash-safe. Reads support byte slices (bytea STORAGE EXTERNAL) so big
+ * files never load fully.
+ */
+export const aiFileStore = {
+  async createUserUpload(input: {
+    conversationId: string;
+    path: string;
+    bytes: Uint8Array;
+    mediaType?: string;
+    maxFileBytes?: number;
+    maxConversationBytes?: number;
+  }): Promise<AiFileStat> {
+    return createUniqueAiFile({ ...input, origin: "user" });
+  },
+
+  async createAssistantFile(input: {
+    conversationId: string;
+    path: string;
+    bytes: Uint8Array;
+    mediaType?: string;
+    maxFileBytes?: number;
+    maxConversationBytes?: number;
+  }): Promise<AiFileStat> {
+    return createUniqueAiFile({ ...input, origin: "assistant" });
+  },
+
+  async list(input: { conversationId: string; prefix?: string }): Promise<AiFileStat[]> {
+    const prefix = input.prefix ?? "/";
+    const pattern = `${prefix.endsWith("/") ? prefix : `${prefix}/`}%`;
+    const rows = await sql<FileRow[]>`
+      SELECT path, size, media_type, origin, updated_at, version
+      FROM ai.files
+      WHERE conversation_id = ${input.conversationId}
+        AND (path LIKE ${pattern} OR path = ${prefix})
+      ORDER BY updated_at DESC, path ASC
+    `;
+    return rows.map(toStat);
+  },
+
+  async stat(input: { conversationId: string; path: string }): Promise<AiFileStat | null> {
+    const rows = await sql<FileRow[]>`
+      SELECT path, size, media_type, origin, updated_at, version
+      FROM ai.files
+      WHERE conversation_id = ${input.conversationId} AND path = ${input.path}
+    `;
+    return rows[0] ? toStat(rows[0]) : null;
+  },
+
+  async read(input: { conversationId: string; path: string }): Promise<AiFileContent | null> {
+    const rows = await sql<FileContentRow[]>`
+      SELECT path, bytes, size, media_type, origin, updated_at, version
+      FROM ai.files
+      WHERE conversation_id = ${input.conversationId} AND path = ${input.path}
+    `;
+    return rows[0] ? toContent(rows[0]) : null;
+  },
+
+  async readTurnFile(input: { turnId: string; path: string }): Promise<AiFileContent | null> {
+    const rows = await sql<FileContentRow[]>`
+      SELECT path, bytes, size, media_type, origin, updated_at, version
+      FROM ai.turn_files
+      WHERE turn_id = ${input.turnId}::uuid AND path = ${input.path}
+    `;
+    return rows[0] ? toContent(rows[0]) : null;
+  },
+
+  /** Byte slice without loading the whole value (substring on EXTERNAL bytea reads only needed chunks). */
+  async readSlice(input: { conversationId: string; path: string; offset: number; length: number }): Promise<Uint8Array | null> {
+    const offset = Math.max(0, Math.floor(input.offset));
+    const length = Math.max(0, Math.floor(input.length));
+    const rows = await sql<{ chunk: Uint8Array }[]>`
+      SELECT substring(bytes FROM ${offset + 1} FOR ${length}) AS chunk
+      FROM ai.files
+      WHERE conversation_id = ${input.conversationId} AND path = ${input.path}
+    `;
+    if (!rows[0]) return null;
+    return new Uint8Array(rows[0].chunk ?? []);
+  },
+
+  async readSliceWithStat(input: { conversationId: string; path: string; offset: number; length: number }): Promise<AiFileContent | null> {
+    const offset = Math.max(0, Math.floor(input.offset));
+    const length = Math.max(0, Math.floor(input.length));
+    const rows = await sql<FileContentRow[]>`
+      SELECT path, substring(bytes FROM ${offset + 1} FOR ${length}) AS bytes, size, media_type, origin, updated_at, version
+      FROM ai.files
+      WHERE conversation_id = ${input.conversationId} AND path = ${input.path}
+    `;
+    return rows[0] ? toContent(rows[0]) : null;
+  },
+
+  async readTurnSliceWithStat(input: { turnId: string; path: string; offset: number; length: number }): Promise<AiFileContent | null> {
+    const offset = Math.max(0, Math.floor(input.offset));
+    const length = Math.max(0, Math.floor(input.length));
+    const rows = await sql<FileContentRow[]>`
+      SELECT path, substring(bytes FROM ${offset + 1} FOR ${length}) AS bytes, size, media_type, origin, updated_at, version
+      FROM ai.turn_files
+      WHERE turn_id = ${input.turnId}::uuid AND path = ${input.path}
+    `;
+    return rows[0] ? toContent(rows[0]) : null;
+  },
+
+  async readAll(input: { conversationId: string; path: string }): Promise<Uint8Array | null> {
+    const rows = await sql<{ bytes: Uint8Array }[]>`
+      SELECT bytes FROM ai.files
+      WHERE conversation_id = ${input.conversationId} AND path = ${input.path}
+    `;
+    if (!rows[0]) return null;
+    return new Uint8Array(rows[0].bytes ?? []);
+  },
+
+  /**
+   * Upsert one file. Enforces the per-file and per-conversation caps —
+   * here in the store so no command or tool can bypass them.
+   */
+  async write(input: {
+    conversationId: string;
+    path: string;
+    bytes: Uint8Array;
+    mediaType?: string;
+    origin?: "user" | "assistant";
+    allowUserOverwrite?: boolean;
+    maxFileBytes?: number;
+    maxConversationBytes?: number;
+  }): Promise<void> {
+    const maxFile = input.maxFileBytes ?? AI_FILES_MAX_FILE_BYTES_DEFAULT;
+    const maxConversation = input.maxConversationBytes ?? AI_FILES_MAX_CONVERSATION_BYTES_DEFAULT;
+    if (input.bytes.byteLength > maxFile) {
+      throw new Error(`File exceeds the per-file limit of ${Math.floor(maxFile / (1024 * 1024))} MB.`);
+    }
+
+    await sql.begin(async (tx) => {
+      await tx`SELECT id FROM ai.conversations WHERE id = ${input.conversationId} FOR UPDATE`;
+      const totals = await tx<{ total: number | string }[]>`
+        SELECT COALESCE(SUM(size), 0) AS total
+        FROM ai.files
+        WHERE conversation_id = ${input.conversationId} AND path <> ${input.path}
+      `;
+      const otherBytes = Number(totals[0]?.total ?? 0);
+      if (otherBytes + input.bytes.byteLength > maxConversation) {
+        throw new Error(`Conversation storage limit of ${Math.floor(maxConversation / (1024 * 1024))} MB exceeded.`);
+      }
+      if (input.origin === "user") {
+        if (input.allowUserOverwrite) {
+          const written = await tx<{ id: string }[]>`
+            UPDATE ai.files
+            SET bytes = ${input.bytes}, media_type = ${input.mediaType ?? "application/octet-stream"}, size = ${input.bytes.byteLength}, updated_at = now(), version = version + 1
+            WHERE conversation_id = ${input.conversationId} AND path = ${input.path} AND origin = 'user'
+            RETURNING id
+          `;
+          if (!written[0]) throw new Error(`User-uploaded file does not exist: ${input.path}.`);
+        } else {
+          await tx`
+            INSERT INTO ai.files (conversation_id, path, bytes, media_type, size, origin, updated_at)
+            VALUES (${input.conversationId}, ${input.path}, ${input.bytes}, ${input.mediaType ?? "application/octet-stream"}, ${input.bytes.byteLength}, 'user', now())
+          `;
+        }
+      } else {
+        const written = await tx<{ id: string }[]>`
+          INSERT INTO ai.files (conversation_id, path, bytes, media_type, size, origin, updated_at)
+          VALUES (${input.conversationId}, ${input.path}, ${input.bytes}, ${input.mediaType ?? "application/octet-stream"}, ${input.bytes.byteLength}, 'assistant', now())
+          ON CONFLICT (conversation_id, path) DO UPDATE SET
+            bytes = EXCLUDED.bytes,
+            media_type = EXCLUDED.media_type,
+            size = EXCLUDED.size,
+            updated_at = now(),
+            version = ai.files.version + 1
+          WHERE ai.files.origin = 'assistant'
+          RETURNING id
+        `;
+        if (!written[0]) throw new Error(`Cannot overwrite user-uploaded file ${input.path}.`);
+      }
+    });
+  },
+
+  async append(input: {
+    conversationId: string;
+    path: string;
+    bytes: Uint8Array;
+    mediaType?: string;
+    maxFileBytes?: number;
+    maxConversationBytes?: number;
+  }): Promise<void> {
+    const maxFile = input.maxFileBytes ?? AI_FILES_MAX_FILE_BYTES_DEFAULT;
+    const maxConversation = input.maxConversationBytes ?? AI_FILES_MAX_CONVERSATION_BYTES_DEFAULT;
+    await sql.begin(async (tx) => {
+      await tx`SELECT id FROM ai.conversations WHERE id = ${input.conversationId} FOR UPDATE`;
+      const current = await tx<{ size: number }[]>`
+        SELECT size FROM ai.files
+        WHERE conversation_id = ${input.conversationId} AND path = ${input.path}
+      `;
+      const nextSize = Number(current[0]?.size ?? 0) + input.bytes.byteLength;
+      if (nextSize > maxFile) {
+        throw new Error(`File exceeds the per-file limit of ${Math.floor(maxFile / (1024 * 1024))} MB.`);
+      }
+      const totals = await tx<{ total: number | string }[]>`
+        SELECT COALESCE(SUM(size), 0) AS total FROM ai.files WHERE conversation_id = ${input.conversationId}
+      `;
+      if (Number(totals[0]?.total ?? 0) + input.bytes.byteLength > maxConversation) {
+        throw new Error(`Conversation storage limit of ${Math.floor(maxConversation / (1024 * 1024))} MB exceeded.`);
+      }
+      const appended = await tx<{ id: string }[]>`
+        INSERT INTO ai.files (conversation_id, path, bytes, media_type, size, origin, updated_at)
+        VALUES (${input.conversationId}, ${input.path}, ${input.bytes}, ${input.mediaType ?? "application/octet-stream"}, ${input.bytes.byteLength}, 'assistant', now())
+        ON CONFLICT (conversation_id, path) DO UPDATE SET
+          bytes = ai.files.bytes || EXCLUDED.bytes,
+          size = ai.files.size + EXCLUDED.size,
+          updated_at = now(),
+          version = ai.files.version + 1
+        WHERE ai.files.origin = 'assistant'
+        RETURNING id
+      `;
+      if (!appended[0]) throw new Error(`Cannot append to user-uploaded file ${input.path}.`);
+    });
+  },
+
+  async remove(input: { conversationId: string; path: string; recursive?: boolean }): Promise<number> {
+    return sql.begin(async (tx) => {
+      await tx`SELECT id FROM ai.conversations WHERE id = ${input.conversationId} FOR UPDATE`;
+      if (input.recursive) {
+        const pattern = `${input.path.endsWith("/") ? input.path : `${input.path}/`}%`;
+        const rows = await tx<{ id: string }[]>`
+          DELETE FROM ai.files
+          WHERE conversation_id = ${input.conversationId} AND (path = ${input.path} OR path LIKE ${pattern})
+          RETURNING id
+        `;
+        return rows.length;
+      }
+      const rows = await tx<{ id: string }[]>`
+        DELETE FROM ai.files
+        WHERE conversation_id = ${input.conversationId} AND path = ${input.path}
+        RETURNING id
+      `;
+      return rows.length;
+    });
+  },
+
+  async rename(input: { conversationId: string; from: string; to: string }): Promise<"renamed" | "not_found" | "conflict"> {
+    return sql.begin(async (tx) => {
+      await tx`SELECT id FROM ai.conversations WHERE id = ${input.conversationId} FOR UPDATE`;
+      const source = await tx<{ id: string }[]>`
+        SELECT id FROM ai.files WHERE conversation_id = ${input.conversationId} AND path = ${input.from}
+      `;
+      if (!source[0]) return "not_found" as const;
+      const target = await tx<{ id: string }[]>`
+        SELECT id FROM ai.files WHERE conversation_id = ${input.conversationId} AND path = ${input.to}
+      `;
+      if (target[0]) return "conflict" as const;
+      await tx`UPDATE ai.files SET path = ${input.to}, updated_at = now(), version = version + 1 WHERE id = ${source[0].id}::uuid`;
+      return "renamed" as const;
+    });
+  },
+
+  /** Copy every file into another conversation (fork). */
+  async copyToConversation(input: { sourceConversationId: string; targetConversationId: string }): Promise<number> {
+    const rows = await sql<{ id: string }[]>`
+      INSERT INTO ai.files (conversation_id, path, bytes, media_type, size, origin)
+      SELECT ${input.targetConversationId}, path, bytes, media_type, size, origin
+      FROM ai.files
+      WHERE conversation_id = ${input.sourceConversationId}
+      ON CONFLICT (conversation_id, path) DO NOTHING
+      RETURNING id
+    `;
+    return rows.length;
+  },
+
+  async totalBytes(conversationId: string): Promise<number> {
+    const rows = await sql<{ total: number | string }[]>`
+      SELECT COALESCE(SUM(size), 0) AS total FROM ai.files WHERE conversation_id = ${conversationId}
+    `;
+    return Number(rows[0]?.total ?? 0);
+  },
+};
+
+/** Authorized services may expose this read after resolving the conversation owner. */
+export const listAiConversationFiles = (conversationId: string, prefix?: string): Promise<AiFileStat[]> =>
+  aiFileStore.list({ conversationId, prefix });

package/src/ai/firecrawl-tools.ts

@@ -0,0 +1,305 @@
+import { isIP } from "node:net";
+import { truncateMiddle } from "@k2b/nessi";
+import { z } from "zod";
+import { coreSettings } from "../services";
+import { defineAiTool } from "./tools";
+
+export const AI_FIRECRAWL_API_KEY_SETTING_KEY = "ai.firecrawl_api_key";
+
+const FIRECRAWL_BASE_URL = "https://api.firecrawl.dev";
+const FIRECRAWL_SEARCH_LIMIT = 5;
+const FIRECRAWL_TIMEOUT_MS = 30_000;
+const FIRECRAWL_CACHE_MAX_AGE_MS = 172_800_000;
+const WEB_EXTRACT_MAX_CONTENT_CHARS = 1_000_000;
+const WEB_SEARCH_HISTORY_SNIPPET_CHARS = 120;
+const WEB_EXTRACT_HISTORY_CONTENT_CHARS = 4_000;
+
+type FirecrawlFetch = typeof fetch;
+
+type FirecrawlToolConfig = {
+  apiKey?: string | null;
+  fetch?: FirecrawlFetch;
+};
+
+export const CloudAiWebSearchInputSchema = z
+  .object({
+    query: z.string().trim().min(1).max(500),
+  })
+  .strict();
+
+/** Flat result list — no billing metadata, the model only needs the sources. */
+export const CloudAiWebSearchOutputSchema = z.array(
+  z.object({
+    title: z.string(),
+    url: z.string(),
+    snippet: z.string(),
+    position: z.number().int(),
+  }),
+);
+
+export const CloudAiWebExtractInputSchema = z
+  .object({
+    url: z.string().trim().url().max(2_000),
+  })
+  .strict();
+
+export const CloudAiWebExtractOutputSchema = z.object({
+  url: z.string(),
+  title: z.string().optional(),
+  description: z.string().optional(),
+  content: z.string(),
+  truncated: z.boolean(),
+});
+
+const readFirecrawlApiKey = async (): Promise<string> =>
+  String((await coreSettings.get<string>(AI_FIRECRAWL_API_KEY_SETTING_KEY)) ?? "").trim();
+
+export const isCloudAiFirecrawlConfigured = async () => (await readFirecrawlApiKey()).length > 0;
+
+const resolveApiKey = async (apiKey: string | null | undefined) => {
+  const resolved = (apiKey ?? (await readFirecrawlApiKey())).trim();
+  if (!resolved) throw new Error("Firecrawl API key is not configured.");
+  return resolved;
+};
+
+const fetchWithTimeout = async (fetcher: FirecrawlFetch, url: string, init: RequestInit, signal?: AbortSignal): Promise<Response> => {
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), FIRECRAWL_TIMEOUT_MS);
+  const abort = () => controller.abort();
+  if (signal?.aborted) controller.abort();
+  else signal?.addEventListener("abort", abort, { once: true });
+
+  try {
+    return await fetcher(url, { ...init, signal: controller.signal });
+  } finally {
+    clearTimeout(timeout);
+    signal?.removeEventListener("abort", abort);
+  }
+};
+
+const readJson = async (response: Response): Promise<unknown> => {
+  try {
+    return await response.json();
+  } catch {
+    return null;
+  }
+};
+
+const extractErrorMessage = (body: unknown): string | null => {
+  if (!body || typeof body !== "object") return null;
+  const record = body as Record<string, unknown>;
+  if (typeof record.error === "string") return record.error;
+  if (typeof record.message === "string") return record.message;
+  return null;
+};
+
+const firecrawlPost = async (input: {
+  apiKey?: string | null;
+  fetch?: FirecrawlFetch;
+  path: "/v2/search" | "/v2/scrape";
+  body: Record<string, unknown>;
+  signal?: AbortSignal;
+}): Promise<unknown> => {
+  const apiKey = await resolveApiKey(input.apiKey);
+  const fetcher = input.fetch ?? fetch;
+  const response = await fetchWithTimeout(
+    fetcher,
+    `${FIRECRAWL_BASE_URL}${input.path}`,
+    {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${apiKey}`,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify(input.body),
+    },
+    input.signal,
+  );
+  const body = await readJson(response);
+  if (!response.ok) {
+    const message = extractErrorMessage(body) ?? response.statusText;
+    throw new Error(`Firecrawl request failed (HTTP ${response.status}): ${message}`);
+  }
+  return body;
+};
+
+const isRecord = (value: unknown): value is Record<string, unknown> => Boolean(value) && typeof value === "object" && !Array.isArray(value);
+const asString = (value: unknown): string => (typeof value === "string" ? value.trim() : "");
+
+const truncateSnippet = (text: string): string => {
+  if (text.length <= WEB_SEARCH_HISTORY_SNIPPET_CHARS) return text;
+  return `${text.slice(0, WEB_SEARCH_HISTORY_SNIPPET_CHARS - 3).trimEnd()}...`;
+};
+
+const truncate = (text: string, maxChars: number): { text: string; truncated: boolean } => {
+  if (text.length <= maxChars) return { text, truncated: false };
+  return { text: text.slice(0, Math.max(0, maxChars)).trimEnd(), truncated: true };
+};
+
+const isPrivateIpv4 = (hostname: string): boolean => {
+  const parts = hostname.split(".").map((part) => Number(part));
+  if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255)) return true;
+  const [a, b] = parts as [number, number, number, number];
+  if (a === 0 || a === 10 || a === 127) return true;
+  if (a === 100 && b >= 64 && b <= 127) return true;
+  if (a === 169 && b === 254) return true;
+  if (a === 172 && b >= 16 && b <= 31) return true;
+  if (a === 192 && (b === 0 || b === 168)) return true;
+  if (a === 198 && (b === 18 || b === 19)) return true;
+  return a >= 224;
+};
+
+const isPrivateIpv6 = (hostname: string): boolean => {
+  const normalized = hostname.replace(/^\[|\]$/g, "").toLowerCase();
+  return (
+    normalized === "::" ||
+    normalized === "::1" ||
+    normalized.startsWith("fc") ||
+    normalized.startsWith("fd") ||
+    normalized.startsWith("fe80:") ||
+    normalized.startsWith("2001:db8:")
+  );
+};
+
+export const assertPublicHttpUrl = (rawUrl: string): string => {
+  let parsed: URL;
+  try {
+    parsed = new URL(rawUrl);
+  } catch {
+    throw new Error("URL must be valid.");
+  }
+  if (parsed.protocol !== "https:" && parsed.protocol !== "http:") throw new Error("URL must use http or https.");
+
+  const hostname = parsed.hostname.toLowerCase();
+  if (!hostname || hostname === "localhost" || hostname.endsWith(".localhost")) {
+    throw new Error("URL host is not allowed.");
+  }
+
+  const ipKind = isIP(hostname.replace(/^\[|\]$/g, ""));
+  if (ipKind === 4 && isPrivateIpv4(hostname)) throw new Error("URL must not point to a private IP address.");
+  if (ipKind === 6 && isPrivateIpv6(hostname)) throw new Error("URL must not point to a private IP address.");
+  return parsed.toString();
+};
+
+export const runCloudAiWebSearch = async (
+  input: z.infer<typeof CloudAiWebSearchInputSchema>,
+  config: FirecrawlToolConfig & { signal?: AbortSignal } = {},
+): Promise<z.infer<typeof CloudAiWebSearchOutputSchema>> => {
+  const body = await firecrawlPost({
+    apiKey: config.apiKey,
+    fetch: config.fetch,
+    path: "/v2/search",
+    body: {
+      query: input.query,
+      limit: FIRECRAWL_SEARCH_LIMIT,
+      sources: ["web"],
+      timeout: FIRECRAWL_TIMEOUT_MS,
+      ignoreInvalidURLs: true,
+    },
+    signal: config.signal,
+  });
+  if (!isRecord(body) || body.success !== true) {
+    throw new Error(extractErrorMessage(body) ?? "Firecrawl search returned an invalid response.");
+  }
+
+  const data = isRecord(body.data) ? body.data : {};
+  const web = Array.isArray(data.web) ? data.web : [];
+  const results = web
+    .filter(isRecord)
+    .slice(0, FIRECRAWL_SEARCH_LIMIT)
+    .map((item, index) => ({
+      title: asString(item.title) || asString(isRecord(item.metadata) ? item.metadata.title : undefined) || "Untitled",
+      url: asString(item.url) || asString(isRecord(item.metadata) ? (item.metadata.url ?? item.metadata.sourceURL) : undefined),
+      snippet: asString(item.description) || asString(item.snippet),
+      position: index + 1,
+    }))
+    .filter((item) => item.url.length > 0);
+
+  return CloudAiWebSearchOutputSchema.parse(results);
+};
+
+export const runCloudAiWebExtract = async (
+  input: z.infer<typeof CloudAiWebExtractInputSchema>,
+  config: FirecrawlToolConfig & { signal?: AbortSignal; maxChars?: number } = {},
+): Promise<z.infer<typeof CloudAiWebExtractOutputSchema>> => {
+  const url = assertPublicHttpUrl(input.url);
+  const body = await firecrawlPost({
+    apiKey: config.apiKey,
+    fetch: config.fetch,
+    path: "/v2/scrape",
+    body: {
+      url,
+      formats: ["markdown"],
+      onlyMainContent: true,
+      maxAge: FIRECRAWL_CACHE_MAX_AGE_MS,
+      timeout: FIRECRAWL_TIMEOUT_MS,
+      removeBase64Images: true,
+      blockAds: true,
+    },
+    signal: config.signal,
+  });
+  if (!isRecord(body) || body.success !== true || !isRecord(body.data)) {
+    throw new Error(extractErrorMessage(body) ?? "Firecrawl extract returned an invalid response.");
+  }
+
+  const maxChars = config.maxChars ?? WEB_EXTRACT_MAX_CONTENT_CHARS;
+  const data = body.data;
+  const metadata = isRecord(data.metadata) ? data.metadata : {};
+  const normalizedUrl = asString(data.url) || asString(metadata.url) || asString(metadata.sourceURL) || url;
+  const content = asString(data.markdown);
+  const truncated = truncate(content, maxChars);
+
+  return CloudAiWebExtractOutputSchema.parse({
+    url: normalizedUrl,
+    title: asString(metadata.title) || asString(data.title) || undefined,
+    description: asString(metadata.description) || asString(data.description) || undefined,
+    content: truncated.text,
+    truncated: truncated.truncated,
+  });
+};
+
+export const createCloudAiWebSearchTool = (config: FirecrawlToolConfig = {}) =>
+  defineAiTool({
+    name: "web_search",
+    description:
+      "Search the web for current sources. Input only a plain natural-language query. Search again with a focused query when the first results are incomplete, outdated, or conflicting.",
+    inputSchema: CloudAiWebSearchInputSchema,
+    outputSchema: CloudAiWebSearchOutputSchema,
+    approval: "never",
+    timeoutMs: 90_000,
+    promptHint:
+      "search the web for current facts and sources; use additional focused searches when the evidence is incomplete or conflicting.",
+    toHistoricalResult: ({ output }) =>
+      output.map((result) => ({
+        ...result,
+        snippet: truncateSnippet(result.snippet),
+      })),
+  }).server(async (input, ctx) => runCloudAiWebSearch(input, { ...config, signal: ctx.signal }));
+
+export const createCloudAiWebExtractTool = (config: FirecrawlToolConfig = {}) =>
+  defineAiTool({
+    name: "web_extract",
+    description:
+      "Read one web page by URL and return clean Markdown. Inspect the relevant pages needed to support the answer; for research or comparison, prefer primary sources and read more than one useful source when warranted.",
+    inputSchema: CloudAiWebExtractInputSchema,
+    outputSchema: CloudAiWebExtractOutputSchema,
+    approval: "never",
+    timeoutMs: 90_000,
+    promptHint:
+      "read relevant pages found by web_search; for research or comparison, inspect multiple useful sources and prefer primary sources.",
+    toHistoricalResult: ({ output }) => {
+      const content = truncateMiddle(output.content, WEB_EXTRACT_HISTORY_CONTENT_CHARS);
+      return {
+        url: output.url,
+        title: output.title,
+        description: output.description,
+        content,
+        truncated: output.truncated || content !== output.content,
+      };
+    },
+  }).server(async (input, ctx) => runCloudAiWebExtract(input, { ...config, signal: ctx.signal }));
+
+export type CloudAiWebSearchInput = z.infer<typeof CloudAiWebSearchInputSchema>;
+export type CloudAiWebSearchOutput = z.infer<typeof CloudAiWebSearchOutputSchema>;
+export type CloudAiWebExtractInput = z.infer<typeof CloudAiWebExtractInputSchema>;
+export type CloudAiWebExtractOutput = z.infer<typeof CloudAiWebExtractOutputSchema>;