@sanity/ailf 0.1.0

package/dist/agent-observer/agentic-provider.js

@@ -0,0 +1,983 @@
+/**
+ * agentic-provider.ts
+ *
+ * An agentic Promptfoo provider that gives the model web_search and
+ * fetch_page tools, simulating how real AI agents behave when a user
+ * asks a development question.
+ *
+ * Supports two agent modes via the `agentMode` config:
+ *
+ *   - "naive"     — Simulates current agents (Claude Code, ChatGPT, Cursor):
+ *                    uses Jina Reader for search + page fetching because real
+ *                    agents can't render JavaScript-heavy SPAs server-side.
+ *
+ *   - "optimized" — Simulates an ideal agent that knows about Sanity's
+ *                    agent-friendly endpoints: fetches .md versions of doc
+ *                    pages directly, uses llms.txt for doc discovery, and
+ *                    falls back to Jina only for non-Sanity pages.
+ *
+ * All HTTP requests go through the RequestRecorder, so they're automatically
+ * classified as docPageVisits, searchQueries, etc.
+ *
+ * Promptfoo config usage:
+ *
+ *   providers:
+ *     - id: file://dist/agent-observer/agentic-provider.js
+ *       label: "GPT-4o (Naive Agent)"
+ *       config:
+ *         model: gpt-4o
+ *         agentMode: naive       # or "optimized"
+ *         maxToolRounds: 5
+ */
+import { config as loadDotenv } from "dotenv";
+import { randomUUID } from "crypto";
+import { RequestRecorder } from "./proxy.js";
+import { calculateCost } from "./pricing.js";
+import { isAllowedOrigin } from "../sources.js";
+loadDotenv({
+    override: true,
+    path: new URL("../../.env", import.meta.url).pathname,
+});
+// ---------------------------------------------------------------------------
+// Tool definitions — mirror what real agents provide
+// ---------------------------------------------------------------------------
+const AGENT_TOOLS = [
+    {
+        function: {
+            description: "Search the web for information. Use this to find documentation, " +
+                "tutorials, API references, and examples. Returns a list of search " +
+                "results with titles, URLs, and snippets.",
+            name: "web_search",
+            parameters: {
+                properties: {
+                    query: {
+                        description: "The search query to execute",
+                        type: "string",
+                    },
+                },
+                required: ["query"],
+                type: "object",
+            },
+        },
+        type: "function",
+    },
+    {
+        function: {
+            description: "Fetch the content of a web page. Use this to read documentation, " +
+                "code examples, API references, or any other web content. Returns " +
+                "the text content of the page.",
+            name: "fetch_page",
+            parameters: {
+                properties: {
+                    url: {
+                        description: "The URL to fetch",
+                        type: "string",
+                    },
+                },
+                required: ["url"],
+                type: "object",
+            },
+        },
+        type: "function",
+    },
+];
+// Optimized mode gets an extra tool for discovering Sanity's doc structure
+const OPTIMIZED_EXTRA_TOOLS = [
+    {
+        function: {
+            description: "List all available documentation pages for a site. Currently supports " +
+                "the documentation site (via llms.txt). Returns a structured list of doc page titles " +
+                "and URLs. Use this FIRST to discover what documentation is available " +
+                "before fetching specific pages.",
+            name: "list_docs",
+            parameters: {
+                properties: {
+                    site: {
+                        description: 'The documentation site domain, e.g. "sanity.io"',
+                        type: "string",
+                    },
+                },
+                required: ["site"],
+                type: "object",
+            },
+        },
+        type: "function",
+    },
+];
+// ---------------------------------------------------------------------------
+// Sanity docs helpers
+// ---------------------------------------------------------------------------
+/** Default doc base URL — used when no source config is provided */
+const DEFAULT_DOC_BASE_URL = "https://www.sanity.io/docs";
+/** Default llms.txt URL */
+const DEFAULT_LLMS_TXT_URL = "https://www.sanity.io/docs/llms.txt";
+export default class AgenticProvider {
+    config;
+    providerId;
+    agentMode;
+    allowedOrigins;
+    customHeaders;
+    docBaseUrl;
+    docsUrlPattern;
+    llmsTxtUrl;
+    priorityDomain;
+    recorder;
+    searchMode;
+    constructor(options) {
+        this.providerId = options.id ?? "agentic-observer";
+        this.config = options.config ?? {};
+        this.agentMode = this.config.agentMode || "naive";
+        // Documentation source configuration — defaults to Sanity production
+        this.docBaseUrl = this.config.docBaseUrl || DEFAULT_DOC_BASE_URL;
+        this.llmsTxtUrl = this.config.llmsTxtUrl || DEFAULT_LLMS_TXT_URL;
+        this.docsUrlPattern = buildDocsUrlPattern(this.docBaseUrl);
+        // Custom HTTP headers (e.g., Vercel bypass protection token)
+        this.customHeaders =
+            this.config.customHeaders || {};
+        // Extract priority domain from docBaseUrl for search result ranking
+        const baseUrlObj = new URL(this.docBaseUrl);
+        this.priorityDomain =
+            this.config.priorityDomain ||
+                baseUrlObj.hostname.replace(/^www\./, "");
+        // Optional origin sandboxing — restrict which URLs the agent can access
+        this.allowedOrigins = Array.isArray(this.config.allowedOrigins)
+            ? this.config.allowedOrigins.filter(Boolean)
+            : [];
+        // Search mode: controls web_search tool availability and filtering
+        this.searchMode = this.config.searchMode || "open";
+        this.recorder = new RequestRecorder(this.config.observerOptions || {});
+    }
+    /**
+     * Main Promptfoo provider entry point. Runs the full agentic loop.
+     */
+    async callApi(prompt, context) {
+        const sessionId = randomUUID();
+        const taskDescription = context?.vars?.task ||
+            context?.prompt?.label ||
+            "unknown-task";
+        const observe = this.config.observe !== false;
+        if (observe) {
+            this.recorder.start(sessionId, this.id(), taskDescription);
+        }
+        let result;
+        try {
+            result = await this.runAgenticLoop(prompt);
+        }
+        catch (err) {
+            const error = err;
+            result = {
+                error: error.message,
+                output: undefined,
+            };
+        }
+        if (observe) {
+            const behaviorLog = this.recorder.stop();
+            result.metadata = {
+                ...(result.metadata ?? {}),
+                agentBehavior: behaviorLog,
+                agentBehaviorSummary: behaviorLog.summary,
+                agentMode: this.agentMode,
+            };
+        }
+        return result;
+    }
+    /**
+     * Exposes the recorder for external integrations.
+     */
+    getRecorder() {
+        return this.recorder;
+    }
+    id() {
+        return `agentic:${this.agentMode}:${this.providerId}`;
+    }
+    // -------------------------------------------------------------------------
+    // Tool execution
+    // -------------------------------------------------------------------------
+    /**
+     * Build the system prompt based on agent mode and configured doc URLs.
+     */
+    buildSystemPrompt() {
+        const docDomain = this.priorityDomain;
+        const docUrl = this.docBaseUrl;
+        const exampleSlug = "schema-types";
+        // Search guidance varies based on search mode
+        const searchGuidance = this.searchMode === "off"
+            ? "You do NOT have web search. Fetch documentation pages directly " +
+                "using the URLs you know or discover via list_docs."
+            : this.searchMode === "origin-only"
+                ? `You have web search restricted to ${docDomain}. ` +
+                    "Search results will only include pages from this domain."
+                : "You have access to web search.";
+        return this.agentMode === "optimized"
+            ? "You are an expert developer helping with implementation tasks. " +
+                searchGuidance +
+                " You also have page fetching and a doc listing tool. " +
+                "IMPORTANT: The documentation supports agent-friendly endpoints. " +
+                `Start by calling list_docs for "${docDomain}" to discover available doc pages. ` +
+                'When fetching doc pages, append ".md" to the URL to get clean markdown ' +
+                `(e.g., ${docUrl}/${exampleSlug}.md instead of ` +
+                `${docUrl}/${exampleSlug}). ` +
+                "Always verify against the current docs before answering."
+            : "You are an expert developer helping with implementation tasks. " +
+                searchGuidance +
+                " You also have page fetching tools. " +
+                "When asked about APIs, configuration, schemas, GROQ queries, " +
+                "or any topic related to the documentation, " +
+                `USE the tools to look up the latest official documentation at ${docDomain} ` +
+                "before answering. Do not rely solely on your training data — always verify " +
+                "against the current docs. Prefer official documentation over third-party sources.";
+    }
+    /**
+     * Cleans Jina Reader markdown output by removing navigation boilerplate,
+     * cookie banners, and footer content.
+     */
+    cleanJinaContent(markdown) {
+        const lines = markdown.split("\n");
+        const cleanLines = [];
+        let inMainContent = false;
+        let skipCount = 0;
+        for (const line of lines) {
+            if (line.includes("cookie") && line.includes("Privacy Policy"))
+                continue;
+            if (line.includes("Accept Deny Non-Essential"))
+                continue;
+            if (line.trim() ===
+                "Opens in a new window Opens an external website Opens an external website in a new window")
+                continue;
+            if (line.trim().startsWith("* ") &&
+                line.includes(`](${this.docBaseUrl}/`)) {
+                skipCount++;
+                if (skipCount > 3 && !inMainContent)
+                    continue;
+            }
+            else {
+                skipCount = 0;
+            }
+            if (line.startsWith("# ") ||
+                line.startsWith("## ") ||
+                line.startsWith("### ")) {
+                inMainContent = true;
+            }
+            if (inMainContent) {
+                cleanLines.push(line);
+            }
+        }
+        if (cleanLines.length < 10)
+            return markdown;
+        return cleanLines.join("\n").trim();
+    }
+    // -------------------------------------------------------------------------
+    // web_search — different strategies per mode
+    // -------------------------------------------------------------------------
+    /**
+     * Detect which LLM provider to use based on config and model name.
+     * Reads the `provider` config field set by generate-configs, with
+     * fallback heuristics for backward compatibility.
+     */
+    detectProvider() {
+        const explicit = this.config.provider;
+        if (explicit === "anthropic")
+            return "anthropic";
+        if (explicit === "openai")
+            return "openai";
+        // Heuristic fallback: detect from model name
+        const model = this.config.model || "";
+        if (model.startsWith("claude"))
+            return "anthropic";
+        return "openai";
+    }
+    // -------------------------------------------------------------------------
+    // fetch_page — key difference between naive and optimized modes
+    // -------------------------------------------------------------------------
+    async executeFetchPage(url, fetchFn) {
+        const maxContentLength = 12000;
+        // -----------------------------------------------------------------------
+        // ORIGIN SANDBOXING: reject URLs outside allowed origins (supports globs)
+        // -----------------------------------------------------------------------
+        if (this.allowedOrigins.length > 0 &&
+            !isAllowedOrigin(url, this.allowedOrigins)) {
+            return `[Blocked] URL ${url} is outside the allowed origins: ${this.allowedOrigins.join(", ")}`;
+        }
+        // -----------------------------------------------------------------------
+        // OPTIMIZED MODE: Use .md endpoints for docs pages
+        // -----------------------------------------------------------------------
+        if (this.agentMode === "optimized" && this.docsUrlPattern.test(url)) {
+            const mdUrl = toMarkdownUrl(url);
+            const response = await fetchFn(mdUrl, {
+                headers: this.mergeDocHeaders({
+                    Accept: "text/markdown, text/plain",
+                    "User-Agent": "Mozilla/5.0 (compatible; SanityEvalBot/1.0)",
+                }, mdUrl),
+                method: "GET",
+            });
+            if (response.ok) {
+                const contentType = response.headers.get("content-type") ?? "";
+                const text = await response.text();
+                // Verify we got markdown, not HTML (the .md endpoint returns
+                // Content-Type: text/markdown;charset=UTF-8)
+                if (contentType.includes("markdown") || !text.startsWith("<!DOCTYPE")) {
+                    return text.slice(0, maxContentLength);
+                }
+            }
+            // If .md failed, fall through to naive strategy
+        }
+        // -----------------------------------------------------------------------
+        // NAIVE MODE (and fallback): Use Jina Reader for JS-rendered pages
+        // -----------------------------------------------------------------------
+        try {
+            const jinaUrl = `https://r.jina.ai/${url}`;
+            const jinaResponse = await fetchFn(jinaUrl, {
+                headers: {
+                    Accept: "text/plain",
+                    "User-Agent": "Mozilla/5.0 (compatible; SanityEvalBot/1.0)",
+                },
+                method: "GET",
+            });
+            if (jinaResponse.ok) {
+                const markdown = await jinaResponse.text();
+                if (markdown.length > 100) {
+                    return this.cleanJinaContent(markdown).slice(0, maxContentLength);
+                }
+            }
+        }
+        catch {
+            // Jina unavailable — fall through to direct fetch
+        }
+        // -----------------------------------------------------------------------
+        // LAST RESORT: Direct fetch with HTML stripping
+        // -----------------------------------------------------------------------
+        const response = await fetchFn(url, {
+            headers: this.mergeDocHeaders({
+                Accept: "text/html,application/xhtml+xml,text/plain",
+                "User-Agent": "Mozilla/5.0 (compatible; SanityEvalBot/1.0)",
+            }, url),
+            method: "GET",
+        });
+        if (!response.ok) {
+            return JSON.stringify({
+                error: `HTTP ${response.status}: ${response.statusText}`,
+                url,
+            });
+        }
+        const contentType = response.headers.get("content-type") ?? "";
+        const text = await response.text();
+        if (contentType.includes("markdown")) {
+            return text.slice(0, maxContentLength);
+        }
+        if (contentType.includes("html")) {
+            return this.stripHtml(text).slice(0, maxContentLength);
+        }
+        return text.slice(0, maxContentLength);
+    }
+    // -------------------------------------------------------------------------
+    // list_docs — fetches llms.txt (optimized mode only)
+    // -------------------------------------------------------------------------
+    async executeListDocs(site, fetchFn) {
+        // Origin sandboxing for list_docs — block requests to off-origin sites
+        if (this.allowedOrigins.length > 0) {
+            const siteHost = site
+                .replace(/^https?:\/\//, "")
+                .replace(/\/.*$/, "")
+                .replace(/^www\./, "");
+            if (!isAllowedOrigin(`https://${siteHost}`, this.allowedOrigins)) {
+                return JSON.stringify({
+                    error: `list_docs restricted to allowed origins: ${this.allowedOrigins.join(", ")}`,
+                    suggestion: `Try list_docs("${this.priorityDomain}") instead.`,
+                });
+            }
+        }
+        // Use configured llms.txt URL, or construct from the provided site
+        const llmsTxtUrl = site.includes("llms.txt")
+            ? site
+            : site === this.priorityDomain || site.includes(this.priorityDomain)
+                ? this.llmsTxtUrl
+                : `https://${site.replace(/^https?:\/\//, "").replace(/\/.*$/, "")}/llms.txt`;
+        const response = await fetchFn(llmsTxtUrl, {
+            headers: this.mergeDocHeaders({
+                Accept: "text/plain, text/markdown",
+                "User-Agent": "Mozilla/5.0 (compatible; SanityEvalBot/1.0)",
+            }, llmsTxtUrl),
+            method: "GET",
+        });
+        if (!response.ok) {
+            return JSON.stringify({
+                error: `No llms.txt found at ${llmsTxtUrl} (HTTP ${response.status})`,
+                suggestion: "Try using web_search instead to find documentation.",
+            });
+        }
+        const contentType = response.headers.get("content-type") ?? "";
+        const text = await response.text();
+        // Verify it's actually an llms.txt file (markdown with links)
+        if (contentType.includes("html") && text.includes("<!DOCTYPE")) {
+            return JSON.stringify({
+                error: `${llmsTxtUrl} returned HTML, not a docs listing.`,
+                suggestion: "Try using web_search instead to find documentation.",
+            });
+        }
+        // Return the full llms.txt — it's already a clean markdown listing
+        // Trim to reasonable size (llms.txt can be long)
+        return text.slice(0, 15000);
+    }
+    async executeTool(name, argsJson, fetchFn) {
+        try {
+            const args = JSON.parse(argsJson);
+            switch (name) {
+                case "fetch_page":
+                    return await this.executeFetchPage(args.url, fetchFn);
+                case "list_docs":
+                    return await this.executeListDocs(args.site, fetchFn);
+                case "web_search":
+                    return await this.executeWebSearch(args.query, fetchFn);
+                default:
+                    return JSON.stringify({ error: `Unknown tool: ${name}` });
+            }
+        }
+        catch (err) {
+            const error = err;
+            return JSON.stringify({ error: error.message });
+        }
+    }
+    // -------------------------------------------------------------------------
+    // Content cleaning helpers
+    // -------------------------------------------------------------------------
+    async executeWebSearch(query, fetchFn) {
+        let results = [];
+        // Try Google Custom Search API if configured (both modes)
+        const googleApiKey = process.env.GOOGLE_SEARCH_API_KEY;
+        const googleCseId = process.env.GOOGLE_CSE_ID;
+        if (googleApiKey && googleCseId) {
+            const params = new URLSearchParams({
+                cx: googleCseId,
+                key: googleApiKey,
+                num: "5",
+                q: query,
+            });
+            const response = await fetchFn(`https://www.googleapis.com/customsearch/v1?${params}`);
+            const data = (await response.json());
+            if (data.items?.length) {
+                results = data.items.map((item) => ({
+                    snippet: item.snippet,
+                    title: item.title,
+                    url: item.link,
+                }));
+            }
+        }
+        // Fallback: Use Jina Reader to search via DuckDuckGo
+        if (results.length === 0) {
+            const ddgUrl = `https://duckduckgo.com/?q=${encodeURIComponent(query)}`;
+            try {
+                const jinaResponse = await fetchFn(`https://r.jina.ai/${ddgUrl}`, {
+                    headers: { Accept: "text/plain" },
+                });
+                if (jinaResponse.ok) {
+                    const text = await jinaResponse.text();
+                    results = this.parseSearchResults(text);
+                }
+            }
+            catch {
+                // Jina search unavailable
+            }
+        }
+        // Final fallback: construct likely Sanity doc URLs from the query
+        if (results.length === 0) {
+            const sanitized = query
+                .toLowerCase()
+                .replace(/sanity\.?(io)?/gi, "")
+                .trim();
+            const slugGuess = sanitized
+                .replace(/\s+/g, "-")
+                .replace(/[^a-z0-9-]/g, "");
+            results = [
+                {
+                    snippet: `Try the documentation page for: ${sanitized}`,
+                    title: `Documentation: ${query}`,
+                    url: `${this.docBaseUrl}/${slugGuess}`,
+                },
+            ];
+        }
+        // -----------------------------------------------------------------------
+        // ORIGIN FILTERING: in "origin-only" mode, restrict results to allowed
+        // origins. This filters search results the same way fetch_page is
+        // sandboxed — the agent only sees results from permitted domains.
+        // -----------------------------------------------------------------------
+        if (this.searchMode === "origin-only" && this.allowedOrigins.length > 0) {
+            const filtered = results.filter((r) => isAllowedOrigin(r.url, this.allowedOrigins));
+            if (filtered.length > 0) {
+                return JSON.stringify(filtered.slice(0, 8));
+            }
+            // No on-origin results — return a helpful fallback
+            return JSON.stringify([
+                {
+                    snippet: `Search was restricted to ${this.allowedOrigins.join(", ")}. ` +
+                        `Try fetching docs directly at ${this.docBaseUrl} or use list_docs.`,
+                    title: "No results found within allowed origins",
+                    url: this.docBaseUrl,
+                },
+            ]);
+        }
+        // "open" mode: return all results (priority-sorted by parseSearchResults)
+        return JSON.stringify(results.slice(0, 8));
+    }
+    /**
+     * Build the set of tools available to the agent based on search mode
+     * and agent mode. When searchMode is "off", web_search is excluded
+     * entirely — the model can't call what it can't see.
+     */
+    getAvailableTools() {
+        const tools = [];
+        // web_search: included unless search mode is "off"
+        if (this.searchMode !== "off") {
+            tools.push(AGENT_TOOLS[0]); // web_search
+        }
+        // fetch_page: always included (origin sandboxing handles restriction)
+        tools.push(AGENT_TOOLS[1]); // fetch_page
+        // list_docs: optimized mode only
+        if (this.agentMode === "optimized") {
+            tools.push(...OPTIMIZED_EXTRA_TOOLS);
+        }
+        return tools;
+    }
+    /**
+     * Merge custom headers into a request's headers.
+     * Custom headers are injected into doc-site requests only — never into
+     * external APIs (OpenAI, Jina, Google).
+     */
+    mergeDocHeaders(baseHeaders, url) {
+        if (Object.keys(this.customHeaders).length === 0)
+            return baseHeaders;
+        // Only inject custom headers for requests to the doc site
+        try {
+            const urlHost = new URL(url).hostname.replace(/^www\./, "");
+            const docHost = new URL(this.docBaseUrl).hostname.replace(/^www\./, "");
+            if (urlHost !== docHost && !urlHost.endsWith(`.${docHost}`)) {
+                return baseHeaders;
+            }
+        }
+        catch {
+            return baseHeaders;
+        }
+        return { ...baseHeaders, ...this.customHeaders };
+    }
+    /**
+     * Parses search results from Jina Reader markdown output.
+     */
+    parseSearchResults(markdown) {
+        const results = [];
+        const lines = markdown.split("\n");
+        let currentTitle = "";
+        let currentUrl = "";
+        let currentSnippet = "";
+        for (const line of lines) {
+            const linkMatch = line.match(/\[([^\]]+)\]\((https?:\/\/[^)]+)\)/);
+            if (linkMatch) {
+                if (currentUrl && currentTitle) {
+                    results.push({
+                        snippet: currentSnippet || currentTitle,
+                        title: currentTitle,
+                        url: currentUrl,
+                    });
+                }
+                currentTitle = linkMatch[1];
+                currentUrl = linkMatch[2];
+                currentSnippet = "";
+                continue;
+            }
+            const urlMatch = line.match(/^(https?:\/\/\S+)/);
+            if (urlMatch && !currentUrl) {
+                currentUrl = urlMatch[1];
+                continue;
+            }
+            if (currentUrl && line.trim().length > 20) {
+                currentSnippet += (currentSnippet ? " " : "") + line.trim();
+            }
+        }
+        if (currentUrl && currentTitle) {
+            results.push({
+                snippet: currentSnippet || currentTitle,
+                title: currentTitle,
+                url: currentUrl,
+            });
+        }
+        // Prioritize results from the configured documentation domain
+        const domain = this.priorityDomain;
+        const priorityResults = results.filter((r) => r.url.includes(domain));
+        const otherResults = results.filter((r) => !r.url.includes(domain));
+        return [...priorityResults, ...otherResults];
+    }
+    /**
+     * Runs the agentic tool-calling loop. Routes to OpenAI or Anthropic
+     * based on the `provider` config field.
+     */
+    async runAgenticLoop(prompt) {
+        const providerType = this.detectProvider();
+        return providerType === "anthropic"
+            ? this.runAnthropicLoop(prompt)
+            : this.runOpenAILoop(prompt);
+    }
+    // -------------------------------------------------------------------------
+    // OpenAI agentic loop
+    // -------------------------------------------------------------------------
+    async runAnthropicLoop(prompt) {
+        const model = this.config.model || "claude-sonnet-4-20250514";
+        const temperature = this.config.temperature ?? 0.2;
+        const maxTokens = this.config.max_tokens || 4096;
+        const maxToolRounds = this.config.maxToolRounds || 5;
+        const apiKey = this.config.apiKey || process.env.ANTHROPIC_API_KEY;
+        if (!apiKey) {
+            return {
+                error: "ANTHROPIC_API_KEY not set. Configure it in env or provider config.",
+                output: undefined,
+            };
+        }
+        const fetchFn = this.recorder.isRunning()
+            ? this.recorder.fetch.bind(this.recorder)
+            : globalThis.fetch;
+        const openAiTools = this.getAvailableTools();
+        const tools = this.toAnthropicTools(openAiTools);
+        const systemPrompt = this.buildSystemPrompt();
+        // Anthropic uses a separate `system` field, not a system message in the array
+        const anthropicMessages = [
+            { content: prompt, role: "user" },
+        ];
+        let inputTokens = 0;
+        let outputTokens = 0;
+        const startTime = Date.now();
+        for (let round = 0; round <= maxToolRounds; round++) {
+            const isLastRound = round === maxToolRounds;
+            // On the last round, omit tools entirely to force a text-only response.
+            // Anthropic doesn't support tool_choice: "none" — the way to disable
+            // tools is to simply not include them in the request.
+            // We also inject a synthesis prompt so the model knows to produce
+            // a final answer from whatever context it has gathered so far.
+            if (isLastRound) {
+                // Ensure the last message is a user message (Anthropic requires
+                // alternating user/assistant). If the last message is already a
+                // user message (tool_result), we can append text to it or add a
+                // new user message.
+                const lastMsg = anthropicMessages[anthropicMessages.length - 1];
+                const synthesisText = "You've gathered enough information. Based on the documentation " +
+                    "and context you've collected, provide your complete, final answer now. " +
+                    "Include all necessary code, imports, and configuration.";
+                if (lastMsg?.role === "user" && Array.isArray(lastMsg.content)) {
+                    // Last message is tool_result blocks — append a text block
+                    ;
+                    lastMsg.content.push({
+                        text: synthesisText,
+                        type: "text",
+                    });
+                }
+                else {
+                    anthropicMessages.push({
+                        content: synthesisText,
+                        role: "user",
+                    });
+                }
+            }
+            const body = {
+                max_tokens: maxTokens,
+                messages: anthropicMessages,
+                model,
+                system: systemPrompt,
+                temperature,
+            };
+            if (!isLastRound) {
+                body.tools = tools;
+            }
+            const response = await fetchFn("https://api.anthropic.com/v1/messages", {
+                body: JSON.stringify(body),
+                headers: {
+                    "anthropic-version": "2023-06-01",
+                    "Content-Type": "application/json",
+                    "x-api-key": apiKey,
+                },
+                method: "POST",
+            });
+            const data = (await response.json());
+            if (data.error) {
+                return {
+                    error: data.error.message ??
+                        `Anthropic API error: ${JSON.stringify(data.error)}`,
+                    output: undefined,
+                };
+            }
+            inputTokens += data.usage?.input_tokens ?? 0;
+            outputTokens += data.usage?.output_tokens ?? 0;
+            if (!data.content?.length) {
+                // Empty content with end_turn is valid (model chose to say nothing).
+                // Return empty output rather than treating as an error.
+                return {
+                    cost: calculateCost(model, inputTokens, outputTokens),
+                    metadata: {
+                        agentMode: this.agentMode,
+                        emptyResponse: true,
+                        latencyMs: Date.now() - startTime,
+                        model,
+                        provider: "anthropic",
+                        toolRounds: round,
+                    },
+                    output: "",
+                    tokenUsage: {
+                        completion: outputTokens,
+                        prompt: inputTokens,
+                        total: inputTokens + outputTokens,
+                    },
+                };
+            }
+            // Add the assistant response to message history
+            anthropicMessages.push({
+                content: data.content,
+                role: "assistant",
+            });
+            // Check if the model wants to use tools
+            const toolUseBlocks = data.content.filter((block) => block.type === "tool_use");
+            if (data.stop_reason !== "tool_use" || toolUseBlocks.length === 0) {
+                // Model is done — extract text response
+                const textBlocks = data.content.filter((block) => block.type === "text");
+                const output = textBlocks.map((b) => b.text).join("\n") || "";
+                return {
+                    cost: calculateCost(model, inputTokens, outputTokens),
+                    metadata: {
+                        agentMode: this.agentMode,
+                        latencyMs: Date.now() - startTime,
+                        model,
+                        provider: "anthropic",
+                        toolRounds: round,
+                    },
+                    output,
+                    tokenUsage: {
+                        completion: outputTokens,
+                        prompt: inputTokens,
+                        total: inputTokens + outputTokens,
+                    },
+                };
+            }
+            // Execute tool calls and add results
+            const toolResults = [];
+            for (const toolUse of toolUseBlocks) {
+                const argsJson = JSON.stringify(toolUse.input);
+                const result = await this.executeTool(toolUse.name, argsJson, fetchFn);
+                toolResults.push({
+                    content: result,
+                    tool_use_id: toolUse.id,
+                    type: "tool_result",
+                });
+            }
+            // Add tool results as a user message (Anthropic's format)
+            anthropicMessages.push({
+                content: toolResults,
+                role: "user",
+            });
+        }
+        // Safety net: exhausted tool rounds
+        const lastAssistantMsg = [...anthropicMessages]
+            .reverse()
+            .find((m) => m.role === "assistant");
+        let lastText = "";
+        if (lastAssistantMsg && Array.isArray(lastAssistantMsg.content)) {
+            const textBlocks = lastAssistantMsg.content.filter((b) => b.type === "text");
+            lastText = textBlocks.map((b) => b.text).join("\n");
+        }
+        return {
+            cost: calculateCost(model, inputTokens, outputTokens),
+            metadata: {
+                agentMode: this.agentMode,
+                exhaustedRounds: true,
+                latencyMs: Date.now() - startTime,
+                model,
+                provider: "anthropic",
+                toolRounds: maxToolRounds,
+            },
+            output: lastText ||
+                "[Agent exhausted tool rounds without producing a final answer]",
+            tokenUsage: {
+                completion: outputTokens,
+                prompt: inputTokens,
+                total: inputTokens + outputTokens,
+            },
+        };
+    }
+    // -------------------------------------------------------------------------
+    // Anthropic agentic loop
+    // -------------------------------------------------------------------------
+    async runOpenAILoop(prompt) {
+        const model = this.config.model || "gpt-4o";
+        const temperature = this.config.temperature ?? 0.2;
+        const maxToolRounds = this.config.maxToolRounds || 5;
+        const apiKey = this.config.apiKey || process.env.OPENAI_API_KEY;
+        // Newer OpenAI models (gpt-5.x, o-series) use max_completion_tokens
+        // instead of max_tokens. Detect from config or model name.
+        const useMaxCompletionTokens = this.config.max_output_tokens != null ||
+            this.config.max_completion_tokens != null ||
+            model.startsWith("gpt-5") ||
+            model.startsWith("o3") ||
+            model.startsWith("o4");
+        const maxTokensValue = this.config.max_output_tokens ??
+            this.config.max_completion_tokens ??
+            this.config.max_tokens ??
+            4096;
+        const tokenLimitParam = useMaxCompletionTokens
+            ? { max_completion_tokens: maxTokensValue }
+            : { max_tokens: maxTokensValue };
+        if (!apiKey) {
+            return {
+                error: "OPENAI_API_KEY not set. Configure it in env or provider config.",
+                output: undefined,
+            };
+        }
+        const fetchFn = this.recorder.isRunning()
+            ? this.recorder.fetch.bind(this.recorder)
+            : globalThis.fetch;
+        const tools = this.getAvailableTools();
+        const systemPrompt = this.buildSystemPrompt();
+        const messages = [
+            { content: systemPrompt, role: "system" },
+            { content: prompt, role: "user" },
+        ];
+        let totalTokens = 0;
+        let promptTokens = 0;
+        let completionTokens = 0;
+        const startTime = Date.now();
+        for (let round = 0; round <= maxToolRounds; round++) {
+            const isLastRound = round === maxToolRounds;
+            const response = await fetchFn("https://api.openai.com/v1/chat/completions", {
+                body: JSON.stringify({
+                    ...tokenLimitParam,
+                    messages,
+                    model,
+                    temperature,
+                    tool_choice: isLastRound ? "none" : "auto",
+                    tools,
+                }),
+                headers: {
+                    Authorization: `Bearer ${apiKey}`,
+                    "Content-Type": "application/json",
+                },
+                method: "POST",
+            });
+            const data = (await response.json());
+            if (data.error) {
+                return {
+                    error: data.error.message ?? "Unknown OpenAI error",
+                    output: undefined,
+                };
+            }
+            totalTokens += data.usage?.total_tokens ?? 0;
+            promptTokens += data.usage?.prompt_tokens ?? 0;
+            completionTokens += data.usage?.completion_tokens ?? 0;
+            const assistantMessage = data.choices?.[0]?.message;
+            const finishReason = data.choices?.[0]?.finish_reason;
+            if (!assistantMessage) {
+                return { error: "No response from model", output: undefined };
+            }
+            messages.push(assistantMessage);
+            if (finishReason !== "tool_calls" ||
+                !assistantMessage.tool_calls?.length) {
+                return {
+                    cost: calculateCost(model, promptTokens, completionTokens),
+                    metadata: {
+                        agentMode: this.agentMode,
+                        latencyMs: Date.now() - startTime,
+                        model,
+                        provider: "openai",
+                        toolRounds: round,
+                    },
+                    output: assistantMessage.content ?? "",
+                    tokenUsage: {
+                        completion: completionTokens,
+                        prompt: promptTokens,
+                        total: totalTokens,
+                    },
+                };
+            }
+            for (const toolCall of assistantMessage.tool_calls) {
+                const toolResult = await this.executeTool(toolCall.function.name, toolCall.function.arguments, fetchFn);
+                messages.push({
+                    content: toolResult,
+                    role: "tool",
+                    tool_call_id: toolCall.id,
+                });
+            }
+        }
+        const lastAssistant = messages
+            .filter((m) => m.role === "assistant" && m.content)
+            .pop();
+        return {
+            cost: calculateCost(model, promptTokens, completionTokens),
+            metadata: {
+                agentMode: this.agentMode,
+                exhaustedRounds: true,
+                latencyMs: Date.now() - startTime,
+                model,
+                provider: "openai",
+                toolRounds: maxToolRounds,
+            },
+            output: lastAssistant?.content ??
+                "[Agent exhausted tool rounds without producing a final answer]",
+            tokenUsage: {
+                completion: completionTokens,
+                prompt: promptTokens,
+                total: totalTokens,
+            },
+        };
+    }
+    /**
+     * Strips HTML tags and normalizes whitespace. Fallback for when
+     * neither .md endpoints nor Jina are available.
+     */
+    stripHtml(html) {
+        return html
+            .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "")
+            .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "")
+            .replace(/<svg[^>]*>[\s\S]*?<\/svg>/gi, "")
+            .replace(/<[^>]+>/g, " ")
+            .replace(/&amp;/g, "&")
+            .replace(/&lt;/g, "<")
+            .replace(/&gt;/g, ">")
+            .replace(/&quot;/g, '"')
+            .replace(/&#39;/g, "'")
+            .replace(/&nbsp;/g, " ")
+            .replace(/\s+/g, " ")
+            .trim();
+    }
+    /**
+     * Convert our ToolDefinition[] to Anthropic's tool format.
+     * Anthropic uses `input_schema` instead of `parameters`.
+     */
+    toAnthropicTools(tools) {
+        return tools.map((t) => ({
+            description: t.function.description,
+            input_schema: t.function
+                .parameters,
+            name: t.function.name,
+        }));
+    }
+}
+/**
+ * Builds a regex that matches documentation URLs for a given base URL.
+ * Handles both http and https, with and without www prefix.
+ *
+ * @example
+ *   buildDocsUrlPattern("https://www.sanity.io/docs")
+ *   // matches: https://sanity.io/docs/schema-types, https://www.sanity.io/docs/
+ */
+function buildDocsUrlPattern(baseUrl) {
+    const { hostname, pathname } = new URL(baseUrl);
+    const escapedHost = hostname.replace(/^www\./, "").replace(/\./g, "\\.");
+    const escapedPath = pathname.replace(/\/$/, "").replace(/\//g, "\\/");
+    return new RegExp(`^https?:\\/\\/(www\\.)?${escapedHost}${escapedPath}(\\/[^?#]*)?$`);
+}
+// ---------------------------------------------------------------------------
+// Agentic provider implementation
+// ---------------------------------------------------------------------------
+/**
+ * Converts a documentation URL to its .md equivalent.
+ * e.g. https://www.sanity.io/docs/schema-types → https://www.sanity.io/docs/schema-types.md
+ */
+function toMarkdownUrl(url) {
+    // Strip trailing slash
+    const clean = url.replace(/\/$/, "");
+    // Don't double-add .md
+    if (clean.endsWith(".md"))
+        return clean;
+    return clean + ".md";
+}