min-agent 0.2.1 → 0.4.0

package/dist/tools/web_fetch.js

@@ -1,8 +1,20 @@
 import { tool, jsonSchema } from "ai";
 import { truncateToolOutput } from "../tool-output.js";
-const FIRECRAWL_BASE = "https://fireclawl.xc.lonae.com";
+import { assertSafeUrl, readBodyLimited } from "../http.js";
+import { loadConfig } from "../config.js";
+import { configHint, describeFetchError, resolveBackendBase } from "./backend.js";
+import { getEffectiveSandboxPolicy, networkDeniedMessage } from "../sandbox.js";
+export const FIRECRAWL_BACKEND = {
+    envName: "MIN_AGENT_FIRECRAWL_URL",
+    configKey: "webFetchURL",
+    fallback: "https://fireclawl.xc.lonae.com",
+};
+const MAX_BODY_BYTES = 2 * 1024 * 1024;
+const DIRECT_TIMEOUT_MS = 8000;
+const SCRAPE_TIMEOUT_MS = 35000;
+const BROWSER_UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36";
 export const webFetchTool = tool({
-    description: "Fetch content from a URL. Use this to access web pages, APIs, or any HTTP resource. Returns the response body as text (or markdown for HTML pages). Supports JavaScript-rendered SPA pages via fallback.",
+    description: "Fetch content from a URL. Use this to access web pages, APIs, or any HTTP resource. Returns the response body as text, or markdown for HTML pages (JavaScript-rendered pages go through a scrape backend). Do not re-fetch URLs you already have; extract what you need and produce the user's deliverable.",
     inputSchema: jsonSchema({
         type: "object",
         properties: {
@@ -12,72 +24,172 @@ export const webFetchTool = tool({
         required: ["url"],
     }),
     execute: async ({ url, method }) => {
-        // First try: direct fetch
-        const directResult = await directFetch(url, method);
-        // If we got meaningful content, return it
-        if (directResult && hasContent(directResult)) {
-            return directResult;
+        if (getEffectiveSandboxPolicy().network === "deny")
+            return `Error: ${networkDeniedMessage()}`;
+        const target = (url ?? "").trim();
+        if (!target)
+            return 'Error: "url" must be a non-empty string.';
+        const blocked = await assertSafeUrl(target);
+        if (blocked)
+            return `Error: ${blocked}`;
+        const verb = (method ?? "GET").trim().toUpperCase() || "GET";
+        const direct = await directFetch(target, verb);
+        // Plain text / JSON responses are what the caller asked for; return them
+        // without paying for a scrape round trip.
+        if (direct.kind === "ok" && !direct.html && hasContent(direct.text)) {
+            return renderBody(direct.text, direct.bodyTruncated);
         }
-        // Fallback: use Firecrawl for SPA/dynamic pages
-        const firecrawlResult = await firecrawlFetch(url);
-        if (firecrawlResult)
-            return firecrawlResult;
-        // Return whatever we got from direct fetch
-        return directResult || "Failed to fetch content from URL";
+        if (direct.kind === "binary") {
+            return `Error: the response is not text (Content-Type: ${direct.contentType}). Use a different tool to download binary content.`;
+        }
+        // A non-GET request cannot be replayed by the scrape backend (it always
+        // performs a GET), so report the direct outcome as-is.
+        if (verb !== "GET") {
+            if (direct.kind === "ok")
+                return renderBody(direct.text, direct.bodyTruncated);
+            return `Error: ${describeDirect(direct, target)}`;
+        }
+        // HTML (or an empty/JS-rendered shell): the scrape backend turns it into
+        // markdown, which is both cheaper and more readable than raw markup.
+        const scraped = await firecrawlFetch(target);
+        if (scraped.kind === "ok")
+            return renderBody(scraped.markdown, scraped.bodyTruncated);
+        if (direct.kind === "ok" && hasContent(direct.text)) {
+            return [
+                `Note: returning raw HTML — the scrape backend could not convert this page (${scraped.reason}).`,
+                renderBody(direct.text, direct.bodyTruncated),
+            ].join("\n\n");
+        }
+        return [
+            `Failed to fetch content from ${target}.`,
+            `Direct request: ${describeDirect(direct, target)}`,
+            `Scrape backend: ${scraped.reason}`,
+            configHint(FIRECRAWL_BACKEND),
+        ].join("\n");
     },
 });
+function renderBody(text, bodyTruncated) {
+    const { content } = truncateToolOutput(text, { direction: "head" });
+    // readBodyLimited cutting at MAX_BODY_BYTES is a different truncation from
+    // truncateToolOutput's (which persists the full text), so disclose it.
+    if (!bodyTruncated)
+        return content;
+    return `${content}\n\n...(response body was cut off at ${MAX_BODY_BYTES / 1024 / 1024} MB)`;
+}
+function describeDirect(outcome, url) {
+    switch (outcome.kind) {
+        case "ok":
+            return `HTTP 200 but the body had almost no readable content (${url} may require JavaScript)`;
+        case "binary":
+            return `non-text response (${outcome.contentType})`;
+        case "http":
+            return `HTTP ${outcome.status}${outcome.statusText ? ` ${outcome.statusText}` : ""}`;
+        case "network":
+            return outcome.message;
+    }
+}
+function isHtml(contentType) {
+    return /html|xhtml/i.test(contentType);
+}
+/** Content types worth handing to the model as text. */
+function isTextual(contentType) {
+    if (!contentType)
+        return true; // Many APIs omit it; the NUL sniff below catches binaries.
+    const ct = contentType.toLowerCase();
+    if (ct.startsWith("text/"))
+        return true;
+    return /(json|xml|javascript|ecmascript|yaml|csv|graphql|x-www-form-urlencoded|\+json)/.test(ct);
+}
 async function directFetch(url, method) {
+    let response;
     try {
-        const response = await fetch(url, {
-            method: method ?? "GET",
+        response = await fetch(url, {
+            method,
             headers: {
-                "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
-                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+                "User-Agent": BROWSER_UA,
+                Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,application/json;q=0.9,*/*;q=0.8",
             },
-            signal: AbortSignal.timeout(15000),
+            signal: AbortSignal.timeout(DIRECT_TIMEOUT_MS),
         });
-        if (!response.ok)
-            return `HTTP ${response.status}`;
-        const text = await response.text();
-        return truncateToolOutput(text, { direction: "head" }).content;
     }
     catch (err) {
-        return null;
+        return { kind: "network", message: describeFetchError(err) };
+    }
+    const contentType = response.headers.get("content-type") ?? "";
+    if (!response.ok)
+        return { kind: "http", status: response.status, statusText: response.statusText };
+    if (!isTextual(contentType))
+        return { kind: "binary", contentType };
+    try {
+        const { text, truncated } = await readBodyLimited(response, MAX_BODY_BYTES);
+        if (text.includes("\u0000"))
+            return { kind: "binary", contentType: contentType || "unknown (binary)" };
+        return { kind: "ok", text, bodyTruncated: truncated, html: isHtml(contentType) || looksLikeHtml(text) };
+    }
+    catch (err) {
+        return { kind: "network", message: describeFetchError(err) };
     }
 }
 async function firecrawlFetch(url) {
+    const resolved = resolveBackendBase(FIRECRAWL_BACKEND, loadConfig().webFetchURL);
+    if (!resolved.ok)
+        return { kind: "unavailable", reason: resolved.error };
+    let response;
     try {
-        const response = await fetch(`${FIRECRAWL_BASE}/v1/scrape`, {
+        response = await fetch(`${resolved.base}/v1/scrape`, {
             method: "POST",
             headers: { "Content-Type": "application/json" },
-            body: JSON.stringify({
-                url,
-                formats: ["markdown"],
-                onlyMainContent: true,
-                waitFor: 3000,
-                timeout: 30000,
-            }),
-            signal: AbortSignal.timeout(35000),
+            body: JSON.stringify({ url, formats: ["markdown"], onlyMainContent: true, waitFor: 3000, timeout: 30000 }),
+            signal: AbortSignal.timeout(SCRAPE_TIMEOUT_MS),
         });
-        if (!response.ok)
-            return null;
-        const data = await response.json();
-        const markdown = data?.data?.markdown || data?.data?.content;
-        if (!markdown)
-            return null;
-        return truncateToolOutput(markdown, { direction: "head" }).content;
+    }
+    catch (err) {
+        return { kind: "unavailable", reason: `could not reach the scrape backend (${describeFetchError(err)})` };
+    }
+    let text;
+    let truncated;
+    try {
+        const body = await readBodyLimited(response, MAX_BODY_BYTES);
+        text = body.text;
+        truncated = body.truncated;
+    }
+    catch (err) {
+        return { kind: "unavailable", reason: `could not read the scrape response (${describeFetchError(err)})` };
+    }
+    let data;
+    try {
+        data = JSON.parse(text);
     }
     catch {
-        return null;
+        return {
+            kind: "unavailable",
+            reason: response.ok
+                ? "the scrape backend returned a non-JSON response"
+                : `HTTP ${response.status} from the scrape backend`,
+        };
     }
+    const record = typeof data === "object" && data !== null ? data : {};
+    if (!response.ok) {
+        const message = typeof record.error === "string" ? record.error : response.statusText;
+        return {
+            kind: "unavailable",
+            reason: `HTTP ${response.status} from the scrape backend${message ? ` (${message})` : ""}`,
+        };
+    }
+    const payload = typeof record.data === "object" && record.data !== null ? record.data : {};
+    const markdown = typeof payload.markdown === "string" ? payload.markdown : typeof payload.content === "string" ? payload.content : "";
+    if (!markdown.trim())
+        return { kind: "unavailable", reason: "the scrape backend returned no content for this URL" };
+    return { kind: "ok", markdown, bodyTruncated: truncated };
+}
+function looksLikeHtml(body) {
+    return /^\s*(<!doctype html|<html\b|<head\b)/i.test(body);
 }
-function hasContent(html) {
-    // Check if the response has meaningful content (not just an empty SPA shell)
-    if (html.length < 200)
+/** Whether a body carries enough readable text to be worth returning as-is. */
+function hasContent(body) {
+    if (body.trim().length < 200)
         return false;
-    // SPA shells typically have very little text content outside of script tags
-    const withoutScripts = html.replace(/<script[\s\S]*?<\/script>/gi, "");
+    const withoutScripts = body.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "");
     const textContent = withoutScripts.replace(/<[^>]*>/g, "").trim();
-    // If after removing scripts and tags there's less than 100 chars, it's likely an empty shell
     return textContent.length > 100;
 }

package/dist/tools/web_search.js

@@ -1,40 +1,228 @@
 import { tool, jsonSchema } from "ai";
-import { truncateToolOutput } from "../tool-output.js";
-const SEARXNG_BASE = "https://searxng.xc.lonae.com";
+import { readBodyLimited } from "../http.js";
+import { describeFetchError } from "./backend.js";
+import { getEffectiveSandboxPolicy, networkDeniedMessage } from "../sandbox.js";
+import { runSerper } from "./search-serper.js";
+import { runSearxng, searxngConfigured } from "./search-searxng.js";
+export { SEARXNG_BACKEND } from "./search-searxng.js";
+export const MAX_BODY_BYTES = 2 * 1024 * 1024;
+const DEFAULT_MAX_RESULTS = 10;
+const MAX_RESULTS_LIMIT = 30;
+const SNIPPET_MAX_CHARS = 300;
+export const VALID_TIME_RANGES = new Set(["day", "week", "month", "year"]);
+/** ISO 639-1/2 with an optional region subtag, or SearXNG's "all". */
+export const LANGUAGE_RE = /^(all|[a-z]{2,3}([-_][a-zA-Z]{2,4})?)$/;
+const EXACT_TRACKING_PARAMS = new Set([
+    "ref",
+    "ref_src",
+    "referrer",
+    "fbclid",
+    "gclid",
+    "msclkid",
+    "igshid",
+    "mc_cid",
+    "mc_eid",
+    "spm",
+    "source",
+]);
+export function isString(v) {
+    return typeof v === "string";
+}
+export function collapse(s) {
+    return s.replace(/\s+/g, " ").trim();
+}
+export function clamp(s, max) {
+    const chars = Array.from(s);
+    if (chars.length <= max)
+        return s;
+    return chars.slice(0, max - 1).join("") + "…";
+}
+export function hostOf(url) {
+    try {
+        return new URL(url).hostname.replace(/^www\./, "");
+    }
+    catch {
+        return url;
+    }
+}
+/** Normalize a published date to YYYY-MM-DD; pass unparseable values through. */
+function formatPublished(raw) {
+    if (!raw)
+        return "";
+    const ms = Date.parse(raw);
+    if (Number.isNaN(ms))
+        return collapse(raw);
+    return new Date(ms).toISOString().slice(0, 10);
+}
+/**
+ * Identity key for deduplication: scheme, "www.", trailing slash, fragment and
+ * tracking parameters all describe the same page.
+ */
+export function dedupeKey(raw) {
+    try {
+        const u = new URL(raw);
+        const host = u.hostname.toLowerCase().replace(/^www\./, "");
+        const path = u.pathname.replace(/\/+$/, "");
+        const params = [...u.searchParams].filter(([k]) => !k.toLowerCase().startsWith("utm_") && !EXACT_TRACKING_PARAMS.has(k.toLowerCase()));
+        params.sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0));
+        const query = params.map(([k, v]) => `${k}=${v}`).join("&");
+        return `${host}${path}${query ? `?${query}` : ""}`;
+    }
+    catch {
+        return raw.trim().toLowerCase();
+    }
+}
+export function dedupeByUrl(results) {
+    const seen = new Set();
+    return results.filter((r) => {
+        const key = dedupeKey(r.url);
+        if (seen.has(key))
+            return false;
+        seen.add(key);
+        return true;
+    });
+}
+/**
+ * One block per result:
+ *   1. Title
+ *      https://url
+ *      [2026-08-18 · google]
+ *      snippet on a single line
+ * Snippets are whitespace-collapsed and clamped so the indented block
+ * structure survives multi-line engine output.
+ */
+export function formatResults(results) {
+    return results
+        .map((r, i) => {
+        const lines = [`${i + 1}. ${collapse(r.title) || hostOf(r.url)}`, `   ${r.url}`];
+        const meta = [];
+        const date = formatPublished(r.publishedDate);
+        if (date)
+            meta.push(date);
+        if (r.engine)
+            meta.push(r.engine);
+        if (meta.length > 0)
+            lines.push(`   [${meta.join(" · ")}]`);
+        const snippet = clamp(collapse(r.content), SNIPPET_MAX_CHARS);
+        if (snippet)
+            lines.push(`   ${snippet}`);
+        return lines.join("\n");
+    })
+        .join("\n\n");
+}
+export function resolveMaxResults(raw, notes) {
+    if (raw === undefined)
+        return DEFAULT_MAX_RESULTS;
+    const n = Math.floor(Number(raw));
+    if (!Number.isFinite(n) || n < 1) {
+        notes.push(`ignored invalid max_results "${raw}"; using ${DEFAULT_MAX_RESULTS}`);
+        return DEFAULT_MAX_RESULTS;
+    }
+    if (n > MAX_RESULTS_LIMIT) {
+        notes.push(`max_results capped at ${MAX_RESULTS_LIMIT}`);
+        return MAX_RESULTS_LIMIT;
+    }
+    return n;
+}
+export function emptyResponse() {
+    return {
+        results: [],
+        answers: [],
+        suggestions: [],
+        corrections: [],
+        infoboxes: [],
+        unresponsive_engines: [],
+    };
+}
+/** Backends report parameter problems as {"error": "..."} — sometimes with HTTP 200. */
+function backendErrorMessage(data) {
+    if (typeof data !== "object" || data === null)
+        return "";
+    const o = data;
+    for (const key of ["error", "detail", "message"]) {
+        const v = o[key];
+        if (isString(v) && v.trim())
+            return collapse(v);
+        if (typeof v === "object" && v !== null) {
+            const nested = v.message;
+            if (isString(nested) && nested.trim())
+                return collapse(nested);
+        }
+    }
+    return "";
+}
+/**
+ * Shared fetch scaffolding for both search backends: read the body with a
+ * size cap, then distinguish HTTP errors, non-JSON bodies and backend-reported
+ * errors before handing valid JSON to the backend-specific parser. Error
+ * bodies are valid JSON even on a 4xx, so parse before branching on the
+ * status; that way the backend's own message reaches the caller.
+ */
+export async function fetchJsonOutcome(url, init, parse) {
+    let response;
+    try {
+        response = await fetch(url, init);
+    }
+    catch (err) {
+        return { kind: "network", message: describeFetchError(err) };
+    }
+    let text;
+    let bodyTooLarge;
+    try {
+        const body = await readBodyLimited(response, MAX_BODY_BYTES);
+        text = body.text;
+        bodyTooLarge = body.truncated;
+    }
+    catch (err) {
+        return { kind: "network", message: describeFetchError(err) };
+    }
+    let parsed;
+    let parseFailed = false;
+    try {
+        parsed = JSON.parse(text);
+    }
+    catch {
+        parseFailed = true;
+    }
+    const backendMessage = parseFailed ? "" : backendErrorMessage(parsed);
+    if (!response.ok)
+        return { kind: "http", status: response.status, backendMessage };
+    if (parseFailed) {
+        return { kind: "invalid-json", bodyTooLarge, sample: clamp(collapse(text), 100) };
+    }
+    // A truncated body that still parsed would be missing results silently.
+    if (bodyTooLarge)
+        return { kind: "invalid-json", bodyTooLarge: true, sample: "" };
+    if (backendMessage)
+        return { kind: "http", status: response.status, backendMessage };
+    return { kind: "ok", data: parse(parsed) };
+}
 export const webSearchTool = tool({
-    description: "Search the web for information. Returns search results with titles, URLs, and snippets. Use this when you need current information, news, documentation, or answers that require up-to-date knowledge.",
+    description: `Search the web via Google. Returns ranked results with title, URL, source and a snippet. Use this for current information, news, documentation, or anything needing up-to-date knowledge. Today's date: ${new Date().toISOString().slice(0, 10)}. When the user asks about the latest or newest state of something, use the current year in the query or set time_range (day, week, month, year) — do not append an older year from your training knowledge. A few targeted queries plus fetching the best sources is enough; then produce the user's deliverable instead of searching exhaustively.`,
     inputSchema: jsonSchema({
         type: "object",
         properties: {
             query: { type: "string", description: "The search query" },
-            categories: { type: "string", description: "Search categories: general, news, images, science, it (default: general)" },
+            language: { type: "string", description: "Search language code, e.g. zh, en, zh-CN (default: en)" },
+            time_range: { type: "string", description: "Time range filter: day, week, month, year" },
+            max_results: {
+                type: "number",
+                description: `Maximum number of results to return (default ${DEFAULT_MAX_RESULTS}, max ${MAX_RESULTS_LIMIT})`,
+            },
         },
         required: ["query"],
     }),
-    execute: async ({ query, categories }) => {
-        try {
-            const params = new URLSearchParams({
-                q: query,
-                format: "json",
-                categories: categories ?? "general",
-            });
-            const response = await fetch(`${SEARXNG_BASE}/search?${params}`, {
-                headers: { "Accept": "application/json" },
-                signal: AbortSignal.timeout(15000),
-            });
-            if (!response.ok)
-                return `Search error: HTTP ${response.status}`;
-            const data = await response.json();
-            const results = (data.results ?? []).slice(0, 10);
-            if (results.length === 0)
-                return "No search results found. Try a different query.";
-            const text = results
-                .map((r, i) => `${i + 1}. ${r.title}\n   ${r.url}\n   ${r.content ?? ""}`)
-                .join("\n\n");
-            return truncateToolOutput(text, { direction: "head" }).content;
-        }
-        catch (err) {
-            return `Search error: ${err.message}`;
-        }
+    execute: async (input) => {
+        if (getEffectiveSandboxPolicy().network === "deny")
+            return `Search error: ${networkDeniedMessage()}`;
+        const query = (input.query ?? "").trim();
+        if (!query)
+            return 'Search error: "query" must be a non-empty string.';
+        const notes = [];
+        // SearXNG is only used when the user explicitly configures an instance;
+        // otherwise Serper is used.
+        if (searxngConfigured())
+            return runSearxng(input, query, notes);
+        return runSerper(input, query, notes);
     },
 });

package/dist/tools/write.js

@@ -1,32 +1,55 @@
 import { tool, jsonSchema } from "ai";
-import { writeFileSync, mkdirSync, existsSync } from "fs";
+import { mkdir, writeFile } from "fs/promises";
+import { existsSync } from "fs";
 import path from "path";
-import { confirm, isAutoApprove } from "../confirm.js";
+import { confirm, isEditAutoApprove } from "../confirm.js";
+import { pathAccessError } from "../sandbox.js";
+import { atomicWriteFile } from "./atomic-file.js";
+const MAX_WRITE_BYTES = 10 * 1024 * 1024;
 export const writeTool = tool({
-    description: "Write content to a file. Creates the file if it doesn't exist, overwrites if it does. Creates parent directories as needed.",
+    description: "Write content to a file. Creates the file if it doesn't exist, overwrites if it does (unless `append` is true). Creates parent directories as needed.",
     inputSchema: jsonSchema({
         type: "object",
         properties: {
             filePath: { type: "string", description: "Path to the file to write (relative to cwd or absolute)" },
             content: { type: "string", description: "The content to write to the file" },
+            append: { type: "boolean", description: "If true, append to the file instead of overwriting" },
         },
         required: ["filePath", "content"],
     }),
-    execute: async ({ filePath, content }) => {
+    execute: async ({ filePath, content, append }) => {
+        if (Buffer.byteLength(content, "utf-8") > MAX_WRITE_BYTES) {
+            return `Error: content exceeds ${MAX_WRITE_BYTES} bytes. Write in smaller chunks or check the content size.`;
+        }
+        const denied = pathAccessError(filePath, "write");
+        if (denied)
+            return `Error: ${denied}`;
         const resolved = path.resolve(process.cwd(), filePath);
-        // Confirm overwriting existing files
-        if (!isAutoApprove() && existsSync(resolved)) {
-            const approved = await confirm(`Overwrite existing file: ${filePath}`);
+        // Same confirmation semantics as edit: in ask mode every mutation —
+        // create, overwrite or append — is confirmed; accept-edits skips them all.
+        if (!isEditAutoApprove()) {
+            const exists = existsSync(resolved);
+            const action = append
+                ? `Append to file: ${filePath}`
+                : exists
+                    ? `Overwrite existing file: ${filePath}`
+                    : `Create new file: ${filePath}`;
+            const approved = await confirm(action);
             if (!approved)
                 return "Write rejected by user.";
         }
         try {
-            mkdirSync(path.dirname(resolved), { recursive: true });
-            writeFileSync(resolved, content, "utf-8");
-            return `Written ${content.length} bytes to ${filePath}`;
+            await mkdir(path.dirname(resolved), { recursive: true });
+            if (append) {
+                await writeFile(resolved, content, { flag: "a", encoding: "utf-8" });
+            }
+            else {
+                await atomicWriteFile(resolved, content);
+            }
+            return `Written ${Buffer.byteLength(content, "utf-8")} bytes to ${filePath}${append ? " (appended)" : ""}`;
         }
         catch (err) {
-            return `Error writing file: ${err.message}`;
+            return `Error writing file: ${err instanceof Error ? err.message : String(err)}`;
         }
     },
 });