@robzilla1738/agentswarm 0.5.0 → 0.6.0

package/dist/tools.js

@@ -40,6 +40,7 @@ exports.synthToolset = synthToolset;
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const crawltools_1 = require("./crawltools");
+const searchcore_1 = require("./searchcore");
 const util_1 = require("./util");
 const webtools_1 = require("./webtools");
 // ---------- safety ----------
@@ -62,9 +63,48 @@ function checkCommand(cmd, cfg) {
 function resolveRead(p, ctx) {
     return path.resolve(ctx.workdir, p);
 }
+/** Single-quote a string for sh. */
+function shq(s) {
+    return `'${s.replace(/'/g, `'\\''`)}'`;
+}
+/**
+ * Where a write actually lands: realpath of the deepest existing ancestor plus
+ * the not-yet-created remainder. Confinement checks must use this, or a
+ * symlink inside the workdir smuggles writes anywhere on the host.
+ */
+function realDestination(abs) {
+    let dir = abs;
+    const tail = [];
+    while (!fs.existsSync(dir)) {
+        tail.unshift(path.basename(dir));
+        const parent = path.dirname(dir);
+        if (parent === dir)
+            break;
+        dir = parent;
+    }
+    try {
+        dir = fs.realpathSync(dir);
+    }
+    catch {
+        /* races/permissions: keep the lexical path */
+    }
+    return path.join(dir, ...tail);
+}
+function realBase(base) {
+    try {
+        return fs.realpathSync(base);
+    }
+    catch {
+        return base;
+    }
+}
 function resolveWrite(p, ctx) {
     const abs = path.resolve(ctx.workdir, p);
-    const ok = (0, util_1.pathInside)(ctx.workdir, abs) || (0, util_1.pathInside)(ctx.runDirPath, abs) || !ctx.cfg.safeMode;
+    // Remote sandboxes own their filesystem — host-side realpath is meaningless there.
+    const real = ctx.sandbox.localFs ? realDestination(abs) : abs;
+    const ok = (0, util_1.pathInside)(realBase(ctx.workdir), real) ||
+        (0, util_1.pathInside)(realBase(ctx.runDirPath), real) ||
+        !ctx.cfg.safeMode;
     if (!ok) {
         throw new Error(`safeMode: writes are restricted to the working directory (${ctx.workdir}). ` +
             `Use a relative path, or save deliverables with save_artifact.`);
@@ -171,7 +211,7 @@ function workerToolset(cfg) {
     tools.replace_in_file = {
         schema: {
             name: "replace_in_file",
-            description: "Exact string replacement in a file. `find` must match exactly (including whitespace). Fails if not found, or if ambiguous when all=false.",
+            description: "Exact string replacement in a file. `find` must match exactly (including whitespace). Fails if not found, or if ambiguous when all=false. For several edits to the same file, pass `edits` — they apply in order, all-or-nothing, in one call.",
             parameters: {
                 type: "object",
                 properties: {
@@ -179,25 +219,96 @@ function workerToolset(cfg) {
                     find: { type: "string" },
                     replace: { type: "string" },
                     all: { type: "boolean", description: "Replace every occurrence (default false)" },
+                    edits: {
+                        type: "array",
+                        description: "Batch mode: multiple find/replace pairs applied in order, atomically (replaces top-level find/replace)",
+                        items: {
+                            type: "object",
+                            properties: {
+                                find: { type: "string" },
+                                replace: { type: "string" },
+                                all: { type: "boolean" },
+                            },
+                            required: ["find", "replace"],
+                        },
+                    },
                 },
-                required: ["path", "find", "replace"],
+                required: ["path"],
             },
         },
         run: async (args, ctx) => {
             const abs = resolveWrite(String(args.path), ctx);
             const raw = await readFileVia(ctx, abs);
-            const find = String(args.find);
-            const replace = String(args.replace);
-            const count = raw.split(find).length - 1;
-            if (count === 0)
-                throw new Error("find string not found in file");
-            if (count > 1 && !args.all) {
-                throw new Error(`find string matches ${count} times; provide more context or set all=true`);
+            const edits = Array.isArray(args.edits) && args.edits.length
+                ? args.edits.map((e) => ({
+                    find: String(e.find ?? ""),
+                    replace: String(e.replace ?? ""),
+                    all: Boolean(e.all),
+                }))
+                : args.find !== undefined && args.replace !== undefined
+                    ? [{ find: String(args.find), replace: String(args.replace), all: Boolean(args.all) }]
+                    : null;
+            if (!edits)
+                throw new Error("provide find+replace, or an edits array");
+            // Validate-then-apply against the progressively edited content:
+            // any failing edit aborts the whole batch with nothing written.
+            let next = raw;
+            let total = 0;
+            const at = (i) => (edits.length > 1 ? `edit ${i + 1}: ` : "");
+            for (let i = 0; i < edits.length; i++) {
+                const { find, replace, all } = edits[i];
+                if (!find)
+                    throw new Error(`${at(i)}find must not be empty`);
+                const count = next.split(find).length - 1;
+                if (count === 0) {
+                    throw new Error(`${at(i)}find string not found in file${edits.length > 1 ? " — no edits were applied" : ""}`);
+                }
+                if (count > 1 && !all) {
+                    throw new Error(`${at(i)}find string matches ${count} times; provide more context or set all=true${edits.length > 1 ? " — no edits were applied" : ""}`);
+                }
+                next = all ? next.split(find).join(replace) : next.replace(find, replace);
+                total += all ? count : 1;
             }
-            const next = args.all ? raw.split(find).join(replace) : raw.replace(find, replace);
             await writeFileVia(ctx, abs, next);
             const warn = ctx.checkClaim?.(String(args.path));
-            return `replaced ${args.all ? count : 1} occurrence(s) in ${abs}${warn ? `\n${warn}` : ""}`;
+            return `replaced ${total} occurrence(s) via ${edits.length} edit(s) in ${abs}${warn ? `\n${warn}` : ""}`;
+        },
+    };
+    tools.grep_files = {
+        schema: {
+            name: "grep_files",
+            description: "Search file contents with a regex (grep -E syntax). Returns matching lines as path:line:text. Use this to locate code or text instead of shell grep pipelines — one round-trip, works identically in remote sandboxes, skips node_modules/.git/build output.",
+            parameters: {
+                type: "object",
+                properties: {
+                    pattern: { type: "string", description: "Extended regex (grep -E)" },
+                    path: { type: "string", description: "Directory or file to search (default: working directory)" },
+                    glob: { type: "string", description: "Filename filter, e.g. *.ts" },
+                    ignore_case: { type: "boolean" },
+                    max_results: { type: "number", description: "Default 50, max 200" },
+                },
+                required: ["pattern"],
+            },
+        },
+        run: async (args, ctx) => {
+            const pattern = String(args.pattern ?? "");
+            if (!pattern.trim())
+                throw new Error("pattern is required");
+            const root = args.path ? resolveRead(String(args.path), ctx) : ctx.workdir;
+            const max = Math.min(Math.max(Number(args.max_results) || 50, 1), 200);
+            const flags = `-rnE${args.ignore_case ? "i" : ""}`;
+            const include = args.glob ? ` --include=${shq(String(args.glob))}` : "";
+            const excludes = ["node_modules", ".git", "dist", ".next", "out", "build", "target", "__pycache__", ".venv"]
+                .map((d) => ` --exclude-dir=${d}`)
+                .join("");
+            const cmd = `grep ${flags}${include}${excludes} -e ${shq(pattern)} ${shq(root)} | head -n ${max + 1}`;
+            const r = await ctx.sandbox.exec(cmd, { cwd: ctx.workdir, timeoutSec: 60, signal: ctx.signal });
+            const lines = r.out.split("\n").filter(Boolean);
+            if (!lines.length)
+                return "no matches";
+            const shown = lines.slice(0, max);
+            const more = lines.length > max ? `\n…more matches truncated (raise max_results or narrow the pattern)` : "";
+            return shown.join("\n") + more;
         },
     };
     tools.list_dir = {
@@ -295,6 +406,39 @@ function workerToolset(cfg) {
                 .join("\n");
         },
     };
+    tools.academic_search = {
+        schema: {
+            name: "academic_search",
+            description: "Search scholarly sources: arXiv preprints and Crossref journal/conference metadata (keyless APIs). Returns papers with title, link (arXiv/DOI), abstract snippet, and date. Use for scientific or technical questions where peer-reviewed and preprint sources beat the open web.",
+            parameters: {
+                type: "object",
+                properties: {
+                    query: { type: "string" },
+                    count: { type: "number", description: "Max results, default 8, max 20" },
+                },
+                required: ["query"],
+            },
+        },
+        run: async (args, ctx) => {
+            const count = Math.min(Math.max(Number(args.count) || 8, 1), 20);
+            const q = String(args.query);
+            const settled = await Promise.allSettled([
+                (0, webtools_1.arxivSearch)(q, count, ctx.signal),
+                (0, webtools_1.crossrefSearch)(q, count, ctx.signal),
+            ]);
+            const candidates = settled.flatMap((s) => (s.status === "fulfilled" ? s.value : []));
+            if (!candidates.length) {
+                const err = settled.find((s) => s.status === "rejected");
+                if (err)
+                    throw err.reason;
+                return "no results";
+            }
+            const merged = (0, searchcore_1.mergeCandidates)(candidates, count);
+            return merged
+                .map((h, i) => `${i + 1}. ${h.title}${h.date ? ` (${h.date})` : ""} [${h.engine}]\n   ${h.url}\n   ${h.snippet}`)
+                .join("\n");
+        },
+    };
     tools.fetch_url = {
         schema: {
             name: "fetch_url",
@@ -318,7 +462,7 @@ function workerToolset(cfg) {
     tools.note = {
         schema: {
             name: "note",
-            description: "Post a durable fact/discovery to the swarm's shared blackboard so the conductor and other agents can see it. Use sparingly — facts other tasks need, not progress chatter. Mark kind='decision' for choices the rest of the mission must respect (these are never trimmed from digests).",
+            description: "Post a durable fact/discovery to the swarm's shared blackboard so the conductor and other agents can see it. Use sparingly — facts other tasks need, not progress chatter. Mark kind='decision' for choices the rest of the mission must respect, and kind='conflict' when independent sources disagree on a material fact (both are never trimmed from digests).",
             parameters: {
                 type: "object",
                 properties: {
@@ -326,18 +470,20 @@ function workerToolset(cfg) {
                     key: { type: "string", description: "Optional short label" },
                     kind: {
                         type: "string",
-                        enum: ["finding", "decision", "open-question", "handoff", "claim"],
-                        description: "Category (default finding). kind='claim' with key=<file path> advertises you are editing that file",
+                        enum: ["finding", "decision", "conflict", "open-question", "handoff", "claim"],
+                        description: "Category (default finding). kind='conflict' flags sources that disagree — name both. kind='claim' with key=<file path> advertises you are editing that file",
                     },
+                    url: { type: "string", description: "Source URL backing this note, when it came from the web" },
                 },
                 required: ["text"],
             },
         },
         run: async (args, ctx) => {
-            const kind = ["finding", "decision", "open-question", "handoff", "claim"].includes(String(args.kind))
+            const kind = ["finding", "decision", "conflict", "open-question", "handoff", "claim"].includes(String(args.kind))
                 ? String(args.kind)
                 : undefined;
-            ctx.addNote(String(args.text), args.key ? String(args.key) : undefined, kind);
+            const url = /^https?:\/\//.test(String(args.url ?? "")) ? String(args.url) : undefined;
+            ctx.addNote(String(args.text), args.key ? String(args.key) : undefined, kind, url);
             return "noted on the blackboard";
         },
     };
@@ -416,8 +562,12 @@ function workerToolset(cfg) {
         },
         run: async (args, ctx) => {
             const name = String(args.name).replace(/^\/+/, "");
-            const dest = path.join(ctx.runDirPath, "artifacts", name);
-            if (!(0, util_1.pathInside)(path.join(ctx.runDirPath, "artifacts"), dest)) {
+            const artifactsRoot = path.join(ctx.runDirPath, "artifacts");
+            (0, util_1.ensureDir)(artifactsRoot);
+            const dest = path.join(artifactsRoot, name);
+            // Realpath-based: neither ../ traversal nor a planted symlink may move
+            // the artifact outside the run's artifacts folder.
+            if (!(0, util_1.pathInside)(realBase(artifactsRoot), realDestination(dest))) {
                 throw new Error("artifact name must stay inside the artifacts folder");
             }
             (0, util_1.ensureDir)(path.dirname(dest));
@@ -546,6 +696,20 @@ exports.REPORT_TOOL = {
                 items: { type: "string" },
                 description: "Every file you created or modified (exact paths)",
             },
+            sources: {
+                type: "array",
+                description: "Web sources your findings rely on — REQUIRED whenever your work drew on the web. They flow into the final report's bibliography; a web-sourced claim without an entry here cannot be cited.",
+                items: {
+                    type: "object",
+                    properties: {
+                        url: { type: "string" },
+                        title: { type: "string" },
+                        date: { type: "string", description: "Publication date if known (ISO or year)" },
+                        note: { type: "string", description: "What this source supports" },
+                    },
+                    required: ["url"],
+                },
+            },
         },
         required: ["status", "report"],
     },
@@ -561,6 +725,19 @@ exports.VERDICT_TOOL = {
                 type: "string",
                 description: "If fail: exactly what is wrong and where. If pass: one-line confirmation of the evidence.",
             },
+            issues: {
+                type: "array",
+                description: "On fail: one entry per concrete problem. The worker's retry sees these verbatim — make each actionable.",
+                items: {
+                    type: "object",
+                    properties: {
+                        problem: { type: "string", description: "What is wrong" },
+                        evidence: { type: "string", description: "What you observed that proves it (command output, file content, URL)" },
+                        fix: { type: "string", description: "The exact change that would resolve it" },
+                    },
+                    required: ["problem"],
+                },
+            },
         },
         required: ["pass", "feedback"],
     },

package/dist/util.js

@@ -48,6 +48,7 @@ exports.ensureDir = ensureDir;
 exports.readJson = readJson;
 exports.writeJson = writeJson;
 exports.pathInside = pathInside;
+exports.validateArtifactFormat = validateArtifactFormat;
 exports.decodeEntities = decodeEntities;
 exports.htmlToText = htmlToText;
 const fs = __importStar(require("fs"));
@@ -147,6 +148,90 @@ function pathInside(parent, child) {
     const rel = path.relative(path.resolve(parent), path.resolve(child));
     return rel === "" || (!rel.startsWith("..") && !path.isAbsolute(rel));
 }
+// ---------- artifact validation ----------
+/**
+ * Cheap structural checks for common deliverable formats — catches a worker
+ * shipping malformed JSON/CSV/stub HTML before an LLM verifier spends tokens
+ * on it. Returns a problem description, or null when the file looks sound
+ * (or is a format we don't check).
+ */
+function validateArtifactFormat(absPath) {
+    const ext = path.extname(absPath).toLowerCase();
+    if (![".json", ".csv", ".html", ".htm"].includes(ext))
+        return null;
+    let raw;
+    try {
+        raw = fs.readFileSync(absPath, "utf8");
+    }
+    catch {
+        return null; // existence/size is the caller's check
+    }
+    if (ext === ".json") {
+        try {
+            JSON.parse(raw);
+            return null;
+        }
+        catch (e) {
+            return `not valid JSON (${errMsg(e)})`;
+        }
+    }
+    if (ext === ".csv") {
+        const counts = csvFieldCounts(raw, 50);
+        if (!counts.length)
+            return "CSV has no records";
+        const expect = counts[0];
+        const bad = counts.findIndex((c) => c !== expect);
+        if (bad > 0)
+            return `inconsistent CSV: record 1 has ${expect} field(s), record ${bad + 1} has ${counts[bad]}`;
+        return null;
+    }
+    // .html / .htm — catch empty shells and plain text passed off as HTML.
+    if (raw.length < 200 || !/<[a-z!][^>]*>/i.test(raw) || !/<\/[a-z][a-z0-9]*>/i.test(raw)) {
+        return "HTML looks like a stub (too short or no real markup)";
+    }
+    return null;
+}
+/** Field count per CSV record (quote-aware, handles newlines inside quotes). */
+function csvFieldCounts(raw, maxRecords) {
+    const counts = [];
+    let fields = 1;
+    let chars = 0; // non-separator chars seen in the current record
+    let inQ = false;
+    for (let i = 0; i < raw.length && counts.length < maxRecords; i++) {
+        const ch = raw[i];
+        if (inQ) {
+            if (ch === '"') {
+                if (raw[i + 1] === '"')
+                    i++;
+                else
+                    inQ = false;
+            }
+            chars++;
+        }
+        else if (ch === '"') {
+            inQ = true;
+            chars++;
+        }
+        else if (ch === ",") {
+            fields++;
+            chars++;
+        }
+        else if (ch === "\n" || ch === "\r") {
+            if (ch === "\r" && raw[i + 1] === "\n")
+                i++;
+            if (chars > 0)
+                counts.push(fields); // skip blank lines
+            fields = 1;
+            chars = 0;
+        }
+        else {
+            chars++;
+        }
+    }
+    if (chars > 0 && counts.length < maxRecords)
+        counts.push(fields);
+    return counts;
+}
 // ---------- html ----------
 const ENTITIES = {
     amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ",

package/dist/webtools.js

@@ -1,9 +1,16 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.webSearch = webSearch;
+exports._resetEngineCooldowns = _resetEngineCooldowns;
+exports.tinyfishSearch = tinyfishSearch;
+exports.ddgSearch = ddgSearch;
+exports.bingSearch = bingSearch;
+exports.arxivSearch = arxivSearch;
+exports.crossrefSearch = crossrefSearch;
 exports.parseBingHtml = parseBingHtml;
 exports.fetchUrl = fetchUrl;
 const crawltools_1 = require("./crawltools");
+const pdftext_1 = require("./pdftext");
 const searchcore_1 = require("./searchcore");
 const util_1 = require("./util");
 const UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36 agentswarm/0.1";
@@ -20,7 +27,7 @@ const DEEP_PASSAGES = 3;
  * and re-ranks by content quality. Ranking/passage algorithms live in
  * searchcore.ts.
  */
-async function webSearch(cfg, query, count, signal, deep = false, warn) {
+async function webSearch(cfg, query, count, signal, deep = false, warn, _retried = false) {
     // Deep searches widen recall by issuing complementary phrasings; the fast
     // path stays a single query so an agent's tool loop isn't slowed.
     const queries = deep ? (0, searchcore_1.expandQueries)(query) : [query];
@@ -37,18 +44,30 @@ async function webSearch(cfg, query, count, signal, deep = false, warn) {
             }
         }
     }
+    // Scholarly questions also sweep the keyless academic APIs (deep mode only).
+    if (deep && (0, searchcore_1.looksAcademic)(query)) {
+        engineCalls.push(arxivSearch(query, perEngine, signal), crossrefSearch(query, perEngine, signal));
+    }
     const settled = await Promise.allSettled(engineCalls);
     const candidates = settled.flatMap((s) => (s.status === "fulfilled" ? s.value : []));
     if (!candidates.length) {
         const firstErr = settled.find((s) => s.status === "rejected");
+        if (firstErr && settled.every((s) => s.status === "rejected"))
+            throw firstErr.reason;
+        // Engines answered but nothing parsed/matched: one retry with a
+        // simplified phrasing before giving up.
+        if (!_retried) {
+            const alt = (0, searchcore_1.reformulate)(query);
+            if (alt) {
+                warn?.(`no results for "${query}" — retrying as "${alt}"`);
+                return webSearch(cfg, alt, count, signal, deep, warn, true);
+            }
+        }
         if (firstErr)
             throw firstErr.reason;
         return [];
     }
     const failures = settled.filter((s) => s.status === "rejected").length;
-    if (failures && failures === settled.length) {
-        throw (settled.find((s) => s.status === "rejected")).reason;
-    }
     if (failures) {
         warn?.(`${failures}/${settled.length} search engine calls failed; results come from the rest`);
     }
@@ -111,16 +130,23 @@ async function fetchReadable(url, signal) {
         }
     }
     const res = await fetch(url, {
-        headers: { "user-agent": UA, accept: "text/html,text/*;q=0.9,*/*;q=0.5" },
+        headers: { "user-agent": UA, accept: "text/html,application/pdf,text/*;q=0.9,*/*;q=0.5" },
         signal: mergeSignal(20_000, signal),
         redirect: "follow",
     });
     if (!res.ok)
         throw new Error(`HTTP ${res.status}`);
     const ctype = res.headers.get("content-type") || "";
+    if (/application\/pdf/i.test(ctype)) {
+        const buf = Buffer.from(await res.arrayBuffer());
+        const pdf = buf.length <= 20_000_000 ? (0, pdftext_1.extractPdfText)(buf) : null;
+        if (!pdf)
+            throw new Error("pdf with no extractable text");
+        return clip(pdf.text);
+    }
     if (!/text\/|html|xml|json/i.test(ctype))
         throw new Error(`not textual: ${ctype}`);
-    const body = await res.text();
+    const body = decodeBody(Buffer.from(await res.arrayBuffer()), ctype);
     const text = /html/i.test(ctype) ? (0, util_1.htmlToText)(body) : body;
     return clip(text);
 }
@@ -135,6 +161,36 @@ function mergeSignal(timeoutMs, signal) {
     return typeof AbortSignal.any === "function" ? AbortSignal.any([t, signal]) : signal;
 }
 // ---------------------------------------------------------------- engines
+/**
+ * Per-engine rate-limit cooldowns: an engine that answers 429/403/503 sits
+ * out (60s, or the server's retry-after up to 120s) instead of getting
+ * hammered into a long block mid-research. A tiny retry-after (≤5s) is
+ * honored once in-call.
+ */
+const engineCooldown = new Map();
+/** Test hook. */
+function _resetEngineCooldowns() {
+    engineCooldown.clear();
+}
+async function engineFetch(engine, url, init, signal) {
+    const until = engineCooldown.get(engine) ?? 0;
+    if (until > Date.now()) {
+        throw new Error(`${engine} is cooling down after a rate limit (${Math.ceil((until - Date.now()) / 1000)}s left)`);
+    }
+    for (let attempt = 0;; attempt++) {
+        const res = await fetch(url, { ...init, signal: mergeSignal(20_000, signal) });
+        if (![429, 403, 503].includes(res.status))
+            return res;
+        const retryAfter = Number(res.headers.get("retry-after"));
+        if (attempt === 0 && Number.isFinite(retryAfter) && retryAfter > 0 && retryAfter <= 5) {
+            await new Promise((r) => setTimeout(r, retryAfter * 1000));
+            continue;
+        }
+        const ms = Number.isFinite(retryAfter) && retryAfter > 0 ? Math.min(retryAfter, 120) * 1000 : 60_000;
+        engineCooldown.set(engine, Date.now() + ms);
+        throw new Error(`${engine} rate-limited (HTTP ${res.status}); cooling down ${Math.round(ms / 1000)}s`);
+    }
+}
 async function tinyfishSearch(cfg, query, count, signal) {
     const url = `https://api.search.tinyfish.ai?query=${encodeURIComponent(query)}`;
     const res = await fetch(url, {
@@ -173,10 +229,9 @@ async function ddgSearch(query, count, signal) {
     let reachedAny = false;
     for (const ep of DDG_ENDPOINTS) {
         try {
-            const res = await fetch(ep.url + encodeURIComponent(query), {
+            const res = await engineFetch("duckduckgo", ep.url + encodeURIComponent(query), {
                 headers: { "user-agent": UA },
-                signal: mergeSignal(20_000, signal),
-            });
+            }, signal);
             if (!res.ok)
                 throw new Error(`search failed: HTTP ${res.status}`);
             reachedAny = true;
@@ -221,14 +276,56 @@ function parseDdgHtml(html, count, linkRe) {
 }
 /** Bing's HTML results page: each hit is an <li class="b_algo"> with an <h2><a> link. */
 async function bingSearch(query, count, signal) {
-    const res = await fetch(`https://www.bing.com/search?q=${encodeURIComponent(query)}`, {
+    const res = await engineFetch("bing", `https://www.bing.com/search?q=${encodeURIComponent(query)}`, {
         headers: { "user-agent": UA, "accept-language": "en-US,en;q=0.9" },
-        signal: mergeSignal(20_000, signal),
-    });
+    }, signal);
     if (!res.ok)
         throw new Error(`bing search ${res.status}`);
     return parseBingHtml(await res.text(), count);
 }
+// ---------------------------------------------------------------- academic engines (keyless)
+/** arXiv's Atom API — preprints with abstracts, no key needed. */
+async function arxivSearch(query, count, signal) {
+    const url = `https://export.arxiv.org/api/query?search_query=all:${encodeURIComponent(query)}&max_results=${Math.min(count, 15)}`;
+    const res = await engineFetch("arxiv", url, { headers: { "user-agent": UA } }, signal);
+    if (!res.ok)
+        throw new Error(`arxiv search ${res.status}`);
+    const xml = await res.text();
+    const out = [];
+    for (const entry of xml.split(/<entry>/).slice(1)) {
+        if (out.length >= count)
+            break;
+        const title = strip((/<title>([\s\S]*?)<\/title>/.exec(entry) || [])[1] || "");
+        const id = ((/<id>([\s\S]*?)<\/id>/.exec(entry) || [])[1] || "").trim();
+        const summary = strip((/<summary>([\s\S]*?)<\/summary>/.exec(entry) || [])[1] || "");
+        const published = (/<published>(\d{4}-\d{2}-\d{2})/.exec(entry) || [])[1];
+        if (!id || !title || !/^https?:\/\//.test(id))
+            continue;
+        out.push({ title, url: id, snippet: summary.slice(0, 300), rank: out.length + 1, engine: "arxiv", date: published });
+    }
+    return out;
+}
+/** Crossref's works API — journal/conference metadata with DOIs, no key needed. */
+async function crossrefSearch(query, count, signal) {
+    const url = `https://api.crossref.org/works?query=${encodeURIComponent(query)}&rows=${Math.min(count, 15)}&select=title,DOI,abstract,issued,container-title`;
+    const res = await engineFetch("crossref", url, { headers: { "user-agent": UA } }, signal);
+    if (!res.ok)
+        throw new Error(`crossref search ${res.status}`);
+    const data = await res.json();
+    const out = [];
+    for (const it of data?.message?.items ?? []) {
+        if (out.length >= count)
+            break;
+        const title = strip(String(Array.isArray(it.title) ? it.title[0] ?? "" : it.title ?? ""));
+        if (!title || !it.DOI)
+            continue;
+        const date = Array.isArray(it.issued?.["date-parts"]?.[0]) ? it.issued["date-parts"][0].join("-") : undefined;
+        const venue = Array.isArray(it["container-title"]) ? it["container-title"][0] : "";
+        const snippet = (strip(String(it.abstract ?? "")) || venue || "").slice(0, 300);
+        out.push({ title, url: `https://doi.org/${it.DOI}`, snippet, rank: out.length + 1, engine: "crossref", date });
+    }
+    return out;
+}
 function parseBingHtml(html, count) {
     const hits = [];
     const blocks = html.split(/<li class="b_algo[^"]*"/i).slice(1);
@@ -302,18 +399,51 @@ async function fetchUrl(cfg, url, raw, maxChars, signal) {
         }
     }
     const res = await fetch(url, {
-        headers: { "user-agent": UA, accept: "text/html,application/json,text/*;q=0.9,*/*;q=0.5" },
+        headers: { "user-agent": UA, accept: "text/html,application/json,application/pdf,text/*;q=0.9,*/*;q=0.5" },
         signal: signal ?? AbortSignal.timeout(25000),
         redirect: "follow",
     });
     const ctype = res.headers.get("content-type") || "";
-    const body = await res.text();
     if (!res.ok) {
-        return `HTTP ${res.status} ${res.statusText}\n${(0, util_1.truncateMiddle)(body, 2000, "chars")}`;
+        // An error page is not content: returning it as a successful result lets
+        // "HTTP 403 ... subscribe to continue" become a "fact" in someone's report.
+        const body = await res.text().catch(() => "");
+        throw new Error(`HTTP ${res.status} ${res.statusText} — page is not usable as a source (paywall/login/blocked?). ` +
+            `Try web_search for an alternative source.${body ? ` Server said: ${(0, util_1.oneLine)((0, util_1.htmlToText)(body), 200)}` : ""}`);
+    }
+    const buf = Buffer.from(await res.arrayBuffer());
+    if (/application\/pdf/i.test(ctype) || buf.subarray(0, 5).toString("latin1") === "%PDF-") {
+        if (buf.length > 20_000_000)
+            throw new Error(`PDF is ${Math.round(buf.length / 1e6)}MB — too large to extract`);
+        const pdf = (0, pdftext_1.extractPdfText)(buf);
+        if (!pdf) {
+            throw new Error("PDF contains no extractable text (likely scanned or encrypted) — find an HTML version of this source.");
+        }
+        return (0, util_1.truncateMiddle)(`[PDF, ${pdf.pages} page${pdf.pages > 1 ? "s" : ""}]\n${pdf.text}`, maxChars, "chars");
     }
+    const body = decodeBody(buf, ctype);
     const text = !raw && /html/i.test(ctype) ? (0, util_1.htmlToText)(body) : body;
+    if (!raw && /html/i.test(ctype)) {
+        const trimmed = text.trim();
+        if (trimmed.length < 400 && /subscrib|sign.?in|log.?in|enable javascript|access denied|are you a (human|robot)|captcha/i.test(trimmed)) {
+            return `WARNING: this page returned only a paywall/anti-bot shell — the text below is probably not the real content. Try web_search for an alternative source.\n\n${trimmed}`;
+        }
+    }
     return (0, util_1.truncateMiddle)(text, maxChars, "chars");
 }
+/** Decode a response body honoring its content-type charset (UTF-8 fallback). */
+function decodeBody(buf, ctype) {
+    const charset = /charset=([\w-]+)/i.exec(ctype)?.[1]?.toLowerCase();
+    if (charset && charset !== "utf-8" && charset !== "utf8") {
+        try {
+            return new TextDecoder(charset).decode(buf);
+        }
+        catch {
+            /* unknown label — fall through to utf-8 */
+        }
+    }
+    return buf.toString("utf8");
+}
 async function tinyfishFetch(cfg, url, signal) {
     const res = await fetch("https://api.fetch.tinyfish.ai", {
         method: "POST",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@robzilla1738/agentswarm",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "publishConfig": {
     "access": "public"
   },