@robzilla1738/agentswarm 0.3.0 → 0.6.0

package/dist/hub.js

@@ -34,6 +34,7 @@ var __importStar = (this && this.__importStar) || (function () {
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.startHub = startHub;
+exports.publicConfig = publicConfig;
 const fs = __importStar(require("fs"));
 const http = __importStar(require("http"));
 const os = __importStar(require("os"));
@@ -41,6 +42,8 @@ const path = __importStar(require("path"));
 const url_1 = require("url");
 const config_1 = require("./config");
 const control_1 = require("./control");
+const crawltools_1 = require("./crawltools");
+const webtools_1 = require("./webtools");
 const deepseek_1 = require("./deepseek");
 const providers_1 = require("./providers");
 const journal_1 = require("./journal");
@@ -81,9 +84,16 @@ function startHub(opts) {
 async function handle(req, res, opts) {
     const url = new url_1.URL(req.url || "/", `http://localhost:${opts.port}`);
     const p = url.pathname;
-    res.setHeader("access-control-allow-origin", "*");
-    res.setHeader("access-control-allow-methods", "GET, POST, DELETE, OPTIONS");
-    res.setHeader("access-control-allow-headers", "content-type");
+    // Localhost-only CORS. The hub launches runs and reads reports with the
+    // operator's keys — a random website's JS must never get a readable
+    // response. The dev UI on another localhost port is the one legitimate
+    // cross-origin client; everyone else gets no CORS headers at all.
+    const origin = String(req.headers.origin || "");
+    if (/^https?:\/\/(localhost|127\.0\.0\.1|\[::1\])(:\d+)?$/.test(origin)) {
+        res.setHeader("access-control-allow-origin", origin);
+        res.setHeader("access-control-allow-methods", "GET, POST, DELETE, OPTIONS");
+        res.setHeader("access-control-allow-headers", "content-type");
+    }
     if (req.method === "OPTIONS") {
         res.writeHead(204);
         res.end();
@@ -157,6 +167,51 @@ async function api(req, res, url, opts) {
         const r = await (0, sandbox_1.testSandbox)(cfg, kind);
         return sendJson(res, 200, { kind, ...r });
     }
+    // Settings diagnostics: prove the search engines / crawl backend actually
+    // work with the saved keys before a mission depends on them.
+    if (p === "/api/search/test" && method === "POST") {
+        const q = "open source vector database";
+        const probe = async (engine, fn) => {
+            try {
+                const hits = await fn();
+                return { engine, ok: hits.length > 0, detail: `${hits.length} result(s)` };
+            }
+            catch (e) {
+                return { engine, ok: false, detail: (0, util_1.errMsg)(e) };
+            }
+        };
+        const checks = [probe("duckduckgo", () => (0, webtools_1.ddgSearch)(q, 3)), probe("bing", () => (0, webtools_1.bingSearch)(q, 3))];
+        if (cfg.tinyfishApiKey)
+            checks.push(probe("tinyfish", () => (0, webtools_1.tinyfishSearch)(cfg, q, 3)));
+        const engines = await Promise.all(checks);
+        return sendJson(res, 200, { ok: engines.some((e) => e.ok), engines });
+    }
+    if (p === "/api/crawl/test" && method === "POST") {
+        const backend = (0, crawltools_1.resolveCrawlBackend)(cfg);
+        if (!backend) {
+            return sendJson(res, 200, { ok: false, backend: null, detail: "no crawl backend configured — add a key first" });
+        }
+        try {
+            if ((0, crawltools_1.hasScrapeBackend)(cfg)) {
+                const text = await (0, crawltools_1.scrapeUrl)(cfg, "https://example.com/");
+                return sendJson(res, 200, {
+                    ok: Boolean(text && text.length > 50),
+                    backend,
+                    detail: text ? `scraped ${text.length} chars` : "empty scrape result",
+                });
+            }
+            // deepcrawl has no single-page scrape — smoke a 1-page crawl instead.
+            const out = await (0, crawltools_1.crawlSite)(cfg, { url: "https://example.com/", maxPages: 1 });
+            return sendJson(res, 200, {
+                ok: out.pages.length > 0,
+                backend,
+                detail: out.pages.length ? `crawled ${out.pages.length} page(s)` : out.warnings.join("; ") || "no pages",
+            });
+        }
+        catch (e) {
+            return sendJson(res, 200, { ok: false, backend, detail: (0, util_1.errMsg)(e) });
+        }
+    }
     if (p === "/api/models" && method === "GET") {
         try {
             const models = await (0, deepseek_1.listModels)(cfg);
@@ -255,7 +310,7 @@ async function api(req, res, url, opts) {
             return sendJson(res, 200, { events, live: (0, run_1.isRunLive)(id) });
         }
         if (sub === "/stream" && method === "GET") {
-            return streamEvents(res, id);
+            return streamEvents(res, id, url.searchParams.get("quiet") === "1");
         }
         if (sub === "/note" && method === "POST") {
             const body = await readBody(req);
@@ -295,6 +350,14 @@ async function api(req, res, url, opts) {
             res.end(fs.readFileSync(file));
             return;
         }
+        if (sub === "/plan" && method === "GET") {
+            const file = path.join((0, config_1.runDir)(id), "artifacts", "mission-plan.md");
+            if (!fs.existsSync(file))
+                return sendJson(res, 404, { error: "no plan yet" });
+            res.writeHead(200, { "content-type": "text/markdown; charset=utf-8" });
+            res.end(fs.readFileSync(file));
+            return;
+        }
         if (sub === "/artifacts" && method === "GET") {
             return sendJson(res, 200, { artifacts: listArtifactFiles(id) });
         }
@@ -316,7 +379,7 @@ async function api(req, res, url, opts) {
     }
     sendJson(res, 404, { error: "not found" });
 }
-function streamEvents(res, id) {
+function streamEvents(res, id, quiet = false) {
     res.writeHead(200, {
         "content-type": "text/event-stream",
         "cache-control": "no-cache, no-transform",
@@ -335,6 +398,9 @@ function streamEvents(res, id) {
             return;
         }
         for (const ev of evs) {
+            // quiet mode: skip streaming chatter for clients rendering many agents.
+            if (quiet && ev.type === "agent.delta")
+                continue;
             res.write(`data: ${JSON.stringify(ev)}\n\n`);
         }
     };
@@ -392,7 +458,15 @@ function publicConfig(cfg) {
         tinyfishKeySet: Boolean(cfg.tinyfishApiKey),
         tinyfishKeyMasked: (0, config_1.maskKey)(cfg.tinyfishApiKey),
         searchBackend: cfg.searchBackend,
-        searchkitCmd: cfg.searchkitCmd,
+        crawlBackend: cfg.crawlBackend,
+        crawlResolved: (0, crawltools_1.resolveCrawlBackend)(cfg),
+        firecrawlKeySet: Boolean(cfg.firecrawlApiKey),
+        firecrawlKeyMasked: (0, config_1.maskKey)(cfg.firecrawlApiKey),
+        contextdevKeySet: Boolean(cfg.contextdevApiKey),
+        contextdevKeyMasked: (0, config_1.maskKey)(cfg.contextdevApiKey),
+        deepcrawlKeySet: Boolean(cfg.deepcrawlApiKey),
+        deepcrawlKeyMasked: (0, config_1.maskKey)(cfg.deepcrawlApiKey),
+        deepcrawlBaseUrl: cfg.deepcrawlBaseUrl,
         sandboxRuntime: cfg.sandboxRuntime,
         sandboxResolved: (0, sandbox_1.resolveSandboxKind)(cfg),
         sandboxImage: cfg.sandboxImage,
@@ -416,6 +490,9 @@ function publicConfig(cfg) {
         reasoningEffort: cfg.reasoningEffort,
         safeMode: cfg.safeMode,
         contextTokenLimit: cfg.contextTokenLimit,
+        contextWindows: cfg.contextWindows,
+        cheapModel: cfg.cheapModel,
+        strongModel: cfg.strongModel,
         knownModels,
         pricing: cfg.pricing,
     };
@@ -469,6 +546,8 @@ function snapshot(state, id) {
         operatorNotes: state.operatorNotes,
         usageByModel: Object.fromEntries(state.usageByModel),
         cost: state.cost,
+        budgetSeries: state.budgetSeries,
+        planExcerpt: state.planExcerpt,
         finalSummary: state.finalSummary,
         finalReportPath: state.finalReportPath,
         live: (0, run_1.isRunLive)(id),

package/dist/journal.js

@@ -33,7 +33,7 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Journal = void 0;
+exports.Journal = exports.TeamJournal = void 0;
 exports.eventsFile = eventsFile;
 exports.readEvents = readEvents;
 exports.lastSeq = lastSeq;
@@ -41,14 +41,36 @@ exports.readNewEvents = readNewEvents;
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 /**
- * Append-only event journal. events.jsonl is the single source of truth for a
- * run: the executor writes it, the terminal renderer and the hub (web UI) read
- * and tail it. Tolerant of a torn final line after a crash.
+ * A child swarm's view of its parent's journal: same file, same sequence,
+ * every event stamped with the owning team's task id so the reducer can
+ * partition team activity away from the root run.
  */
+class TeamJournal {
+    inner;
+    teamId;
+    constructor(inner, teamId) {
+        this.inner = inner;
+        this.teamId = teamId;
+    }
+    append(type, payload = {}) {
+        return this.inner.append(type, { teamId: this.teamId, ...payload });
+    }
+    flush() {
+        return this.inner.flush();
+    }
+    get degraded() {
+        return this.inner.degraded;
+    }
+}
+exports.TeamJournal = TeamJournal;
 class Journal {
     file;
     seq;
     chain = Promise.resolve();
+    buf = "";
+    failures = 0;
+    /** Set after repeated append failures: the source of truth is no longer being persisted. */
+    degraded = false;
     onEvent;
     constructor(runDirPath, startSeq) {
         this.file = path.join(runDirPath, "events.jsonl");
@@ -56,12 +78,8 @@ class Journal {
     }
     append(type, payload = {}) {
         const ev = { seq: this.seq++, t: Date.now(), type, ...payload };
-        const line = JSON.stringify(ev) + "\n";
-        this.chain = this.chain
-            .then(() => fs.promises.appendFile(this.file, line, "utf8"))
-            .catch(() => {
-            /* never break the run on journal IO; next append retries the chain */
-        });
+        this.buf += JSON.stringify(ev) + "\n";
+        this.chain = this.chain.then(() => this.drain());
         try {
             this.onEvent?.(ev);
         }
@@ -70,8 +88,40 @@ class Journal {
         }
         return ev;
     }
+    async drain() {
+        if (!this.buf)
+            return;
+        const chunk = this.buf;
+        this.buf = "";
+        try {
+            await fs.promises.appendFile(this.file, chunk, "utf8");
+            this.failures = 0;
+        }
+        catch (e) {
+            // Keep the unwritten events buffered so the next append/flush retries
+            // them in order; after repeated failures, stop pretending it's fine.
+            this.buf = chunk + this.buf;
+            this.failures++;
+            if (this.failures >= 5 && !this.degraded) {
+                this.degraded = true;
+                process.stderr.write(`agentswarm: journal writes are failing (${String(e)}); run state is no longer durable\n`);
+            }
+        }
+    }
     flush() {
-        return this.chain;
+        return this.chain.then(() => this.drain());
+    }
+    /** Last-gasp synchronous flush for signal handlers and exit paths. */
+    flushSync() {
+        if (!this.buf)
+            return;
+        try {
+            fs.appendFileSync(this.file, this.buf, "utf8");
+            this.buf = "";
+        }
+        catch {
+            /* nothing left to do */
+        }
     }
 }
 exports.Journal = Journal;

package/dist/memory.js

@@ -0,0 +1,84 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.memoryFile = memoryFile;
+exports.loadMemory = loadMemory;
+exports.appendMemory = appendMemory;
+exports.memoryBlock = memoryBlock;
+const crypto = __importStar(require("crypto"));
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const config_1 = require("./config");
+const util_1 = require("./util");
+const MAX_ENTRIES = 20;
+function memoryFile(cwd) {
+    const hash = crypto.createHash("sha1").update(path.resolve(cwd)).digest("hex").slice(0, 12);
+    return path.join((0, config_1.home)(), "memory", `${hash}.json`);
+}
+function loadMemory(cwd) {
+    try {
+        const raw = JSON.parse(fs.readFileSync(memoryFile(cwd), "utf8"));
+        return Array.isArray(raw?.entries) ? raw.entries : [];
+    }
+    catch {
+        return [];
+    }
+}
+function appendMemory(cwd, entry) {
+    try {
+        // Same-run entries replace (interim → final); writeJson is temp+rename so
+        // a crash mid-write never loses the prior history.
+        const prior = loadMemory(cwd).filter((e) => !(entry.runId && e.runId === entry.runId));
+        const entries = [...prior, entry].slice(-MAX_ENTRIES);
+        (0, util_1.writeJson)(memoryFile(cwd), { cwd: path.resolve(cwd), entries });
+    }
+    catch {
+        /* memory is best-effort */
+    }
+}
+/** Prompt block for the conductor, or "" when there's no history. */
+function memoryBlock(cwd) {
+    const entries = loadMemory(cwd);
+    if (!entries.length)
+        return "";
+    const lines = entries.slice(-8).map((e) => {
+        const when = new Date(e.finishedAt).toISOString().slice(0, 10);
+        const decisions = e.keyDecisions.length
+            ? ` Decisions: ${e.keyDecisions.map((d) => (0, util_1.oneLine)(d, 100)).join("; ")}`
+            : "";
+        return `- [${when}, ${e.status}] "${(0, util_1.oneLine)(e.mission, 100)}" — ${(0, util_1.oneLine)(e.summary, 200)}${decisions}`;
+    });
+    return (0, util_1.clip)(`PRIOR RUNS IN THIS WORKSPACE (build on them; don't redo settled decisions without reason):\n${lines.join("\n")}`, 4000);
+}

package/dist/pdftext.js

@@ -0,0 +1,211 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extractPdfText = extractPdfText;
+const zlib = __importStar(require("zlib"));
+/**
+ * Minimal zero-dependency PDF text extraction: inflate FlateDecode content
+ * streams (Node's built-in zlib) and interpret the text-showing operators
+ * (Tj / TJ / ' / "). Good enough for most digitally-produced text PDFs;
+ * returns null for scanned, encrypted, or exotic-encoding documents so the
+ * caller can tell the agent to find an HTML source instead.
+ */
+function extractPdfText(buf) {
+    if (buf.subarray(0, 5).toString("latin1") !== "%PDF-")
+        return null;
+    // latin1 preserves bytes 1:1, so stream offsets in the string match the buffer.
+    const raw = buf.toString("latin1");
+    const pages = (raw.match(/\/Type\s*\/Pages?\b/g) || []).filter((m) => !/Pages/.test(m)).length || 1;
+    let text = "";
+    const streamRe = /<<([\s\S]{0,2000}?)>>\s*stream\r?\n/g;
+    let m;
+    while ((m = streamRe.exec(raw))) {
+        const dict = m[1];
+        const start = m.index + m[0].length;
+        const end = raw.indexOf("endstream", start);
+        if (end < 0)
+            continue;
+        streamRe.lastIndex = end;
+        // Only plain or Flate-compressed streams are supported.
+        if (/\/Filter/.test(dict) && !/FlateDecode/.test(dict))
+            continue;
+        let len = end;
+        while (len > start && (raw[len - 1] === "\n" || raw[len - 1] === "\r"))
+            len--;
+        let data = buf.subarray(start, len);
+        if (/FlateDecode/.test(dict)) {
+            try {
+                data = zlib.inflateSync(data);
+            }
+            catch {
+                continue;
+            }
+        }
+        const content = data.toString("latin1");
+        if (!/\bBT\b/.test(content))
+            continue; // not a text content stream
+        const extracted = extractFromContent(content);
+        if (extracted.trim())
+            text += extracted + "\n";
+    }
+    const cleaned = text
+        .replace(/[^\S\n]+/g, " ")
+        .replace(/ ?\n ?/g, "\n")
+        .replace(/\n{3,}/g, "\n\n")
+        .trim();
+    // CID/Type0 fonts yield glyph-index garbage; require a body of real characters.
+    const printable = cleaned.replace(/[^\x20-\x7E\n -￿]/g, "");
+    if (printable.replace(/\s/g, "").length < 40)
+        return null;
+    return { text: printable, pages };
+}
+/** Walk a content stream, collecting strings shown by Tj/TJ/'/" with newline heuristics. */
+function extractFromContent(src) {
+    let out = "";
+    let pending = [];
+    const n = src.length;
+    let i = 0;
+    while (i < n) {
+        const ch = src[i];
+        if (ch === "(") {
+            const [s, next] = parseLiteralString(src, i);
+            pending.push(s);
+            i = next;
+        }
+        else if (ch === "<" && src[i + 1] !== "<") {
+            const close = src.indexOf(">", i + 1);
+            if (close < 0)
+                break;
+            pending.push(decodeHexString(src.slice(i + 1, close)));
+            i = close + 1;
+        }
+        else if (ch === "%") {
+            // comment to end of line
+            while (i < n && src[i] !== "\n" && src[i] !== "\r")
+                i++;
+        }
+        else if (/[A-Za-z'"*]/.test(ch)) {
+            let j = i;
+            while (j < n && /[A-Za-z'"*]/.test(src[j]))
+                j++;
+            const op = src.slice(i, j);
+            if (op === "Tj" || op === "TJ") {
+                out += pending.join("");
+            }
+            else if (op === "'" || op === '"') {
+                out += "\n" + pending.join("");
+            }
+            else if (op === "Td" || op === "TD" || op === "T*" || op === "Tm" || op === "ET") {
+                if (pending.length)
+                    out += pending.join("");
+                if (!out.endsWith("\n"))
+                    out += "\n";
+            }
+            pending = [];
+            i = j;
+        }
+        else if (ch === "-" || (ch >= "0" && ch <= "9") || ch === ".") {
+            let j = i + 1;
+            while (j < n && /[0-9.]/.test(src[j]))
+                j++;
+            // Large negative kerning inside a TJ array is a word gap.
+            const num = parseFloat(src.slice(i, j));
+            if (num <= -180 && pending.length && !pending[pending.length - 1].endsWith(" "))
+                pending.push(" ");
+            i = j;
+        }
+        else {
+            i++;
+        }
+    }
+    return out;
+}
+/** PDF literal string: balanced parens, backslash escapes, octal codes. */
+function parseLiteralString(src, start) {
+    let out = "";
+    let depth = 0;
+    let i = start;
+    for (; i < src.length; i++) {
+        const ch = src[i];
+        if (ch === "\\") {
+            const next = src[i + 1];
+            if (next >= "0" && next <= "7") {
+                let oct = "";
+                for (let k = 1; k <= 3 && src[i + k] >= "0" && src[i + k] <= "7"; k++)
+                    oct += src[i + k];
+                out += String.fromCharCode(parseInt(oct, 8));
+                i += oct.length;
+            }
+            else {
+                const map = { n: "\n", r: "\r", t: "\t", b: "\b", f: "\f", "(": "(", ")": ")", "\\": "\\" };
+                out += map[next] ?? next ?? "";
+                i++;
+            }
+        }
+        else if (ch === "(") {
+            depth++;
+            if (depth > 1)
+                out += ch;
+        }
+        else if (ch === ")") {
+            depth--;
+            if (depth === 0) {
+                i++;
+                break;
+            }
+            out += ch;
+        }
+        else {
+            out += ch;
+        }
+    }
+    return [out, i];
+}
+/** PDF hex string: byte pairs; a UTF-16BE BOM switches to two-byte chars. */
+function decodeHexString(hex) {
+    const clean = hex.replace(/[^0-9a-fA-F]/g, "");
+    const bytes = [];
+    for (let i = 0; i + 1 < clean.length; i += 2)
+        bytes.push(parseInt(clean.slice(i, i + 2), 16));
+    if (clean.length % 2)
+        bytes.push(parseInt(clean[clean.length - 1] + "0", 16));
+    if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff) {
+        let s = "";
+        for (let i = 2; i + 1 < bytes.length; i += 2)
+            s += String.fromCharCode((bytes[i] << 8) | bytes[i + 1]);
+        return s;
+    }
+    return bytes.map((b) => String.fromCharCode(b)).join("");
+}