@robzilla1738/agentswarm 0.3.0 → 0.6.0

package/dist/crawltools.js

@@ -0,0 +1,247 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.resolveCrawlBackend = resolveCrawlBackend;
+exports.hasScrapeBackend = hasScrapeBackend;
+exports.crawlSite = crawlSite;
+exports.scrapeUrl = scrapeUrl;
+exports.slugForUrl = slugForUrl;
+const util_1 = require("./util");
+const PER_PAGE_CHAR_CAP = 200_000;
+const TOTAL_CHAR_BUDGET = 8_000_000;
+const CRAWL_DEADLINE_MS = 120_000;
+/** auto = first configured: Firecrawl → context.dev → deepcrawl. "off" or nothing configured → null. */
+function resolveCrawlBackend(cfg) {
+    if (cfg.crawlBackend === "off")
+        return null;
+    const configured = {
+        firecrawl: Boolean(cfg.firecrawlApiKey),
+        contextdev: Boolean(cfg.contextdevApiKey),
+        deepcrawl: Boolean(cfg.deepcrawlApiKey && cfg.deepcrawlBaseUrl),
+    };
+    if (cfg.crawlBackend !== "auto")
+        return configured[cfg.crawlBackend] ? cfg.crawlBackend : null;
+    for (const id of ["firecrawl", "contextdev", "deepcrawl"]) {
+        if (configured[id])
+            return id;
+    }
+    return null;
+}
+/** Backends usable for single-page scrape in fetch_url (the custom deepcrawl contract has no scrape endpoint). */
+function hasScrapeBackend(cfg) {
+    const b = resolveCrawlBackend(cfg);
+    return b === "firecrawl" || b === "contextdev";
+}
+async function crawlSite(cfg, opts) {
+    const backend = resolveCrawlBackend(cfg);
+    if (!backend)
+        throw new Error("no crawl backend configured — add a Firecrawl/context.dev/deepcrawl key in Settings");
+    const warnings = [];
+    let pages;
+    if (backend === "firecrawl")
+        pages = await firecrawlCrawl(cfg, opts, warnings);
+    else if (backend === "contextdev")
+        pages = await contextdevCrawl(cfg, opts);
+    else
+        pages = await deepcrawlCrawl(cfg, opts);
+    // Normalize: drop empty/binary pages, cap per-page and total size.
+    const clean = [];
+    let skipped = 0;
+    let total = 0;
+    for (const p of pages) {
+        if (clean.length >= opts.maxPages)
+            break;
+        const md = (p.markdown || "").trim();
+        if (!md || md.includes("\u0000")) {
+            skipped++;
+            continue;
+        }
+        const body = (0, util_1.truncateMiddle)(md, PER_PAGE_CHAR_CAP, "chars");
+        if (total + body.length > TOTAL_CHAR_BUDGET) {
+            warnings.push(`stopped at ${clean.length} pages: total content budget reached`);
+            break;
+        }
+        total += body.length;
+        clean.push({ url: p.url, title: p.title, markdown: body });
+    }
+    if (skipped)
+        warnings.push(`${skipped} empty page${skipped > 1 ? "s" : ""} skipped`);
+    return { backend, pages: clean, warnings };
+}
+/** Single-page scrape via the configured backend. Throws on failure — callers fall through to their own fetch path. */
+async function scrapeUrl(cfg, url, signal) {
+    const backend = resolveCrawlBackend(cfg);
+    if (backend === "firecrawl") {
+        const data = await callJson("firecrawl", "https://api.firecrawl.dev/v1/scrape", cfg.firecrawlApiKey, { url, formats: ["markdown"] }, 30_000, signal);
+        const md = String(data?.data?.markdown ?? "");
+        if (!md.trim())
+            throw new Error("firecrawl: empty scrape result");
+        const title = data?.data?.metadata?.title;
+        return title ? `# ${title}\n\n${md}` : md;
+    }
+    if (backend === "contextdev") {
+        const data = await callJson("context.dev", "https://api.context.dev/v1/web/scrape", cfg.contextdevApiKey, { url }, 30_000, signal);
+        const md = String(data?.markdown ?? data?.results?.[0]?.markdown ?? "");
+        if (!md.trim())
+            throw new Error("context.dev: empty scrape result");
+        const title = data?.metadata?.title ?? data?.results?.[0]?.metadata?.title;
+        return title ? `# ${title}\n\n${md}` : md;
+    }
+    throw new Error("no scrape-capable crawl backend configured");
+}
+/** "https://docs.foo.com/a/b?x=1" → filesystem-safe { host, slug } with no separators or traversal. */
+function slugForUrl(url) {
+    let u;
+    try {
+        u = new URL(url);
+    }
+    catch {
+        return { host: "site", slug: sanitize(url) || "page" };
+    }
+    const host = sanitize(u.hostname) || "site";
+    const slug = sanitize(u.pathname.replace(/\/+$/, "")) || "index";
+    return { host, slug };
+}
+function sanitize(s) {
+    return s
+        .toLowerCase()
+        .replace(/[^a-z0-9._-]+/g, "-")
+        .replace(/\.{2,}/g, ".")
+        .replace(/-{2,}/g, "-")
+        .replace(/^[-.]+|[-.]+$/g, "")
+        .slice(0, 120);
+}
+// ---------------------------------------------------------------- backends
+async function firecrawlCrawl(cfg, opts, warnings) {
+    const start = await callJson("firecrawl", "https://api.firecrawl.dev/v1/crawl", cfg.firecrawlApiKey, {
+        url: opts.url,
+        limit: opts.maxPages,
+        ...(opts.includePaths?.length ? { includePaths: opts.includePaths } : {}),
+        scrapeOptions: { formats: ["markdown"] },
+    }, 30_000, opts.signal);
+    const jobId = start?.id;
+    if (!jobId)
+        throw new Error(`firecrawl: crawl did not start (${start?.error || "no job id"})`);
+    const pollMs = opts.pollMs ?? 3000;
+    const deadline = Date.now() + CRAWL_DEADLINE_MS;
+    let last = null;
+    for (;;) {
+        opts.signal?.throwIfAborted();
+        last = await getJson("firecrawl", `https://api.firecrawl.dev/v1/crawl/${jobId}`, cfg.firecrawlApiKey, opts.signal);
+        if (last?.status === "completed")
+            break;
+        if (last?.status === "failed")
+            throw new Error(`firecrawl: crawl failed (${last?.error || "unknown error"})`);
+        if (Date.now() > deadline) {
+            const partial = mapFirecrawlPages(last);
+            if (!partial.length)
+                throw new Error("firecrawl: crawl still running after 120s with no pages yet — try fewer pages");
+            warnings.push(`crawl still running after 120s; returning ${partial.length} partial pages`);
+            return partial;
+        }
+        await sleep(pollMs, opts.signal);
+    }
+    // Completed: collect pages, following `next` pagination until maxPages.
+    const pages = mapFirecrawlPages(last);
+    let next = last?.next;
+    while (next && pages.length < opts.maxPages) {
+        const more = await getJson("firecrawl", String(next), cfg.firecrawlApiKey, opts.signal);
+        pages.push(...mapFirecrawlPages(more));
+        next = more?.next;
+    }
+    return pages;
+}
+function mapFirecrawlPages(res) {
+    const data = Array.isArray(res?.data) ? res.data : [];
+    return data.map((d) => ({
+        url: String(d?.metadata?.sourceURL ?? d?.metadata?.url ?? ""),
+        title: String(d?.metadata?.title ?? ""),
+        markdown: String(d?.markdown ?? ""),
+    }));
+}
+async function contextdevCrawl(cfg, opts) {
+    const data = await callJson("context.dev", "https://api.context.dev/v1/web/crawl", cfg.contextdevApiKey, {
+        url: opts.url,
+        max_pages: opts.maxPages,
+        ...(opts.includePaths?.length ? { include_paths: opts.includePaths } : {}),
+    }, CRAWL_DEADLINE_MS, opts.signal);
+    const results = Array.isArray(data?.results) ? data.results : [];
+    return results.map((r) => ({
+        url: String(r?.metadata?.url ?? r?.url ?? ""),
+        title: String(r?.metadata?.title ?? r?.title ?? ""),
+        markdown: String(r?.markdown ?? ""),
+    }));
+}
+async function deepcrawlCrawl(cfg, opts) {
+    const base = cfg.deepcrawlBaseUrl.replace(/\/+$/, "");
+    const data = await callJson("deepcrawl", `${base}/crawl`, cfg.deepcrawlApiKey, {
+        url: opts.url,
+        max_pages: opts.maxPages,
+        ...(opts.includePaths?.length ? { include_paths: opts.includePaths } : {}),
+    }, CRAWL_DEADLINE_MS, opts.signal);
+    // Accept either the context.dev-compatible shape or a flat pages[] list.
+    if (Array.isArray(data?.results)) {
+        return data.results.map((r) => ({
+            url: String(r?.metadata?.url ?? r?.url ?? ""),
+            title: String(r?.metadata?.title ?? r?.title ?? ""),
+            markdown: String(r?.markdown ?? ""),
+        }));
+    }
+    if (Array.isArray(data?.pages)) {
+        return data.pages.map((p) => ({
+            url: String(p?.url ?? ""),
+            title: String(p?.title ?? ""),
+            markdown: String(p?.markdown ?? p?.content ?? ""),
+        }));
+    }
+    throw new Error("deepcrawl: unrecognized response shape (expected results[] or pages[])");
+}
+// ---------------------------------------------------------------- plumbing
+function friendlyHttpError(service, status, body) {
+    if (status === 401 || status === 403) {
+        return new Error(`${service} API key invalid or unauthorized (HTTP ${status}) — check Settings → Crawl integrations`);
+    }
+    if (status === 402)
+        return new Error(`${service}: quota or credits exhausted (HTTP 402)`);
+    if (status === 429)
+        return new Error(`${service}: rate limited (HTTP 429) — retry later`);
+    return new Error(`${service}: HTTP ${status} ${(0, util_1.truncateMiddle)(body, 300, "chars")}`);
+}
+function mergeSignal(timeoutMs, signal) {
+    const t = AbortSignal.timeout(timeoutMs);
+    if (!signal)
+        return t;
+    return typeof AbortSignal.any === "function" ? AbortSignal.any([t, signal]) : signal;
+}
+async function callJson(service, url, key, body, timeoutMs, signal) {
+    const res = await fetch(url, {
+        method: "POST",
+        headers: { authorization: `Bearer ${key}`, "content-type": "application/json" },
+        body: JSON.stringify(body),
+        signal: mergeSignal(timeoutMs, signal),
+    });
+    if (!res.ok)
+        throw friendlyHttpError(service, res.status, await res.text().catch(() => ""));
+    return res.json();
+}
+async function getJson(service, url, key, signal) {
+    const res = await fetch(url, {
+        headers: { authorization: `Bearer ${key}` },
+        signal: mergeSignal(30_000, signal),
+    });
+    if (!res.ok)
+        throw friendlyHttpError(service, res.status, await res.text().catch(() => ""));
+    return res.json();
+}
+function sleep(ms, signal) {
+    return new Promise((resolve, reject) => {
+        const t = setTimeout(() => {
+            signal?.removeEventListener("abort", onAbort);
+            resolve();
+        }, ms);
+        const onAbort = () => {
+            clearTimeout(t);
+            reject(new Error("aborted"));
+        };
+        signal?.addEventListener("abort", onAbort, { once: true });
+    });
+}

package/dist/deepseek.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.CancelledError = exports.ApiError = void 0;
+exports.CallGate = exports.CancelledError = exports.ApiError = void 0;
+exports.gateFor = gateFor;
 exports.chat = chat;
 exports.isFatalAuthError = isFatalAuthError;
 exports.validateAuth = validateAuth;
@@ -13,10 +14,13 @@ function providerOf(cfg) {
 class ApiError extends Error {
     status;
     body;
-    constructor(status, body) {
+    /** Parsed Retry-After (ms) when the server sent one with a 429. */
+    retryAfterMs;
+    constructor(status, body, retryAfterMs) {
         super(`API ${status}: ${body.slice(0, 600)}`);
         this.status = status;
         this.body = body;
+        this.retryAfterMs = retryAfterMs;
     }
 }
 exports.ApiError = ApiError;
@@ -66,25 +70,135 @@ function sanitizeMessages(messages, thinking) {
     });
 }
 /**
- * One streaming chat-completions call with retries. Streams deltas (text and
- * reasoning), accumulates tool calls, captures usage from the final chunk.
+ * Bounds concurrent streaming calls per provider endpoint so a 100-agent swarm
+ * doesn't turn into a 429 storm. AIMD: a 429 halves the ceiling (min 2) and
+ * imposes the server's Retry-After as a cool-down; sustained successes recover
+ * it additively back toward the configured max. Two-tier FIFO: "high" priority
+ * (conductor/orchestration) jumps ahead so queued workers can't starve the
+ * brain of the swarm.
+ */
+class CallGate {
+    max;
+    ceiling;
+    active = 0;
+    high = [];
+    low = [];
+    successes = 0;
+    cooldownUntil = 0;
+    onState;
+    constructor(max) {
+        this.max = Math.max(1, max);
+        this.ceiling = this.max;
+    }
+    state() {
+        return { ceiling: this.ceiling, active: this.active, queued: this.high.length + this.low.length };
+    }
+    configure(max) {
+        this.max = Math.max(1, max);
+        if (this.ceiling > this.max)
+            this.ceiling = this.max;
+        this.pump();
+    }
+    async acquire(priority, signal) {
+        const wait = this.cooldownUntil - Date.now();
+        if (wait > 0)
+            await (0, util_1.sleep)(wait);
+        if (signal?.aborted)
+            throw new CancelledError();
+        if (this.active < this.ceiling) {
+            this.active++;
+            return;
+        }
+        await new Promise((resolve, reject) => {
+            const queue = priority === "high" ? this.high : this.low;
+            const entry = () => {
+                signal?.removeEventListener("abort", onAbort);
+                resolve();
+            };
+            const onAbort = () => {
+                const i = queue.indexOf(entry);
+                if (i >= 0)
+                    queue.splice(i, 1);
+                reject(new CancelledError());
+            };
+            signal?.addEventListener("abort", onAbort, { once: true });
+            queue.push(entry);
+        });
+    }
+    release() {
+        this.active = Math.max(0, this.active - 1);
+        this.pump();
+    }
+    reportRateLimit(retryAfterMs) {
+        this.ceiling = Math.max(2, Math.floor(this.ceiling / 2));
+        this.successes = 0;
+        if (retryAfterMs && retryAfterMs > 0) {
+            this.cooldownUntil = Math.max(this.cooldownUntil, Date.now() + Math.min(retryAfterMs, 60_000));
+        }
+        this.onState?.(this.state());
+    }
+    reportSuccess() {
+        if (this.ceiling >= this.max)
+            return;
+        if (++this.successes >= 10) {
+            this.successes = 0;
+            this.ceiling++;
+            this.onState?.(this.state());
+            this.pump();
+        }
+    }
+    pump() {
+        while (this.active < this.ceiling) {
+            const next = this.high.shift() ?? this.low.shift();
+            if (!next)
+                break;
+            this.active++;
+            next();
+        }
+    }
+}
+exports.CallGate = CallGate;
+const gates = new Map();
+function gateFor(cfg) {
+    const key = cfg.baseUrl;
+    let g = gates.get(key);
+    if (!g) {
+        g = new CallGate(cfg.maxConcurrentCalls);
+        gates.set(key, g);
+    }
+    g.configure(cfg.maxConcurrentCalls);
+    return g;
+}
+/**
+ * One streaming chat-completions call with retries, behind the global gate.
+ * The retry backoff sleeps OUTSIDE the gate so a waiting call never holds a
+ * concurrency slot.
  */
 async function chat(cfg, o) {
+    const gate = gateFor(cfg);
     let lastErr;
     const attempts = 4;
     for (let i = 0; i < attempts; i++) {
+        await gate.acquire(o.priority ?? "normal", o.signal);
         try {
-            return await chatOnce(cfg, o);
+            const res = await chatOnce(cfg, o);
+            gate.reportSuccess();
+            return res;
         }
         catch (e) {
             lastErr = e;
+            if (e instanceof ApiError && e.status === 429)
+                gate.reportRateLimit(e.retryAfterMs);
             if (!retryable(e) || i === attempts - 1)
                 throw e;
-            const backoff = [1500, 5000, 15000][i] ?? 15000;
-            await (0, util_1.sleep)(backoff + Math.random() * 1000);
-            if (o.signal?.aborted)
-                throw new CancelledError();
         }
+        finally {
+            gate.release();
+        }
+        const backoff = [1500, 5000, 15000][i] ?? 15000;
+        await (0, util_1.sleep)(backoff + Math.random() * 1000);
+        if (o.signal?.aborted)
+            throw new CancelledError();
     }
     throw lastErr;
 }
@@ -149,7 +263,8 @@ async function chatOnce(cfg, o) {
         });
         if (!res.ok) {
             const text = await res.text().catch(() => "");
-            throw new ApiError(res.status, text);
+            const ra = Number(res.headers.get("retry-after"));
+            throw new ApiError(res.status, text, Number.isFinite(ra) && ra >= 0 ? ra * 1000 : undefined);
         }
         if (!res.body)
             throw new ApiError(0, "empty response body");