@askalf/deepdive 0.1.0 → 0.4.0

package/dist/robots.js

@@ -0,0 +1,183 @@
+// robots.txt support — per-host fetch + parse + per-URL allow/deny check.
+//
+// Production-grade crawlers respect robots.txt. deepdive's fetch volume is
+// low (~12 URLs per query) but it's still the polite thing; sites with
+// explicit scraper deny rules shouldn't be surprised. --ignore-robots is
+// provided for operators who know what they're doing.
+//
+// Cache is per-run (in-memory) and keyed by `<scheme>://<host>`. We don't
+// persist to disk because the expected hit count per run is small and
+// robots.txt content can change rapidly on the publisher's end.
+export const DEFAULT_USER_AGENT = "deepdive-bot";
+export function createRobotsCache() {
+    const store = new Map();
+    return {
+        get: (origin) => store.get(origin),
+        set: (origin, parsed) => void store.set(origin, parsed),
+    };
+}
+export async function canFetch(url, opts) {
+    let origin;
+    let path;
+    try {
+        const u = new URL(url);
+        if (u.protocol !== "http:" && u.protocol !== "https:")
+            return "allow";
+        origin = `${u.protocol}//${u.host}`;
+        path = u.pathname + u.search;
+    }
+    catch {
+        return "allow";
+    }
+    const cached = opts.cache?.get(origin);
+    const parsed = cached === undefined
+        ? await fetchAndParse(origin, opts)
+        : cached;
+    if (opts.cache && cached === undefined)
+        opts.cache.set(origin, parsed);
+    if (parsed === null)
+        return "unknown"; // couldn't reach robots.txt
+    return isPathAllowed(parsed, path, opts.userAgent) ? "allow" : "deny";
+}
+async function fetchAndParse(origin, opts) {
+    const fetchImpl = opts.fetchImpl ?? fetch;
+    const timeoutMs = opts.timeoutMs ?? 5_000;
+    const timeout = AbortSignal.timeout(timeoutMs);
+    const signal = opts.signal
+        ? AbortSignal.any([opts.signal, timeout])
+        : timeout;
+    try {
+        const res = await fetchImpl(`${origin}/robots.txt`, {
+            headers: { "user-agent": opts.userAgent },
+            signal,
+        });
+        // Per RFC 9309: 4xx → no restrictions (no robots file); 5xx → treat as
+        // "full disallow" conservatively. We lean permissive for 5xx too since
+        // it's often transient and we don't want to lock out a run because the
+        // publisher's server is flaky. Callers can pass --ignore-robots if they
+        // want to bypass robots entirely.
+        if (res.status >= 400) {
+            return { rules: [] };
+        }
+        const text = await res.text();
+        return parseRobotsTxt(text);
+    }
+    catch {
+        return null;
+    }
+}
+// Exported for unit tests.
+export function parseRobotsTxt(text) {
+    const lines = text.split(/\r?\n/);
+    const grouped = [];
+    let currentAgents = [];
+    let sawRuleThisGroup = false;
+    let crawlDelay;
+    for (const rawLine of lines) {
+        const line = stripComment(rawLine).trim();
+        if (!line)
+            continue;
+        const match = /^([a-zA-Z-]+)\s*:\s*(.*)$/.exec(line);
+        if (!match)
+            continue;
+        const [, key, value] = match;
+        const lower = key.toLowerCase();
+        if (lower === "user-agent") {
+            if (sawRuleThisGroup) {
+                // new group
+                currentAgents = [];
+                sawRuleThisGroup = false;
+            }
+            currentAgents.push(value.trim().toLowerCase());
+        }
+        else if (lower === "disallow" || lower === "allow") {
+            sawRuleThisGroup = true;
+            for (const agent of currentAgents) {
+                grouped.push({
+                    agent,
+                    allow: lower === "allow",
+                    path: value.trim(),
+                });
+            }
+        }
+        else if (lower === "crawl-delay") {
+            const n = Number(value.trim());
+            if (Number.isFinite(n) && n >= 0)
+                crawlDelay = n;
+        }
+    }
+    return {
+        rules: grouped.map((g) => ({ allow: g.allow, path: g.path })),
+        crawlDelaySec: crawlDelay,
+        // We stash the grouping by keeping a hidden field. But since we want a
+        // clean exported type, bake agent-matching in: we'll re-do the parse at
+        // check time. Simpler: re-parse cheaply or store a bigger structure.
+        // Actually let's just store the grouped form and compute at check time:
+        ...{ _grouped: grouped },
+    };
+}
+// Exported for unit tests.
+export function isPathAllowed(parsed, path, userAgent) {
+    const grouped = parsed._grouped ?? [];
+    if (grouped.length === 0)
+        return true;
+    const ua = userAgent.toLowerCase();
+    // Pick matching rules: prefer exact agent match; fall back to '*'.
+    let applicable = grouped.filter((g) => g.agent && ua.includes(g.agent));
+    if (applicable.length === 0)
+        applicable = grouped.filter((g) => g.agent === "*");
+    if (applicable.length === 0)
+        return true;
+    // Pick the longest-matching rule. Tie → allow wins (RFC 9309).
+    let bestLen = -1;
+    let bestAllow = true;
+    for (const rule of applicable) {
+        if (!rule.path) {
+            // Empty Disallow: means allow everything. Empty Allow: is a no-op.
+            if (!rule.allow) {
+                if (bestLen < 0) {
+                    bestLen = 0;
+                    bestAllow = true; // empty Disallow explicitly grants
+                }
+            }
+            continue;
+        }
+        if (!pathMatches(rule.path, path))
+            continue;
+        if (rule.path.length > bestLen || (rule.path.length === bestLen && rule.allow)) {
+            bestLen = rule.path.length;
+            bestAllow = rule.allow;
+        }
+    }
+    return bestLen < 0 ? true : bestAllow;
+}
+function pathMatches(pattern, path) {
+    // Robots.txt patterns support * as wildcard and $ as end-anchor. For the
+    // simpler prefix patterns — which is what 95% of robots.txt files use — a
+    // startsWith check suffices. Anything fancier: compile to a regex.
+    if (!pattern.includes("*") && !pattern.endsWith("$")) {
+        return path.startsWith(pattern);
+    }
+    // Convert to regex, escaping other regex-special chars.
+    let re = "";
+    for (let i = 0; i < pattern.length; i++) {
+        const c = pattern[i];
+        if (c === "*")
+            re += ".*";
+        else if (c === "$" && i === pattern.length - 1)
+            re += "$";
+        else
+            re += c.replace(/[.+?^${}()|[\]\\]/g, "\\$&");
+    }
+    try {
+        return new RegExp("^" + re).test(path);
+    }
+    catch {
+        return false;
+    }
+}
+function stripComment(s) {
+    const i = s.indexOf("#");
+    return i === -1 ? s : s.slice(0, i);
+}
+//# sourceMappingURL=robots.js.map

package/dist/robots.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"robots.js","sourceRoot":"","sources":["../src/robots.ts"],"names":[],"mappings":"AAAA,0EAA0E;AAC1E,EAAE;AACF,2EAA2E;AAC3E,uEAAuE;AACvE,yEAAyE;AACzE,sDAAsD;AACtD,EAAE;AACF,0EAA0E;AAC1E,sEAAsE;AACtE,gEAAgE;AA6BhE,MAAM,CAAC,MAAM,kBAAkB,GAAG,cAAc,CAAC;AAEjD,MAAM,UAAU,iBAAiB;IAC/B,MAAM,KAAK,GAAG,IAAI,GAAG,EAA+B,CAAC;IACrD,OAAO;QACL,GAAG,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC;QAClC,GAAG,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC;KACxD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,GAAW,EACX,IAAqB;IAErB,IAAI,MAAc,CAAC;IACnB,IAAI,IAAY,CAAC;IACjB,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,CAAC,QAAQ,KAAK,OAAO,IAAI,CAAC,CAAC,QAAQ,KAAK,QAAQ;YAAE,OAAO,OAAO,CAAC;QACtE,MAAM,GAAG,GAAG,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;QACpC,IAAI,GAAG,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACvC,MAAM,MAAM,GACV,MAAM,KAAK,SAAS;QAClB,CAAC,CAAC,MAAM,aAAa,CAAC,MAAM,EAAE,IAAI,CAAC;QACnC,CAAC,CAAC,MAAM,CAAC;IACb,IAAI,IAAI,CAAC,KAAK,IAAI,MAAM,KAAK,SAAS;QAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEvE,IAAI,MAAM,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC,CAAC,4BAA4B;IACnE,OAAO,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;AACxE,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,MAAc,EACd,IAAqB;IAErB,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC;IAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC;IAC1C,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM;QACxB,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACzC,CAAC,CAAC,OAAO,CAAC;IACZ,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,GAAG,MAAM,aAAa,EAAE;YAClD,OAAO,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,SAAS,EAAE;YACzC,MAAM;SACP,CAAC,CAAC;QACH,uEAAuE;QACvE,uEAAuE;QACvE,uEAAuE;QACvE,wEAAwE;QACxE,kCAAkC;QAClC,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;YACtB,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;QACvB,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9B,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,2BAA2B;AAC3B,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAMlC,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,IAAI,aAAa,GAAa,EAAE,CAAC;IACjC,IAAI,gBAAgB,GAAG,KAAK,CAAC;IAC7B,IAAI,UAA8B,CAAC;IAEnC,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,MAAM,KAAK,GAAG,2BAA2B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrD,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,MAAM,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,KAAK,CAAC;QAC7B,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;QAChC,IAAI,KAAK,KAAK,YAAY,EAAE,CAAC;YAC3B,IAAI,gBAAgB,EAAE,CAAC;gBACrB,YAAY;gBACZ,aAAa,GAAG,EAAE,CAAC;gBACnB,gBAAgB,GAAG,KAAK,CAAC;YAC3B,CAAC;YACD,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;QACjD,CAAC;aAAM,IAAI,KAAK,KAAK,UAAU,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;YACrD,gBAAgB,GAAG,IAAI,CAAC;YACxB,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;gBAClC,OAAO,CAAC,IAAI,CAAC;oBACX,KAAK;oBACL,KAAK,EAAE,KAAK,KAAK,OAAO;oBACxB,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE;iBACnB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;aAAM,IAAI,KAAK,KAAK,aAAa,EAAE,CAAC;YACnC,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;YAC/B,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,UAAU,GAAG,CAAC,CAAC;QACnD,CAAC;IACH,CAAC;IAED,OAAO;QACL,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7D,aAAa,EAAE,UAAU;QACzB,uEAAuE;QACvE,wEAAwE;QACxE,qEAAqE;QACrE,wEAAwE;QACxE,GAAI,EAAE,QAAQ,EAAE,OAAO,EAAa;KACrB,CAAC;AACpB,CAAC;AAED,2BAA2B;AAC3B,MAAM,UAAU,aAAa,CAC3B,MAAoB,EACpB,IAAY,EACZ,SAAiB;IAEjB,MAAM,OAAO,GAAI,MAAsF,CAAC,QAAQ,IAAI,EAAE,CAAC;IACvH,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACtC,MAAM,EAAE,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC;IAEnC,mEAAmE;IACnE,IAAI,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IACxE,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,GAAG,CAAC,CAAC;IACjF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEzC,+DAA+D;IAC/D,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC;IACjB,IAAI,SAAS,GAAG,IAAI,CAAC;IACrB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACf,mEAAmE;YACnE,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;gBAChB,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;oBAChB,OAAO,GAAG,CAAC,CAAC;oBACZ,SAAS,GAAG,IAAI,CAAC,CAAC,mCAAmC;gBACvD,CAAC;YACH,CAAC;YACD,SAAS;QACX,CAAC;QACD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC;YAAE,SAAS;QAC5C,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/E,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YAC3B,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;AACxC,CAAC;AAED,SAAS,WAAW,CAAC,OAAe,EAAE,IAAY;IAChD,yEAAyE;IACzE,0EAA0E;IAC1E,mEAAmE;IACnE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACrD,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IAClC,CAAC;IACD,wDAAwD;IACxD,IAAI,EAAE,GAAG,EAAE,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG;YAAE,EAAE,IAAI,IAAI,CAAC;aACrB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,OAAO,CAAC,MAAM,GAAG,CAAC;YAAE,EAAE,IAAI,GAAG,CAAC;;YACrD,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,oBAAoB,EAAE,MAAM,CAAC,CAAC;IACrD,CAAC;IACD,IAAI,CAAC;QACH,OAAO,IAAI,MAAM,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACzC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,CAAS;IAC7B,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACzB,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACtC,CAAC"}

package/dist/search/duckduckgo.d.ts

@@ -4,4 +4,6 @@ export declare class DuckDuckGoSearch implements SearchAdapter {
     search(query: string, limit: number, signal?: AbortSignal): Promise<SearchResult[]>;
 }
 export declare function parseDuckDuckGoHTML(html: string, limit: number): SearchResult[];
+export declare function decodeHtmlEntities(s: string): string;
+export declare function stripTags(s: string): string;
 //# sourceMappingURL=duckduckgo.d.ts.map

package/dist/search/duckduckgo.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"duckduckgo.d.ts","sourceRoot":"","sources":["../../src/search/duckduckgo.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAIhE,qBAAa,gBAAiB,YAAW,aAAa;IACpD,QAAQ,CAAC,IAAI,gBAAgB;IAEvB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;CAiB1F;AAGD,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,CAuB/E"}
+{"version":3,"file":"duckduckgo.d.ts","sourceRoot":"","sources":["../../src/search/duckduckgo.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAIhE,qBAAa,gBAAiB,YAAW,aAAa;IACpD,QAAQ,CAAC,IAAI,gBAAgB;IAEvB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;CAiB1F;AAGD,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,CAuB/E;AA+BD,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAsBpD;AAWD,wBAAgB,SAAS,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAE3C"}

package/dist/search/duckduckgo.js

@@ -57,7 +57,8 @@ function unwrapDDGRedirect(href) {
         href = "https:" + href;
     try {
         const u = new URL(href);
-        if (u.hostname.endsWith("duckduckgo.com") && u.pathname === "/l/") {
+        const isDDG = u.hostname === "duckduckgo.com" || u.hostname.endsWith(".duckduckgo.com");
+        if (isDDG && u.pathname === "/l/") {
             const uddg = u.searchParams.get("uddg");
             if (uddg)
                 return decodeURIComponent(uddg);
@@ -77,18 +78,42 @@ function isValidHttpUrl(s) {
         return false;
     }
 }
-function decodeHtmlEntities(s) {
-    return s
-        .replace(/&/g, "&")
-        .replace(/&lt;/g, "<")
-        .replace(/&gt;/g, ">")
-        .replace(/&quot;/g, '"')
-        .replace(/&#39;/g, "'")
-        .replace(/&nbsp;/g, " ")
-        .replace(/&#x2F;/g, "/")
-        .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(parseInt(n, 10)));
+// Exported for unit tests. Single-pass HTML entity decoder — each `&...;`
+// token is resolved exactly once with no rescan, so `&amp;#39;` decodes to
+// the literal `&#39;` rather than double-unescaping to `'`.
+export function decodeHtmlEntities(s) {
+    return s.replace(/&(#[0-9]+|#[xX][0-9a-fA-F]+|[a-zA-Z]+);/g, (match, name) => {
+        const named = {
+            amp: "&",
+            lt: "<",
+            gt: ">",
+            quot: '"',
+            apos: "'",
+            nbsp: " ",
+        };
+        const low = name.toLowerCase();
+        if (low in named)
+            return named[low];
+        if (name.startsWith("#x") || name.startsWith("#X")) {
+            const code = parseInt(name.slice(2), 16);
+            return isValidCodePoint(code) ? String.fromCodePoint(code) : match;
+        }
+        if (name.startsWith("#")) {
+            const code = parseInt(name.slice(1), 10);
+            return isValidCodePoint(code) ? String.fromCodePoint(code) : match;
+        }
+        return match;
+    });
+}
+function isValidCodePoint(n) {
+    return Number.isFinite(n) && n >= 0 && n <= 0x10ffff;
 }
-function stripTags(s) {
-    return s.replace(/<[^>]+>/g, "");
+// Exported for unit tests. Strips well-formed tags, then drops any stray `<`
+// to defuse malformed/partial tags (e.g. `<scrip` with no closing `>`).
+// Output is plain text destined for markdown citation rows — not
+// HTML-rendered — but we harden here so a malformed snippet can never leak
+// a tag opener into downstream consumers.
+export function stripTags(s) {
+    return s.replace(/<[^>]*>/g, " ").split("<").join(" ");
 }
 //# sourceMappingURL=duckduckgo.js.map

package/dist/search/duckduckgo.js.map

@@ -1 +1 @@
-{"version":3,"file":"duckduckgo.js","sourceRoot":"","sources":["../../src/search/duckduckgo.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,4EAA4E;AAC5E,6EAA6E;AAC7E,0DAA0D;AAC1D,EAAE;AACF,gFAAgF;AAIhF,MAAM,QAAQ,GAAG,mCAAmC,CAAC;AAErD,MAAM,OAAO,gBAAgB;IAClB,IAAI,GAAG,YAAY,CAAC;IAE7B,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,KAAa,EAAE,MAAoB;QAC7D,MAAM,IAAI,GAAG,IAAI,eAAe,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/C,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;YAChC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,mCAAmC;gBACnD,YAAY,EACV,+DAA+D;oBAC/D,oDAAoD;aACvD;YACD,IAAI,EAAE,IAAI,CAAC,QAAQ,EAAE;YACrB,MAAM;SACP,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,cAAc,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QAC3E,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9B,OAAO,mBAAmB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAC1C,CAAC;CACF;AAED,2BAA2B;AAC3B,MAAM,UAAU,mBAAmB,CAAC,IAAY,EAAE,KAAa;IAC7D,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,oEAAoE;IACpE,6DAA6D;IAC7D,MAAM,OAAO,GACX,8JAA8J,CAAC;IAEjK,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,CAAyB,CAAC;IAC9B,OAAO,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACzC,MAAM,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC;QAC5C,MAAM,GAAG,GAAG,iBAAiB,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC;QAC3D,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC;YAAE,SAAS;QACnC,IAAI,EAAE,CAAC;QACP,OAAO,CAAC,IAAI,CAAC;YACX,GAAG;YACH,KAAK,EAAE,SAAS,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE;YACrD,OAAO,EAAE,SAAS,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,EAAE;YACzD,IAAI;SACL,CAAC,CAAC;QACH,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK;YAAE,MAAM;IACrC,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY;IACrC,2EAA2E;IAC3E,wCAAwC;IACxC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,IAAI,GAAG,QAAQ,GAAG,IAAI,CAAC;IAClD,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;QACxB,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,QAAQ,KAAK,KAAK,EAAE,CAAC;YAClE,MAAM,IAAI,GAAG,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACxC,IAAI,IAAI;gBAAE,OAAO,kBAAkB,CAAC,IAAI,CAAC,CAAC;QAC5C,CAAC;QACD,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;IACtB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,CAAS;IAC/B,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;QACrB,OAAO,CAAC,CAAC,QAAQ,KAAK,OAAO,IAAI,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC;IAC3D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,CAAS;IACnC,OAAO,CAAC;SACL,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;AAC1E,CAAC;AAED,SAAS,SAAS,CAAC,CAAS;IAC1B,OAAO,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;AACnC,CAAC"}
+{"version":3,"file":"duckduckgo.js","sourceRoot":"","sources":["../../src/search/duckduckgo.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,4EAA4E;AAC5E,6EAA6E;AAC7E,0DAA0D;AAC1D,EAAE;AACF,gFAAgF;AAIhF,MAAM,QAAQ,GAAG,mCAAmC,CAAC;AAErD,MAAM,OAAO,gBAAgB;IAClB,IAAI,GAAG,YAAY,CAAC;IAE7B,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,KAAa,EAAE,MAAoB;QAC7D,MAAM,IAAI,GAAG,IAAI,eAAe,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/C,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;YAChC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,mCAAmC;gBACnD,YAAY,EACV,+DAA+D;oBAC/D,oDAAoD;aACvD;YACD,IAAI,EAAE,IAAI,CAAC,QAAQ,EAAE;YACrB,MAAM;SACP,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,cAAc,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QAC3E,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9B,OAAO,mBAAmB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAC1C,CAAC;CACF;AAED,2BAA2B;AAC3B,MAAM,UAAU,mBAAmB,CAAC,IAAY,EAAE,KAAa;IAC7D,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,oEAAoE;IACpE,6DAA6D;IAC7D,MAAM,OAAO,GACX,8JAA8J,CAAC;IAEjK,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,CAAyB,CAAC;IAC9B,OAAO,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACzC,MAAM,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC;QAC5C,MAAM,GAAG,GAAG,iBAAiB,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC;QAC3D,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC;YAAE,SAAS;QACnC,IAAI,EAAE,CAAC;QACP,OAAO,CAAC,IAAI,CAAC;YACX,GAAG;YACH,KAAK,EAAE,SAAS,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE;YACrD,OAAO,EAAE,SAAS,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,EAAE;YACzD,IAAI;SACL,CAAC,CAAC;QACH,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK;YAAE,MAAM;IACrC,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY;IACrC,2EAA2E;IAC3E,wCAAwC;IACxC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,IAAI,GAAG,QAAQ,GAAG,IAAI,CAAC;IAClD,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;QACxB,MAAM,KAAK,GAAG,CAAC,CAAC,QAAQ,KAAK,gBAAgB,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QACxF,IAAI,KAAK,IAAI,CAAC,CAAC,QAAQ,KAAK,KAAK,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACxC,IAAI,IAAI;gBAAE,OAAO,kBAAkB,CAAC,IAAI,CAAC,CAAC;QAC5C,CAAC;QACD,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;IACtB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,CAAS;IAC/B,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;QACrB,OAAO,CAAC,CAAC,QAAQ,KAAK,OAAO,IAAI,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC;IAC3D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,0EAA0E;AAC1E,2EAA2E;AAC3E,4DAA4D;AAC5D,MAAM,UAAU,kBAAkB,CAAC,CAAS;IAC1C,OAAO,CAAC,CAAC,OAAO,CAAC,0CAA0C,EAAE,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;QAC3E,MAAM,KAAK,GAA2B;YACpC,GAAG,EAAE,GAAG;YACR,EAAE,EAAE,GAAG;YACP,EAAE,EAAE,GAAG;YACP,IAAI,EAAE,GAAG;YACT,IAAI,EAAE,GAAG;YACT,IAAI,EAAE,GAAG;SACV,CAAC;QACF,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAC/B,IAAI,GAAG,IAAI,KAAK;YAAE,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACnD,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACzC,OAAO,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QACrE,CAAC;QACD,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACzC,OAAO,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QACrE,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,gBAAgB,CAAC,CAAS;IACjC,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC;AACvD,CAAC;AAED,6EAA6E;AAC7E,wEAAwE;AACxE,iEAAiE;AACjE,2EAA2E;AAC3E,0CAA0C;AAC1C,MAAM,UAAU,SAAS,CAAC,CAAS;IACjC,OAAO,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzD,CAAC"}

package/dist/search/exa.d.ts

@@ -0,0 +1,17 @@
+import type { SearchAdapter, SearchResult } from "../search.js";
+interface ExaResponseItem {
+    url: string;
+    title?: string | null;
+    text?: string;
+    highlights?: string[];
+    summary?: string;
+}
+export declare class ExaSearch implements SearchAdapter {
+    private readonly key;
+    readonly name = "exa";
+    constructor(key: string);
+    search(query: string, limit: number, signal?: AbortSignal): Promise<SearchResult[]>;
+}
+export declare function mapExaResults(items: ExaResponseItem[], limit: number): SearchResult[];
+export {};
+//# sourceMappingURL=exa.d.ts.map

package/dist/search/exa.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"exa.d.ts","sourceRoot":"","sources":["../../src/search/exa.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAEhE,UAAU,eAAe;IACvB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAMD,qBAAa,SAAU,YAAW,aAAa;IAEjC,OAAO,CAAC,QAAQ,CAAC,GAAG;IADhC,QAAQ,CAAC,IAAI,SAAS;gBACO,GAAG,EAAE,MAAM;IAElC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;CAqB1F;AAGD,wBAAgB,aAAa,CAAC,KAAK,EAAE,eAAe,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,CAOrF"}

package/dist/search/exa.js

@@ -0,0 +1,62 @@
+// Exa adapter. Requires DEEPDIVE_EXA_KEY. Exa is a neural search API tuned
+// for research workloads — the planner's sub-queries (and especially the
+// critic's gap-filling follow-ups) tend to be long, intent-rich strings,
+// which is the shape Exa's embedding-based retrieval is built for.
+//
+// We request highlights (short, query-focused excerpts) for each result so
+// the snippet shown in events/logs is informative. Full-page text is left
+// to deepdive's normal Playwright fetch step so the downstream extract /
+// synthesis path is identical across adapters.
+export class ExaSearch {
+    key;
+    name = "exa";
+    constructor(key) {
+        this.key = key;
+    }
+    async search(query, limit, signal) {
+        const res = await fetch("https://api.exa.ai/search", {
+            method: "POST",
+            headers: {
+                "content-type": "application/json",
+                accept: "application/json",
+                "x-api-key": this.key,
+                "x-exa-integration": "deepdive",
+            },
+            body: JSON.stringify({
+                query,
+                numResults: Math.min(limit, 100),
+                type: "auto",
+                contents: { highlights: { numSentences: 2 } },
+            }),
+            signal,
+        });
+        if (!res.ok)
+            throw new Error(`exa ${res.status} ${res.statusText}`);
+        const json = (await res.json());
+        return mapExaResults(json.results ?? [], limit);
+    }
+}
+// Exported for unit tests. Pure, deterministic, no I/O.
+export function mapExaResults(items, limit) {
+    return items.slice(0, limit).map((r, i) => ({
+        url: r.url,
+        title: r.title ?? "",
+        snippet: extractSnippet(r),
+        rank: i + 1,
+    }));
+}
+// Snippet cascade: highlights → text → summary → "". Exa may return any
+// combination depending on what was requested or what the page yielded.
+function extractSnippet(r) {
+    if (r.highlights && r.highlights.length > 0) {
+        return r.highlights.join(" … ").trim();
+    }
+    if (r.text && r.text.length > 0) {
+        return r.text.slice(0, 500).trim();
+    }
+    if (r.summary && r.summary.length > 0) {
+        return r.summary.trim();
+    }
+    return "";
+}
+//# sourceMappingURL=exa.js.map

package/dist/search/exa.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"exa.js","sourceRoot":"","sources":["../../src/search/exa.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,yEAAyE;AACzE,yEAAyE;AACzE,mEAAmE;AACnE,EAAE;AACF,2EAA2E;AAC3E,0EAA0E;AAC1E,yEAAyE;AACzE,+CAA+C;AAgB/C,MAAM,OAAO,SAAS;IAES;IADpB,IAAI,GAAG,KAAK,CAAC;IACtB,YAA6B,GAAW;QAAX,QAAG,GAAH,GAAG,CAAQ;IAAG,CAAC;IAE5C,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,KAAa,EAAE,MAAoB;QAC7D,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,2BAA2B,EAAE;YACnD,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,MAAM,EAAE,kBAAkB;gBAC1B,WAAW,EAAE,IAAI,CAAC,GAAG;gBACrB,mBAAmB,EAAE,UAAU;aAChC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK;gBACL,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC;gBAChC,IAAI,EAAE,MAAM;gBACZ,QAAQ,EAAE,EAAE,UAAU,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,EAAE;aAC9C,CAAC;YACF,MAAM;SACP,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,OAAO,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACpE,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAgB,CAAC;QAC/C,OAAO,aAAa,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,EAAE,KAAK,CAAC,CAAC;IAClD,CAAC;CACF;AAED,wDAAwD;AACxD,MAAM,UAAU,aAAa,CAAC,KAAwB,EAAE,KAAa;IACnE,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAC1C,GAAG,EAAE,CAAC,CAAC,GAAG;QACV,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,EAAE;QACpB,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC;QAC1B,IAAI,EAAE,CAAC,GAAG,CAAC;KACZ,CAAC,CAAC,CAAC;AACN,CAAC;AAED,wEAAwE;AACxE,wEAAwE;AACxE,SAAS,cAAc,CAAC,CAAkB;IACxC,IAAI,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5C,OAAO,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;IACzC,CAAC;IACD,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACrC,CAAC;IACD,IAAI,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtC,OAAO,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC"}

package/dist/search/searxng.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"searxng.d.ts","sourceRoot":"","sources":["../../src/search/searxng.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAEhE,qBAAa,aAAc,YAAW,aAAa;IAErC,OAAO,CAAC,QAAQ,CAAC,OAAO;IADpC,QAAQ,CAAC,IAAI,aAAa;gBACG,OAAO,EAAE,MAAM;IAEtC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;CAoB1F"}
+{"version":3,"file":"searxng.d.ts","sourceRoot":"","sources":["../../src/search/searxng.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAGhE,qBAAa,aAAc,YAAW,aAAa;IAErC,OAAO,CAAC,QAAQ,CAAC,OAAO;IADpC,QAAQ,CAAC,IAAI,aAAa;gBACG,OAAO,EAAE,MAAM;IAEtC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;CAoB1F"}

package/dist/search/searxng.js

@@ -1,5 +1,6 @@
 // SearXNG adapter. Points at an existing SearXNG instance (self-hosted or
 // public). Requires DEEPDIVE_SEARXNG_URL. Uses the JSON output format.
+import { trimTrailingSlashes } from "../url-util.js";
 export class SearXNGSearch {
     baseUrl;
     name = "searxng";
@@ -7,7 +8,7 @@ export class SearXNGSearch {
         this.baseUrl = baseUrl;
     }
     async search(query, limit, signal) {
-        const url = new URL(this.baseUrl.replace(/\/+$/, "") + "/search");
+        const url = new URL(trimTrailingSlashes(this.baseUrl) + "/search");
         url.searchParams.set("q", query);
         url.searchParams.set("format", "json");
         const res = await fetch(url, {

package/dist/search/searxng.js.map

@@ -1 +1 @@
-{"version":3,"file":"searxng.js","sourceRoot":"","sources":["../../src/search/searxng.ts"],"names":[],"mappings":"AAAA,0EAA0E;AAC1E,uEAAuE;AAIvE,MAAM,OAAO,aAAa;IAEK;IADpB,IAAI,GAAG,SAAS,CAAC;IAC1B,YAA6B,OAAe;QAAf,YAAO,GAAP,OAAO,CAAQ;IAAG,CAAC;IAEhD,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,KAAa,EAAE,MAAoB;QAC7D,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;QAClE,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACjC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACvC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC3B,OAAO,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE;YACvC,MAAM;SACP,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,WAAW,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACxE,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAE7B,CAAC;QACF,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;QACjC,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YAC1C,GAAG,EAAE,CAAC,CAAC,GAAG;YACV,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,EAAE;YACpB,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,EAAE;YACxB,IAAI,EAAE,CAAC,GAAG,CAAC;SACZ,CAAC,CAAC,CAAC;IACN,CAAC;CACF"}
+{"version":3,"file":"searxng.js","sourceRoot":"","sources":["../../src/search/searxng.ts"],"names":[],"mappings":"AAAA,0EAA0E;AAC1E,uEAAuE;AAGvE,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAErD,MAAM,OAAO,aAAa;IAEK;IADpB,IAAI,GAAG,SAAS,CAAC;IAC1B,YAA6B,OAAe;QAAf,YAAO,GAAP,OAAO,CAAQ;IAAG,CAAC;IAEhD,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,KAAa,EAAE,MAAoB;QAC7D,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,mBAAmB,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC;QACnE,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACjC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACvC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC3B,OAAO,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE;YACvC,MAAM;SACP,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,WAAW,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACxE,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAE7B,CAAC;QACF,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;QACjC,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YAC1C,GAAG,EAAE,CAAC,CAAC,GAAG;YACV,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,EAAE;YACpB,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,EAAE;YACxB,IAAI,EAAE,CAAC,GAAG,CAAC;SACZ,CAAC,CAAC,CAAC;IACN,CAAC;CACF"}

package/dist/search.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;CACrF;AAED,wBAAsB,oBAAoB,CACxC,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,GACtC,OAAO,CAAC,aAAa,CAAC,CA4BxB;AAED,wBAAgB,WAAW,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,YAAY,EAAE,CAUnE"}
+{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;CACrF;AAED,wBAAsB,oBAAoB,CACxC,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,GACtC,OAAO,CAAC,aAAa,CAAC,CAkCxB;AAED,wBAAgB,WAAW,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,YAAY,EAAE,CAUnE"}

package/dist/search.js

@@ -1,5 +1,6 @@
 // Search adapter interface. A "search" returns candidate URLs with metadata.
 // Adapters live under src/search/*. Default is DuckDuckGo HTML (no API key).
+import { dedupeKey } from "./url-util.js";
 export async function resolveSearchAdapter(name, env) {
     switch (name) {
         case "duckduckgo":
@@ -28,6 +29,13 @@ export async function resolveSearchAdapter(name, env) {
                 throw new Error("tavily adapter requires DEEPDIVE_TAVILY_KEY");
             return new TavilySearch(key);
         }
+        case "exa": {
+            const { ExaSearch } = await import("./search/exa.js");
+            const key = env.DEEPDIVE_EXA_KEY;
+            if (!key)
+                throw new Error("exa adapter requires DEEPDIVE_EXA_KEY");
+            return new ExaSearch(key);
+        }
         default:
             throw new Error(`unknown search adapter: ${name}`);
     }
@@ -36,7 +44,7 @@ export function dedupeByUrl(results) {
     const seen = new Set();
     const out = [];
     for (const r of results) {
-        const key = r.url.replace(/#.*$/, "").replace(/\/+$/, "");
+        const key = dedupeKey(r.url);
         if (seen.has(key))
             continue;
         seen.add(key);

package/dist/search.js.map

@@ -1 +1 @@
-{"version":3,"file":"search.js","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAC7E,6EAA6E;AAc7E,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAAY,EACZ,GAAuC;IAEvC,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,YAAY,CAAC;QAClB,KAAK,KAAK,CAAC,CAAC,CAAC;YACX,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,wBAAwB,CAAC,CAAC;YACpE,OAAO,IAAI,gBAAgB,EAAE,CAAC;QAChC,CAAC;QACD,KAAK,SAAS,CAAC,CAAC,CAAC;YACf,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAC;YAC9D,MAAM,GAAG,GAAG,GAAG,CAAC,oBAAoB,CAAC;YACrC,IAAI,CAAC,GAAG;gBAAE,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;YAC3E,OAAO,IAAI,aAAa,CAAC,GAAG,CAAC,CAAC;QAChC,CAAC;QACD,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;YAC1D,MAAM,GAAG,GAAG,GAAG,CAAC,kBAAkB,CAAC;YACnC,IAAI,CAAC,GAAG;gBAAE,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;YACvE,OAAO,IAAI,WAAW,CAAC,GAAG,CAAC,CAAC;QAC9B,CAAC;QACD,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;YAC5D,MAAM,GAAG,GAAG,GAAG,CAAC,mBAAmB,CAAC;YACpC,IAAI,CAAC,GAAG;gBAAE,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;YACzE,OAAO,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC;QAC/B,CAAC;QACD;YACE,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC;IACvD,CAAC;AACH,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,OAAuB;IACjD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,GAAG,GAAmB,EAAE,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC1D,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QAC5B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACd,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
+{"version":3,"file":"search.js","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAC7E,6EAA6E;AAE7E,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAc1C,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAAY,EACZ,GAAuC;IAEvC,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,YAAY,CAAC;QAClB,KAAK,KAAK,CAAC,CAAC,CAAC;YACX,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,wBAAwB,CAAC,CAAC;YACpE,OAAO,IAAI,gBAAgB,EAAE,CAAC;QAChC,CAAC;QACD,KAAK,SAAS,CAAC,CAAC,CAAC;YACf,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAC;YAC9D,MAAM,GAAG,GAAG,GAAG,CAAC,oBAAoB,CAAC;YACrC,IAAI,CAAC,GAAG;gBAAE,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;YAC3E,OAAO,IAAI,aAAa,CAAC,GAAG,CAAC,CAAC;QAChC,CAAC;QACD,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;YAC1D,MAAM,GAAG,GAAG,GAAG,CAAC,kBAAkB,CAAC;YACnC,IAAI,CAAC,GAAG;gBAAE,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;YACvE,OAAO,IAAI,WAAW,CAAC,GAAG,CAAC,CAAC;QAC9B,CAAC;QACD,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;YAC5D,MAAM,GAAG,GAAG,GAAG,CAAC,mBAAmB,CAAC;YACpC,IAAI,CAAC,GAAG;gBAAE,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;YACzE,OAAO,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC;QAC/B,CAAC;QACD,KAAK,KAAK,CAAC,CAAC,CAAC;YACX,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,CAAC;YACtD,MAAM,GAAG,GAAG,GAAG,CAAC,gBAAgB,CAAC;YACjC,IAAI,CAAC,GAAG;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YACnE,OAAO,IAAI,SAAS,CAAC,GAAG,CAAC,CAAC;QAC5B,CAAC;QACD;YACE,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC;IACvD,CAAC;AACH,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,OAAuB;IACjD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,GAAG,GAAmB,EAAE,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QAC5B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACd,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}

package/dist/synthesize.d.ts

@@ -3,6 +3,6 @@ import type { Source } from "./citations.js";
 export interface SourceWithContent extends Source {
     content: string;
 }
-export declare function synthesize(question: string, sources: SourceWithContent[], config: LLMConfig, signal?: AbortSignal): Promise<string>;
+export declare function synthesize(question: string, sources: SourceWithContent[], config: LLMConfig, signal?: AbortSignal, onToken?: (text: string) => void): Promise<string>;
 export declare function buildSourcePacket(sources: SourceWithContent[]): string;
 //# sourceMappingURL=synthesize.d.ts.map

package/dist/synthesize.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"synthesize.d.ts","sourceRoot":"","sources":["../src/synthesize.ts"],"names":[],"mappings":"AAIA,OAAO,EAAW,KAAK,SAAS,EAAE,MAAM,UAAU,CAAC;AACnD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAE7C,MAAM,WAAW,iBAAkB,SAAQ,MAAM;IAC/C,OAAO,EAAE,MAAM,CAAC;CACjB;AAkBD,wBAAsB,UAAU,CAC9B,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,iBAAiB,EAAE,EAC5B,MAAM,EAAE,SAAS,EACjB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,MAAM,CAAC,CAgBjB;AAGD,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,iBAAiB,EAAE,GAAG,MAAM,CAOtE"}
+{"version":3,"file":"synthesize.d.ts","sourceRoot":"","sources":["../src/synthesize.ts"],"names":[],"mappings":"AAOA,OAAO,EAAW,KAAK,SAAS,EAAE,MAAM,UAAU,CAAC;AAEnD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAE7C,MAAM,WAAW,iBAAkB,SAAQ,MAAM;IAC/C,OAAO,EAAE,MAAM,CAAC;CACjB;AAkBD,wBAAsB,UAAU,CAC9B,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,iBAAiB,EAAE,EAC5B,MAAM,EAAE,SAAS,EACjB,MAAM,CAAC,EAAE,WAAW,EACpB,OAAO,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC/B,OAAO,CAAC,MAAM,CAAC,CAsBjB;AAGD,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,iBAAiB,EAAE,GAAG,MAAM,CAOtE"}

package/dist/synthesize.js

@@ -1,7 +1,11 @@
 // Final answer synthesis — takes the original question and the collected
 // sources, asks the LLM to produce a cited markdown answer. Sources are
 // passed as a numbered list so the model can cite them inline as [1], [2].
+//
+// When `onToken` is provided, uses the streaming variant so tokens land in
+// front of the user as the model writes them instead of after a 30+s wait.
 import { callLLM } from "./llm.js";
+import { callLLMStream } from "./llm-stream.js";
 const SYNTH_SYSTEM = `You are a careful research assistant. You will be given:
 1. The user's original question.
 2. A numbered list of source documents with titles, URLs, and extracted text.
@@ -17,7 +21,7 @@ Rules:
 - If the sources do not answer the question, say so — do not hallucinate.
 - Do not include a "Sources" section yourself — the caller appends it.
 - Length: match the complexity of the question. A one-line question can get a paragraph; a comparison question may need headers and a table.`;
-export async function synthesize(question, sources, config, signal) {
+export async function synthesize(question, sources, config, signal, onToken) {
     if (sources.length === 0) {
         return "_No sources could be fetched or extracted. Unable to answer._";
     }
@@ -25,7 +29,12 @@ export async function synthesize(question, sources, config, signal) {
     const userMessage = `Question: ${question}\n\n` +
         `Sources (${sources.length}):\n\n${packet}\n\n` +
         `Write the cited markdown answer now.`;
-    const { text } = await callLLM([{ role: "user", content: userMessage }], SYNTH_SYSTEM, config, signal);
+    const messages = [{ role: "user", content: userMessage }];
+    if (onToken) {
+        const { text } = await callLLMStream(messages, SYNTH_SYSTEM, config, { onToken }, signal);
+        return text;
+    }
+    const { text } = await callLLM(messages, SYNTH_SYSTEM, config, signal);
     return text;
 }
 // Exported for unit tests.

package/dist/synthesize.js.map

@@ -1 +1 @@
-{"version":3,"file":"synthesize.js","sourceRoot":"","sources":["../src/synthesize.ts"],"names":[],"mappings":"AAAA,yEAAyE;AACzE,wEAAwE;AACxE,2EAA2E;AAE3E,OAAO,EAAE,OAAO,EAAkB,MAAM,UAAU,CAAC;AAOnD,MAAM,YAAY,GAAG;;;;;;;;;;;;;;6IAcwH,CAAC;AAE9I,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,QAAgB,EAChB,OAA4B,EAC5B,MAAiB,EACjB,MAAoB;IAEpB,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,+DAA+D,CAAC;IACzE,CAAC;IACD,MAAM,MAAM,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAC1C,MAAM,WAAW,GACf,aAAa,QAAQ,MAAM;QAC3B,YAAY,OAAO,CAAC,MAAM,SAAS,MAAM,MAAM;QAC/C,sCAAsC,CAAC;IACzC,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,OAAO,CAC5B,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,EACxC,YAAY,EACZ,MAAM,EACN,MAAM,CACP,CAAC;IACF,OAAO,IAAI,CAAC;AACd,CAAC;AAED,2BAA2B;AAC3B,MAAM,UAAU,iBAAiB,CAAC,OAA4B;IAC5D,OAAO,OAAO;SACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,KAAK,IAAI,YAAY,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC;QACjE,OAAO,GAAG,MAAM,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC;IACrC,CAAC,CAAC;SACD,IAAI,CAAC,aAAa,CAAC,CAAC;AACzB,CAAC"}
+{"version":3,"file":"synthesize.js","sourceRoot":"","sources":["../src/synthesize.ts"],"names":[],"mappings":"AAAA,yEAAyE;AACzE,wEAAwE;AACxE,2EAA2E;AAC3E,EAAE;AACF,2EAA2E;AAC3E,2EAA2E;AAE3E,OAAO,EAAE,OAAO,EAAkB,MAAM,UAAU,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAOhD,MAAM,YAAY,GAAG;;;;;;;;;;;;;;6IAcwH,CAAC;AAE9I,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,QAAgB,EAChB,OAA4B,EAC5B,MAAiB,EACjB,MAAoB,EACpB,OAAgC;IAEhC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,+DAA+D,CAAC;IACzE,CAAC;IACD,MAAM,MAAM,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAC1C,MAAM,WAAW,GACf,aAAa,QAAQ,MAAM;QAC3B,YAAY,OAAO,CAAC,MAAM,SAAS,MAAM,MAAM;QAC/C,sCAAsC,CAAC;IACzC,MAAM,QAAQ,GAAG,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,CAAC;IACnE,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,aAAa,CAClC,QAAQ,EACR,YAAY,EACZ,MAAM,EACN,EAAE,OAAO,EAAE,EACX,MAAM,CACP,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,OAAO,CAAC,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IACvE,OAAO,IAAI,CAAC;AACd,CAAC;AAED,2BAA2B;AAC3B,MAAM,UAAU,iBAAiB,CAAC,OAA4B;IAC5D,OAAO,OAAO;SACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,KAAK,IAAI,YAAY,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC;QACjE,OAAO,GAAG,MAAM,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC;IACrC,CAAC,CAAC;SACD,IAAI,CAAC,aAAa,CAAC,CAAC;AACzB,CAAC"}

package/dist/url-util.d.ts

@@ -0,0 +1,4 @@
+export declare function trimTrailingSlashes(s: string): string;
+export declare function stripHashFragment(s: string): string;
+export declare function dedupeKey(url: string): string;
+//# sourceMappingURL=url-util.d.ts.map

package/dist/url-util.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"url-util.d.ts","sourceRoot":"","sources":["../src/url-util.ts"],"names":[],"mappings":"AAMA,wBAAgB,mBAAmB,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAIrD;AAGD,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAGnD;AAMD,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAE7C"}

package/dist/url-util.js

@@ -0,0 +1,24 @@
+// URL canonicalization helpers. Kept as non-regex string operations so they
+// are trivially linear-time even on pathological inputs — CodeQL flags
+// equivalent regex forms (`/\/+$/`) as polynomial-ReDoS risks despite being
+// end-anchored.
+// Exported for unit tests.
+export function trimTrailingSlashes(s) {
+    let i = s.length;
+    while (i > 0 && s.charCodeAt(i - 1) === 0x2f)
+        i--;
+    return s.slice(0, i);
+}
+// Exported for unit tests.
+export function stripHashFragment(s) {
+    const i = s.indexOf("#");
+    return i === -1 ? s : s.slice(0, i);
+}
+// Exported for unit tests. Returns a dedupe key: scheme+host+path+query with
+// any trailing slashes removed and any URL fragment stripped. Fragment-only
+// changes shouldn't create duplicate cache entries; trailing-slash variants
+// shouldn't either.
+export function dedupeKey(url) {
+    return trimTrailingSlashes(stripHashFragment(url));
+}
+//# sourceMappingURL=url-util.js.map

package/dist/url-util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"url-util.js","sourceRoot":"","sources":["../src/url-util.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,uEAAuE;AACvE,4EAA4E;AAC5E,gBAAgB;AAEhB,2BAA2B;AAC3B,MAAM,UAAU,mBAAmB,CAAC,CAAS;IAC3C,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;IACjB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI;QAAE,CAAC,EAAE,CAAC;IAClD,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACvB,CAAC;AAED,2BAA2B;AAC3B,MAAM,UAAU,iBAAiB,CAAC,CAAS;IACzC,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACzB,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACtC,CAAC;AAED,6EAA6E;AAC7E,4EAA4E;AAC5E,4EAA4E;AAC5E,oBAAoB;AACpB,MAAM,UAAU,SAAS,CAAC,GAAW;IACnC,OAAO,mBAAmB,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC;AACrD,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@askalf/deepdive",
-  "version": "0.1.0",
+  "version": "0.4.0",
   "description": "A local research agent. One command, cited answer. Routes every LLM call through your own proxy (dario, Anthropic-compat, OpenAI-compat). Headless browser + pluggable search + multi-provider LLM — zero hosted dependencies.",
   "type": "module",
   "bin": {
@@ -22,6 +22,7 @@
   ],
   "scripts": {
     "build": "tsc",
+    "typecheck": "tsc --noEmit",
     "test": "node --test --test-concurrency=4 test/*.test.mjs",
     "dev": "tsx src/cli.ts",
     "start": "node dist/cli.js",
@@ -60,7 +61,7 @@
     "playwright": "^1.47.0"
   },
   "devDependencies": {
-    "@types/node": "^22.0.0",
+    "@types/node": "^25.6.0",
     "tsx": "^4.19.0",
     "typescript": "^5.7.0"
   }