@ahkohd/yagami 0.1.2

package/dist/engine/search-discovery.js

@@ -0,0 +1,275 @@
+import { categoryProfile, clampInteger, domainMatches, getHostname, isTrackingOrAdUrl, isValidPublicHostname, normalizeDomainFilter, normalizeWhitespace, parseStringList, stripHtml, unwrapDuckDuckGoHref, } from "./helpers.js";
+import { normalizeUrl } from "./url-utils.js";
+const SEARCH_ENGINE_PRESETS = {
+    duckduckgo: "https://duckduckgo.com/html/?q={query}",
+    bing: "https://www.bing.com/search?q={query}",
+    google: "https://www.google.com/search?q={query}",
+    brave: "https://search.brave.com/search?q={query}",
+};
+function normalizeSearchEngine(value, fallback = "duckduckgo") {
+    const normalized = String(value ?? "")
+        .trim()
+        .toLowerCase();
+    if (normalized === "duckduckgo")
+        return "duckduckgo";
+    if (normalized === "bing")
+        return "bing";
+    if (normalized === "google")
+        return "google";
+    if (normalized === "brave")
+        return "brave";
+    if (normalized === "custom")
+        return "custom";
+    return fallback;
+}
+export function resolveSearchEngineTemplate(engine, customTemplate) {
+    const normalizedTemplate = String(customTemplate ?? "").trim();
+    if (normalizedTemplate) {
+        return {
+            engine: "custom",
+            template: normalizedTemplate,
+        };
+    }
+    if (engine !== "custom") {
+        return {
+            engine,
+            template: SEARCH_ENGINE_PRESETS[engine],
+        };
+    }
+    return {
+        engine: "duckduckgo",
+        template: SEARCH_ENGINE_PRESETS.duckduckgo,
+    };
+}
+function buildSearchUrlFromTemplate(template, query) {
+    const encodedQuery = encodeURIComponent(query);
+    if (template.includes("{query}")) {
+        return template.replace(/\{query\}/g, encodedQuery);
+    }
+    if (template.includes("%s")) {
+        return template.replace(/%s/g, encodedQuery);
+    }
+    try {
+        const url = new URL(template);
+        url.searchParams.set("q", query);
+        return url.toString();
+    }
+    catch {
+        return `${template}${encodedQuery}`;
+    }
+}
+export async function parseDuckDuckGoResults(html, options = {}) {
+    const limit = options.limit ?? 40;
+    const { parseHTML } = (await import("linkedom"));
+    const { document } = parseHTML(html || "");
+    const results = [];
+    const seen = new Set();
+    const resultNodes = Array.from((document.querySelectorAll?.call(document, ".result") || []));
+    for (const node of resultNodes) {
+        const querySelector = node.querySelector;
+        if (!querySelector)
+            continue;
+        const link = querySelector.call(node, "a.result__a") ||
+            querySelector.call(node, "h2 a") ||
+            querySelector.call(node, "a[href]");
+        if (!link)
+            continue;
+        const getAttribute = link.getAttribute;
+        const href = unwrapDuckDuckGoHref((getAttribute?.call(link, "href") || ""));
+        let normalized;
+        try {
+            normalized = normalizeUrl(href);
+        }
+        catch {
+            continue;
+        }
+        const hostname = getHostname(normalized);
+        if (!isValidPublicHostname(hostname))
+            continue;
+        if (isTrackingOrAdUrl(normalized))
+            continue;
+        if (seen.has(normalized))
+            continue;
+        seen.add(normalized);
+        const textContent = link.textContent;
+        const innerHTML = link.innerHTML;
+        const title = normalizeWhitespace(textContent || stripHtml(innerHTML || ""));
+        const snippetNode = querySelector.call(node, ".result__snippet") ||
+            querySelector.call(node, ".result-snippet") ||
+            querySelector.call(node, ".result__extras");
+        const snippet = normalizeWhitespace(String(snippetNode?.textContent || ""));
+        results.push({
+            rank: results.length + 1,
+            url: normalized,
+            title,
+            snippet,
+            domain: hostname,
+        });
+        if (results.length >= limit)
+            break;
+    }
+    if (results.length > 0) {
+        return results;
+    }
+    const fallback = [];
+    const seenFallback = new Set();
+    const regex = /<a[^>]*href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi;
+    const htmlInput = html || "";
+    let match = regex.exec(htmlInput);
+    while (match !== null) {
+        const href = unwrapDuckDuckGoHref(match[1]);
+        let normalized;
+        try {
+            normalized = normalizeUrl(href);
+        }
+        catch {
+            match = regex.exec(htmlInput);
+            continue;
+        }
+        const hostname = getHostname(normalized);
+        if (!isValidPublicHostname(hostname)) {
+            match = regex.exec(htmlInput);
+            continue;
+        }
+        if (isTrackingOrAdUrl(normalized)) {
+            match = regex.exec(htmlInput);
+            continue;
+        }
+        if (seenFallback.has(normalized)) {
+            match = regex.exec(htmlInput);
+            continue;
+        }
+        seenFallback.add(normalized);
+        const title = stripHtml(match[2]);
+        if (!title) {
+            match = regex.exec(htmlInput);
+            continue;
+        }
+        fallback.push({
+            rank: fallback.length + 1,
+            url: normalized,
+            title,
+            snippet: "",
+            domain: hostname,
+        });
+        if (fallback.length >= limit)
+            break;
+        match = regex.exec(htmlInput);
+    }
+    return fallback;
+}
+export async function discoverSearchResults(query, options = {}) {
+    const startedAt = Date.now();
+    const numResults = clampInteger(options.numResults, 12, { min: 1, max: 100 });
+    const category = String(options.category || "")
+        .trim()
+        .toLowerCase();
+    const includeDomains = parseStringList(options.includeDomains)
+        .map((value) => normalizeDomainFilter(value))
+        .filter(Boolean);
+    const excludeDomains = parseStringList(options.excludeDomains)
+        .map((value) => normalizeDomainFilter(value))
+        .filter(Boolean);
+    const includeText = parseStringList(options.includeText).map((value) => value.toLowerCase());
+    const excludeText = parseStringList(options.excludeText).map((value) => value.toLowerCase());
+    const profile = categoryProfile(category);
+    const mergedIncludeDomains = Array.from(new Set([...includeDomains, ...profile.includeDomains.map((value) => normalizeDomainFilter(value))])).filter(Boolean);
+    const mergedIncludeText = Array.from(new Set([...includeText, ...profile.includeText.map((value) => value.toLowerCase())])).filter(Boolean);
+    const queryHint = String(profile.queryHint || "").trim();
+    const effectiveQuery = queryHint ? `${query} ${queryHint}` : query;
+    const requestedEngine = normalizeSearchEngine(options.searchEngine, "duckduckgo");
+    const requestedSearch = resolveSearchEngineTemplate(requestedEngine, options.searchEngineUrlTemplate);
+    const fallbackSearch = requestedSearch.engine === "duckduckgo"
+        ? null
+        : {
+            engine: "duckduckgo",
+            template: SEARCH_ENGINE_PRESETS.duckduckgo,
+        };
+    const attempts = fallbackSearch ? [requestedSearch, fallbackSearch] : [requestedSearch];
+    let activeSearch = requestedSearch;
+    let searchUrl = "";
+    let results = [];
+    let lastError = null;
+    for (const attempt of attempts) {
+        const candidateSearchUrl = buildSearchUrlFromTemplate(attempt.template, effectiveQuery);
+        try {
+            const response = await fetch(candidateSearchUrl, {
+                method: "GET",
+                headers: {
+                    "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36",
+                    "accept-language": "en-US,en;q=0.9",
+                },
+            });
+            if (!response.ok) {
+                const text = await response.text();
+                throw new Error(`Search discovery failed (HTTP ${response.status}): ${text.slice(0, 300)}`);
+            }
+            const html = await response.text();
+            const parsedResults = await parseDuckDuckGoResults(html, { limit: Math.max(numResults * 4, 24) });
+            const minAcceptableResults = Math.max(1, Math.min(2, numResults));
+            if (parsedResults.length < minAcceptableResults && attempt.engine !== "duckduckgo" && fallbackSearch) {
+                continue;
+            }
+            activeSearch = attempt;
+            searchUrl = candidateSearchUrl;
+            results = parsedResults;
+            break;
+        }
+        catch (error) {
+            lastError = error;
+            if (attempt.engine !== "duckduckgo" && fallbackSearch) {
+                continue;
+            }
+            throw error;
+        }
+    }
+    if (!searchUrl) {
+        if (lastError instanceof Error)
+            throw lastError;
+        if (lastError !== null && lastError !== undefined)
+            throw new Error(String(lastError));
+        throw new Error("Search discovery failed: no search URL could be resolved.");
+    }
+    if (category === "pdf") {
+        results = results.filter((result) => String(result.url || "")
+            .toLowerCase()
+            .includes(".pdf"));
+    }
+    if (mergedIncludeDomains.length > 0) {
+        results = results.filter((result) => domainMatches(String(result.domain || ""), mergedIncludeDomains));
+    }
+    if (excludeDomains.length > 0) {
+        results = results.filter((result) => !domainMatches(String(result.domain || ""), excludeDomains));
+    }
+    if (mergedIncludeText.length > 0) {
+        results = results.filter((result) => {
+            const haystack = `${result.title || ""} ${result.snippet || ""} ${result.url || ""}`.toLowerCase();
+            return mergedIncludeText.every((term) => haystack.includes(term));
+        });
+    }
+    if (excludeText.length > 0) {
+        results = results.filter((result) => {
+            const haystack = `${result.title || ""} ${result.snippet || ""} ${result.url || ""}`.toLowerCase();
+            return excludeText.every((term) => !haystack.includes(term));
+        });
+    }
+    results = results.slice(0, numResults).map((result, index) => ({ ...result, rank: index + 1 }));
+    return {
+        query,
+        effectiveQuery,
+        category: category || null,
+        searchEngine: activeSearch.engine,
+        searchEngineRequested: requestedSearch.engine,
+        searchUrlTemplate: activeSearch.template,
+        searchUrl,
+        results,
+        durationMs: Date.now() - startedAt,
+        filters: {
+            includeDomains: mergedIncludeDomains,
+            excludeDomains,
+            includeText: mergedIncludeText,
+            excludeText,
+        },
+    };
+}
+//# sourceMappingURL=search-discovery.js.map

package/dist/engine/search-discovery.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"search-discovery.js","sourceRoot":"","sources":["../../src/engine/search-discovery.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,EACf,YAAY,EACZ,aAAa,EACb,WAAW,EACX,iBAAiB,EACjB,qBAAqB,EACrB,qBAAqB,EACrB,mBAAmB,EACnB,eAAe,EACf,SAAS,EACT,oBAAoB,GACrB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAG9C,MAAM,qBAAqB,GAAoE;IAC7F,UAAU,EAAE,wCAAwC;IACpD,IAAI,EAAE,uCAAuC;IAC7C,MAAM,EAAE,yCAAyC;IACjD,KAAK,EAAE,2CAA2C;CACnD,CAAC;AAEF,SAAS,qBAAqB,CAAC,KAAc,EAAE,WAA+B,YAAY;IACxF,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;SACnC,IAAI,EAAE;SACN,WAAW,EAAE,CAAC;IAEjB,IAAI,UAAU,KAAK,YAAY;QAAE,OAAO,YAAY,CAAC;IACrD,IAAI,UAAU,KAAK,MAAM;QAAE,OAAO,MAAM,CAAC;IACzC,IAAI,UAAU,KAAK,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAC7C,IAAI,UAAU,KAAK,OAAO;QAAE,OAAO,OAAO,CAAC;IAC3C,IAAI,UAAU,KAAK,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAE7C,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,2BAA2B,CACzC,MAA0B,EAC1B,cAAuB;IAEvB,MAAM,kBAAkB,GAAG,MAAM,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAE/D,IAAI,kBAAkB,EAAE,CAAC;QACvB,OAAO;YACL,MAAM,EAAE,QAAQ;YAChB,QAAQ,EAAE,kBAAkB;SAC7B,CAAC;IACJ,CAAC;IAED,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;QACxB,OAAO;YACL,MAAM;YACN,QAAQ,EAAE,qBAAqB,CAAC,MAAM,CAAC;SACxC,CAAC;IACJ,CAAC;IAED,OAAO;QACL,MAAM,EAAE,YAAY;QACpB,QAAQ,EAAE,qBAAqB,CAAC,UAAU;KAC3C,CAAC;AACJ,CAAC;AAED,SAAS,0BAA0B,CAAC,QAAgB,EAAE,KAAa;IACjE,MAAM,YAAY,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAE/C,IAAI,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QACjC,OAAO,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;IACtD,CAAC;IAED,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAC5B,OAAO,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;IAC/C,CAAC;IAED,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC9B,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACjC,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,QAAQ,GAAG,YAAY,EAAE,CAAC;IACtC,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,IAAY,EACZ,UAA8B,EAAE;IAEhC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;IAClC,MAAM,EAAE,SAAS,EAAE,GAAG,CAAC,MAAM,MAAM,CAAC,UAAU,CAAC,CAE9C,CAAC;IAEF,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,IAAI,EAAE,CAA0C,CAAC;IAEpF,MAAM,OAAO,GAAmC,EAAE,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAC5B,CAAE,QAAQ,CAAC,gBAAoD,EAAE,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,IAAI,EAAE,CAAc,CAC7E,CAAC;IAEpC,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC/B,MAAM,aAAa,GAAG,IAAI,CAAC,aAAmF,CAAC;QAC/G,IAAI,CAAC,aAAa;YAAE,SAAS;QAE7B,MAAM,IAAI,GACR,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,aAAa,CAAC;YACvC,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC;YAChC,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QAEtC,IAAI,CAAC,IAAI;YAAE,SAAS;QAEpB,MAAM,YAAY,GAAG,IAAI,CAAC,YAA6D,CAAC;QACxF,MAAM,IAAI,GAAG,oBAAoB,CAAC,CAAC,YAAY,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAW,CAAC,CAAC;QACtF,IAAI,UAAkB,CAAC;QACvB,IAAI,CAAC;YACH,UAAU,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QAClC,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QAED,MAAM,QAAQ,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC;QACzC,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC;YAAE,SAAS;QAC/C,IAAI,iBAAiB,CAAC,UAAU,CAAC;YAAE,SAAS;QAC5C,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC;YAAE,SAAS;QACnC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAErB,MAAM,WAAW,GAAG,IAAI,CAAC,WAAiC,CAAC;QAC3D,MAAM,SAAS,GAAG,IAAI,CAAC,SAA+B,CAAC;QACvD,MAAM,KAAK,GAAG,mBAAmB,CAAC,WAAW,IAAI,SAAS,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC;QAE7E,MAAM,WAAW,GACf,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,kBAAkB,CAAC;YAC5C,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,iBAAiB,CAAC;YAC3C,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;QAC9C,MAAM,OAAO,GAAG,mBAAmB,CAAC,MAAM,CAAC,WAAW,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC,CAAC;QAE5E,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC;YACxB,GAAG,EAAE,UAAU;YACf,KAAK;YACL,OAAO;YACP,MAAM,EAAE,QAAQ;SACjB,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK;YAAE,MAAM;IACrC,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,MAAM,QAAQ,GAAmC,EAAE,CAAC;IACpD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;IACvC,MAAM,KAAK,GAAG,qDAAqD,CAAC;IAEpE,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,IAAI,KAAK,GAA2B,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC1D,OAAO,KAAK,KAAK,IAAI,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5C,IAAI,UAAkB,CAAC;QACvB,IAAI,CAAC;YACH,UAAU,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QAClC,CAAC;QAAC,MAAM,CAAC;YACP,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC9B,SAAS;QACX,CAAC;QAED,MAAM,QAAQ,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC;QACzC,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,EAAE,CAAC;YACrC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC9B,SAAS;QACX,CAAC;QACD,IAAI,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC;YAClC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC9B,SAAS;QACX,CAAC;QACD,IAAI,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;YACjC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC9B,SAAS;QACX,CAAC;QACD,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAE7B,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC9B,SAAS;QACX,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC;YACZ,IAAI,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC;YACzB,GAAG,EAAE,UAAU;YACf,KAAK;YACL,OAAO,EAAE,EAAE;YACX,MAAM,EAAE,QAAQ;SACjB,CAAC,CAAC;QAEH,IAAI,QAAQ,CAAC,MAAM,IAAI,KAAK;YAAE,MAAM;QACpC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAChC,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,KAAa,EACb,UAAmC,EAAE;IAErC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,MAAM,UAAU,GAAG,YAAY,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;IAC9E,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;SAC5C,IAAI,EAAE;SACN,WAAW,EAAE,CAAC;IAEjB,MAAM,cAAc,GAAG,eAAe,CAAC,OAAO,CAAC,cAAc,CAAC;SAC3D,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,qBAAqB,CAAC,KAAK,CAAC,CAAC;SAC5C,MAAM,CAAC,OAAO,CAAC,CAAC;IACnB,MAAM,cAAc,GAAG,eAAe,CAAC,OAAO,CAAC,cAAc,CAAC;SAC3D,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,qBAAqB,CAAC,KAAK,CAAC,CAAC;SAC5C,MAAM,CAAC,OAAO,CAAC,CAAC;IAEnB,MAAM,WAAW,GAAG,eAAe,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;IAC7F,MAAM,WAAW,GAAG,eAAe,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;IAE7F,MAAM,OAAO,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;IAE1C,MAAM,oBAAoB,GAAG,KAAK,CAAC,IAAI,CACrC,IAAI,GAAG,CAAC,CAAC,GAAG,cAAc,EAAE,GAAG,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,qBAAqB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CACrG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAElB,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAClC,IAAI,GAAG,CAAC,CAAC,GAAG,WAAW,EAAE,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CACtF,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAElB,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IACzD,MAAM,cAAc,GAAG,SAAS,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,SAAS,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;IAEnE,MAAM,eAAe,GAAG,qBAAqB,CAAC,OAAO,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;IAClF,MAAM,eAAe,GAAG,2BAA2B,CAAC,eAAe,EAAE,OAAO,CAAC,uBAAuB,CAAC,CAAC;IACtG,MAAM,cAAc,GAClB,eAAe,CAAC,MAAM,KAAK,YAAY;QACrC,CAAC,CAAC,IAAI;QACN,CAAC,CAAC;YACE,MAAM,EAAE,YAAqB;YAC7B,QAAQ,EAAE,qBAAqB,CAAC,UAAU;SAC3C,CAAC;IAER,MAAM,QAAQ,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC,eAAe,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;IAExF,IAAI,YAAY,GAAG,eAAe,CAAC;IACnC,IAAI,SAAS,GAAG,EAAE,CAAC;IACnB,IAAI,OAAO,GAAmC,EAAE,CAAC;IACjD,IAAI,SAAS,GAAY,IAAI,CAAC;IAE9B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,kBAAkB,GAAG,0BAA0B,CAAC,OAAO,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;QAExF,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,kBAAkB,EAAE;gBAC/C,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE;oBACP,YAAY,EACV,mGAAmG;oBACrG,iBAAiB,EAAE,gBAAgB;iBACpC;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACnC,MAAM,IAAI,KAAK,CAAC,iCAAiC,QAAQ,CAAC,MAAM,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;YAC9F,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,MAAM,aAAa,GAAG,MAAM,sBAAsB,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;YAClG,MAAM,oBAAoB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC;YAElE,IAAI,aAAa,CAAC,MAAM,GAAG,oBAAoB,IAAI,OAAO,CAAC,MAAM,KAAK,YAAY,IAAI,cAAc,EAAE,CAAC;gBACrG,SAAS;YACX,CAAC;YAED,YAAY,GAAG,OAAO,CAAC;YACvB,SAAS,GAAG,kBAAkB,CAAC;YAC/B,OAAO,GAAG,aAAa,CAAC;YACxB,MAAM;QACR,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,GAAG,KAAK,CAAC;YAElB,IAAI,OAAO,CAAC,MAAM,KAAK,YAAY,IAAI,cAAc,EAAE,CAAC;gBACtD,SAAS;YACX,CAAC;YAED,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,IAAI,SAAS,YAAY,KAAK;YAAE,MAAM,SAAS,CAAC;QAChD,IAAI,SAAS,KAAK,IAAI,IAAI,SAAS,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;QACtF,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;IAC/E,CAAC;IAED,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;QACvB,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAClC,MAAM,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC;aACrB,WAAW,EAAE;aACb,QAAQ,CAAC,MAAM,CAAC,CACpB,CAAC;IACJ,CAAC;IAED,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpC,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,oBAAoB,CAAC,CAAC,CAAC;IACzG,CAAC;IAED,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,cAAc,CAAC,CAAC,CAAC;IACpG,CAAC;IAED,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE;YAClC,MAAM,QAAQ,GAAG,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,IAAI,MAAM,CAAC,OAAO,IAAI,EAAE,IAAI,MAAM,CAAC,GAAG,IAAI,EAAE,EAAE,CAAC,WAAW,EAAE,CAAC;YACnG,OAAO,iBAAiB,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QACpE,CAAC,CAAC,CAAC;IACL,CAAC;IAED,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE;YAClC,MAAM,QAAQ,GAAG,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,IAAI,MAAM,CAAC,OAAO,IAAI,EAAE,IAAI,MAAM,CAAC,GAAG,IAAI,EAAE,EAAE,CAAC,WAAW,EAAE,CAAC;YACnG,OAAO,WAAW,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,MAAM,EAAE,IAAI,EAAE,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IAEhG,OAAO;QACL,KAAK;QACL,cAAc;QACd,QAAQ,EAAE,QAAQ,IAAI,IAAI;QAC1B,YAAY,EAAE,YAAY,CAAC,MAAM;QACjC,qBAAqB,EAAE,eAAe,CAAC,MAAM;QAC7C,iBAAiB,EAAE,YAAY,CAAC,QAAQ;QACxC,SAAS;QACT,OAAO;QACP,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;QAClC,OAAO,EAAE;YACP,cAAc,EAAE,oBAAoB;YACpC,cAAc;YACd,WAAW,EAAE,iBAAiB;YAC9B,WAAW;SACZ;KACF,CAAC;AACJ,CAAC"}

package/dist/engine/url-utils.js

@@ -0,0 +1,72 @@
+const URL_REGEX = /https?:\/\/[^\s)\]>]+/g;
+export function sanitizeUrlCandidate(input) {
+    let value = String(input ?? "").trim();
+    if (!value)
+        return "";
+    if (value.startsWith("<") && value.endsWith(">")) {
+        value = value.slice(1, -1).trim();
+    }
+    value = value.replace(/^[\s("'`[]+/, "");
+    for (let i = 0; i < 3; i += 1) {
+        const next = value
+            .replace(/(?:\*{1,3}|_{1,3}|`{1,3})+$/g, "")
+            .replace(/[>\]"'`]+$/g, "")
+            .replace(/[.,;:!?]+$/g, "")
+            .trim();
+        if (next === value)
+            break;
+        value = next;
+    }
+    return value;
+}
+export function normalizeUrl(input) {
+    const raw = sanitizeUrlCandidate(input);
+    if (!raw)
+        throw new Error("URL is required");
+    const withProtocol = /^https?:\/\//i.test(raw) ? raw : `https://${raw}`;
+    const url = new URL(withProtocol);
+    if (!["http:", "https:"].includes(url.protocol)) {
+        throw new Error(`Unsupported URL protocol: ${url.protocol}`);
+    }
+    return url.toString();
+}
+export function normalizeUrlForDedupe(input) {
+    const normalized = normalizeUrl(input);
+    const url = new URL(normalized);
+    url.hash = "";
+    if ((url.protocol === "http:" && url.port === "80") || (url.protocol === "https:" && url.port === "443")) {
+        url.port = "";
+    }
+    if (url.pathname !== "/") {
+        const trimmedPath = url.pathname.replace(/\/+$/g, "");
+        url.pathname = trimmedPath || "/";
+    }
+    return url.toString();
+}
+export function normalizeUniqueUrls(values) {
+    const urls = [];
+    const seen = new Set();
+    for (const value of values) {
+        try {
+            const normalized = normalizeUrl(value);
+            const dedupeKey = normalizeUrlForDedupe(normalized);
+            if (seen.has(dedupeKey))
+                continue;
+            seen.add(dedupeKey);
+            urls.push(normalized);
+        }
+        catch {
+            // ignore malformed URL-like fragments
+        }
+    }
+    return urls;
+}
+export function extractSeedUrls(text) {
+    const matches = String(text ?? "").match(URL_REGEX) ?? [];
+    return normalizeUniqueUrls(matches);
+}
+export function extractCitationUrls(text) {
+    const matches = String(text ?? "").match(URL_REGEX) ?? [];
+    return normalizeUniqueUrls(matches);
+}
+//# sourceMappingURL=url-utils.js.map

package/dist/engine/url-utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"url-utils.js","sourceRoot":"","sources":["../../src/engine/url-utils.ts"],"names":[],"mappings":"AAAA,MAAM,SAAS,GAAG,wBAAwB,CAAC;AAE3C,MAAM,UAAU,oBAAoB,CAAC,KAAc;IACjD,IAAI,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IACvC,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,CAAC;IAEtB,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACjD,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACpC,CAAC;IAED,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;IAEzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,GAAG,KAAK;aACf,OAAO,CAAC,8BAA8B,EAAE,EAAE,CAAC;aAC3C,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC;aAC1B,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC;aAC1B,IAAI,EAAE,CAAC;QAEV,IAAI,IAAI,KAAK,KAAK;YAAE,MAAM;QAC1B,KAAK,GAAG,IAAI,CAAC;IACf,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAc;IACzC,MAAM,GAAG,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;IACxC,IAAI,CAAC,GAAG;QAAE,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;IAE7C,MAAM,YAAY,GAAG,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,GAAG,EAAE,CAAC;IACxE,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC;IAElC,IAAI,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC/D,CAAC;IAED,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;AACxB,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,KAAc;IAClD,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IACvC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC;IAEhC,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC;IAEd,IAAI,CAAC,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;QACzG,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC;IAChB,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,KAAK,GAAG,EAAE,CAAC;QACzB,MAAM,WAAW,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACtD,GAAG,CAAC,QAAQ,GAAG,WAAW,IAAI,GAAG,CAAC;IACpC,CAAC;IAED,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;AACxB,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,MAAyB;IAC3D,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YACvC,MAAM,SAAS,GAAG,qBAAqB,CAAC,UAAU,CAAC,CAAC;YACpD,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC;gBAAE,SAAS;YAClC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YACpB,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACxB,CAAC;QAAC,MAAM,CAAC;YACP,sCAAsC;QACxC,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAa;IAC3C,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;IAC1D,OAAO,mBAAmB,CAAC,OAAO,CAAC,CAAC;AACtC,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAAa;IAC/C,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;IAC1D,OAAO,mBAAmB,CAAC,OAAO,CAAC,CAAC;AACtC,CAAC"}