@debriefer/sources 1.0.1

package/dist/obituary/find-a-grave.js

@@ -0,0 +1,173 @@
+/**
+ * Find a Grave obituary source.
+ *
+ * Searches findagrave.com for memorial pages by name, extracts biography
+ * content via Readability (with regex fallback), and returns sanitized text.
+ *
+ * Find a Grave is user-generated content — anyone can create or edit memorials.
+ * Reliability tier is UNRELIABLE_UGC (0.35) per Wikipedia RSP guidelines.
+ */
+import { BaseResearchSource, ReliabilityTier, } from "@debriefer/core";
+import { fetchPage } from "../shared/fetch-page.js";
+import { extractArticleContent } from "../shared/readability-extract.js";
+import { sanitizeSourceText } from "../shared/sanitize-text.js";
+// ============================================================================
+// Constants
+// ============================================================================
+const SEARCH_BASE_URL = "https://www.findagrave.com/memorial/search";
+const MEMORIAL_URL_PATTERN = /\/memorial\/(\d+)\//g;
+const MIN_BIO_LENGTH = 100;
+const BIO_REGEX = /<div[^>]*id="bio"[^>]*>([\s\S]*?)<\/div>/i;
+// ============================================================================
+// Source Implementation
+// ============================================================================
+/**
+ * Find a Grave source for obituary / memorial content.
+ *
+ * Pipeline:
+ * 1. Search findagrave.com by first/last name
+ * 2. Parse memorial URLs from search results HTML
+ * 3. Filter for URLs containing the subject's name (normalized)
+ * 4. Fetch the memorial page via fetchPage (with archive fallback)
+ * 5. Extract bio via Readability, falling back to regex
+ * 6. Sanitize text and return if long enough
+ */
+export class FindAGraveSource extends BaseResearchSource {
+    name = "Find a Grave";
+    type = "find-a-grave";
+    reliabilityTier = ReliabilityTier.UNRELIABLE_UGC;
+    domain = "www.findagrave.com";
+    isFree = true;
+    estimatedCostPerQuery = 0;
+    constructor(options = {}) {
+        super({ rateLimitMs: 2000, ...options });
+    }
+    async fetchResult(subject, signal) {
+        // Step 1: Split name into first/last
+        const nameParts = subject.name.trim().split(/\s+/);
+        const firstName = nameParts[0] ?? "";
+        const lastName = nameParts.slice(1).join(" ") || "";
+        // Step 2: Search for memorials
+        const searchUrl = `${SEARCH_BASE_URL}?firstname=${encodeURIComponent(firstName)}&lastname=${encodeURIComponent(lastName)}&orderby=r`;
+        const searchPage = await fetchPage({
+            url: searchUrl,
+            signal,
+            archiveFallback: false,
+        });
+        if (searchPage.fetchMethod === "none" || !searchPage.content) {
+            return null;
+        }
+        const searchHtml = searchPage.content;
+        // Step 3: Extract memorial URLs
+        const memorialUrls = [];
+        let match;
+        // Reset lastIndex before use
+        MEMORIAL_URL_PATTERN.lastIndex = 0;
+        while ((match = MEMORIAL_URL_PATTERN.exec(searchHtml)) !== null) {
+            const fullMatch = match[0];
+            const anchorIndex = match.index;
+            // Build the full memorial URL from the path found in the HTML
+            // Search result links look like: /memorial/12345/john-wayne
+            const startQuoteIdx = searchHtml.lastIndexOf('"', anchorIndex);
+            const endQuoteIdx = searchHtml.indexOf('"', anchorIndex + fullMatch.length);
+            if (startQuoteIdx === -1 || endQuoteIdx === -1) {
+                continue;
+            }
+            const startIdx = startQuoteIdx + 1;
+            const endIdx = endQuoteIdx;
+            const path = searchHtml.slice(startIdx, endIdx);
+            if (path.startsWith("/memorial/")) {
+                const url = `https://www.findagrave.com${path}`;
+                if (!memorialUrls.includes(url)) {
+                    memorialUrls.push(url);
+                }
+            }
+        }
+        if (memorialUrls.length === 0) {
+            return null;
+        }
+        // Step 4: Prefer exact slug matches for the subject's normalized name
+        const normalizedName = subject.name.toLowerCase().replace(/\s+/g, "-");
+        // First: look for URLs where the slug segment exactly matches normalizedName.
+        const exactSlugMatches = memorialUrls.filter((url) => {
+            try {
+                const { pathname } = new URL(url);
+                const segments = pathname.split("/").filter((s) => s.length > 0);
+                // Find a Grave memorial path: /memorial/{id}/{slug}
+                const slug = segments[2]?.toLowerCase() ?? "";
+                return slug === normalizedName;
+            }
+            catch {
+                return false;
+            }
+        });
+        const matchingUrls = exactSlugMatches.length > 0
+            ? exactSlugMatches
+            : memorialUrls.filter((url) => url.toLowerCase().includes(normalizedName));
+        if (matchingUrls.length === 0) {
+            return null;
+        }
+        const memorialUrl = matchingUrls[0];
+        // Step 5: Fetch the memorial page
+        await this.rateLimiter?.acquire(this.domain, this.options.rateLimitMs);
+        const page = await fetchPage({ url: memorialUrl, signal });
+        if (page.fetchMethod === "none" || !page.content) {
+            return null;
+        }
+        // Step 6: Extract bio content
+        const actualUrl = page.url || memorialUrl;
+        let bioText = null;
+        // Try Readability first
+        const extracted = extractArticleContent(page.content, actualUrl);
+        if (extracted && extracted.text.length >= MIN_BIO_LENGTH) {
+            bioText = extracted.text;
+        }
+        // Fall back to regex if Readability didn't get enough content
+        if (!bioText) {
+            const bioMatch = BIO_REGEX.exec(page.content);
+            if (bioMatch && bioMatch[1]) {
+                // Strip HTML tags from the regex-extracted content
+                const stripped = bioMatch[1]
+                    .replace(/<[^>]*>/g, " ")
+                    .replace(/\s+/g, " ")
+                    .trim();
+                if (stripped.length >= MIN_BIO_LENGTH) {
+                    bioText = stripped;
+                }
+            }
+        }
+        if (!bioText) {
+            return null;
+        }
+        // Step 7: Sanitize and return
+        const text = sanitizeSourceText(bioText);
+        if (text.length < MIN_BIO_LENGTH) {
+            return null;
+        }
+        return {
+            text,
+            confidence: -1,
+            costUsd: 0,
+            url: actualUrl,
+            publication: "Find a Grave",
+            metadata: {
+                memorialUrl: actualUrl,
+            },
+        };
+    }
+}
+// ============================================================================
+// Factory Function
+// ============================================================================
+/**
+ * Create a Find a Grave source instance.
+ *
+ * @example
+ * ```typescript
+ * const source = findAGrave()
+ * ```
+ */
+export function findAGrave(options) {
+    return new FindAGraveSource(options);
+}
+//# sourceMappingURL=find-a-grave.js.map

package/dist/obituary/find-a-grave.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"find-a-grave.js","sourceRoot":"","sources":["../../src/obituary/find-a-grave.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EACL,kBAAkB,EAClB,eAAe,GAIhB,MAAM,iBAAiB,CAAA;AACxB,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAA;AACnD,OAAO,EAAE,qBAAqB,EAAE,MAAM,kCAAkC,CAAA;AACxE,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAE/D,+EAA+E;AAC/E,YAAY;AACZ,+EAA+E;AAE/E,MAAM,eAAe,GAAG,4CAA4C,CAAA;AACpE,MAAM,oBAAoB,GAAG,sBAAsB,CAAA;AACnD,MAAM,cAAc,GAAG,GAAG,CAAA;AAC1B,MAAM,SAAS,GAAG,2CAA2C,CAAA;AAS7D,+EAA+E;AAC/E,wBAAwB;AACxB,+EAA+E;AAE/E;;;;;;;;;;GAUG;AACH,MAAM,OAAO,gBAAiB,SAAQ,kBAAmC;IAC9D,IAAI,GAAG,cAAc,CAAA;IACrB,IAAI,GAAG,cAAc,CAAA;IACrB,eAAe,GAAG,eAAe,CAAC,cAAc,CAAA;IAChD,MAAM,GAAG,oBAAoB,CAAA;IAC7B,MAAM,GAAG,IAAI,CAAA;IACb,qBAAqB,GAAG,CAAC,CAAA;IAElC,YAAY,UAA6B,EAAE;QACzC,KAAK,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAA;IAC1C,CAAC;IAES,KAAK,CAAC,WAAW,CACzB,OAAwB,EACxB,MAAmB;QAEnB,qCAAqC;QACrC,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;QAClD,MAAM,SAAS,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;QACpC,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAA;QAEnD,+BAA+B;QAC/B,MAAM,SAAS,GAAG,GAAG,eAAe,cAAc,kBAAkB,CAAC,SAAS,CAAC,aAAa,kBAAkB,CAAC,QAAQ,CAAC,YAAY,CAAA;QAEpI,MAAM,UAAU,GAAG,MAAM,SAAS,CAAC;YACjC,GAAG,EAAE,SAAS;YACd,MAAM;YACN,eAAe,EAAE,KAAK;SACvB,CAAC,CAAA;QAEF,IAAI,UAAU,CAAC,WAAW,KAAK,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;YAC7D,OAAO,IAAI,CAAA;QACb,CAAC;QAED,MAAM,UAAU,GAAG,UAAU,CAAC,OAAO,CAAA;QAErC,gCAAgC;QAChC,MAAM,YAAY,GAAa,EAAE,CAAA;QACjC,IAAI,KAA6B,CAAA;QACjC,6BAA6B;QAC7B,oBAAoB,CAAC,SAAS,GAAG,CAAC,CAAA;QAClC,OAAO,CAAC,KAAK,GAAG,oBAAoB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAChE,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;YAC1B,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK,CAAA;YAC/B,8DAA8D;YAC9D,4DAA4D;YAC5D,MAAM,aAAa,GAAG,UAAU,CAAC,WAAW,CAAC,GAAG,EAAE,WAAW,CAAC,CAAA;YAC9D,MAAM,WAAW,GAAG,UAAU,CAAC,OAAO,CAAC,GAAG,EAAE,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,CAAA;YAC3E,IAAI,aAAa,KAAK,CAAC,CAAC,IAAI,WAAW,KAAK,CAAC,CAAC,EAAE,CAAC;gBAC/C,SAAQ;YACV,CAAC;YACD,MAAM,QAAQ,GAAG,aAAa,GAAG,CAAC,CAAA;YAClC,MAAM,MAAM,GAAG,WAAW,CAAA;YAC1B,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;YAE/C,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;gBAClC,MAAM,GAAG,GAAG,6BAA6B,IAAI,EAAE,CAAA;gBAC/C,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBAChC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBACxB,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAA;QACb,CAAC;QAED,sEAAsE;QACtE,MAAM,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;QAEtE,8EAA8E;QAC9E,MAAM,gBAAgB,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE;YACnD,IAAI,CAAC;gBACH,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAA;gBACjC,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;gBAChE,oDAAoD;gBACpD,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAA;gBAC7C,OAAO,IAAI,KAAK,cAAc,CAAA;YAChC,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,KAAK,CAAA;YACd,CAAC;QACH,CAAC,CAAC,CAAA;QAEF,MAAM,YAAY,GAChB,gBAAgB,CAAC,MAAM,GAAG,CAAC;YACzB,CAAC,CAAC,gBAAgB;YAClB,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAA;QAE9E,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAA;QACb,CAAC;QAED,MAAM,WAAW,GAAG,YAAY,CAAC,CAAC,CAAC,CAAA;QAEnC,kCAAkC;QAClC,MAAM,IAAI,CAAC,WAAW,EAAE,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,WAAY,CAAC,CAAA;QACvE,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,CAAA;QAE1D,IAAI,IAAI,CAAC,WAAW,KAAK,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACjD,OAAO,IAAI,CAAA;QACb,CAAC;QAED,8BAA8B;QAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,IAAI,WAAW,CAAA;QACzC,IAAI,OAAO,GAAkB,IAAI,CAAA;QAEjC,wBAAwB;QACxB,MAAM,SAAS,GAAG,qBAAqB,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAA;QAChE,IAAI,SAAS,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,IAAI,cAAc,EAAE,CAAC;YACzD,OAAO,GAAG,SAAS,CAAC,IAAI,CAAA;QAC1B,CAAC;QAED,8DAA8D;QAC9D,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAC7C,IAAI,QAAQ,IAAI,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC5B,mDAAmD;gBACnD,MAAM,QAAQ,GAAG,QAAQ,CAAC,CAAC,CAAC;qBACzB,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;qBACxB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;qBACpB,IAAI,EAAE,CAAA;gBACT,IAAI,QAAQ,CAAC,MAAM,IAAI,cAAc,EAAE,CAAC;oBACtC,OAAO,GAAG,QAAQ,CAAA;gBACpB,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,IAAI,CAAA;QACb,CAAC;QAED,8BAA8B;QAC9B,MAAM,IAAI,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAA;QAExC,IAAI,IAAI,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;YACjC,OAAO,IAAI,CAAA;QACb,CAAC;QAED,OAAO;YACL,IAAI;YACJ,UAAU,EAAE,CAAC,CAAC;YACd,OAAO,EAAE,CAAC;YACV,GAAG,EAAE,SAAS;YACd,WAAW,EAAE,cAAc;YAC3B,QAAQ,EAAE;gBACR,WAAW,EAAE,SAAS;aACvB;SACF,CAAA;IACH,CAAC;CACF;AAED,+EAA+E;AAC/E,mBAAmB;AACnB,+EAA+E;AAE/E;;;;;;;GAOG;AACH,MAAM,UAAU,UAAU,CAAC,OAA2B;IACpD,OAAO,IAAI,gBAAgB,CAAC,OAAO,CAAC,CAAA;AACtC,CAAC"}

package/dist/shared/duckduckgo-search.d.ts

@@ -0,0 +1,86 @@
+/**
+ * DuckDuckGo HTML search utility.
+ *
+ * Scrapes DDG's HTML endpoint (no API key required) to extract
+ * search result URLs, titles, and snippets. Includes CAPTCHA detection,
+ * DDG redirect URL cleaning, and domain-based filtering.
+ *
+ * Used by DuckDuckGoSearchSource and future news sources for
+ * `site:domain.com` style queries.
+ */
+/** Options for a DuckDuckGo HTML search. */
+export interface DuckDuckGoSearchOptions {
+    /** Search query string. */
+    query: string;
+    /** Domain to restrict results to (prepended as site: to query). */
+    domainFilter?: string;
+    /** Maximum number of results to return. Default: 10. */
+    maxResults?: number;
+    /** AbortSignal from the caller (combined with timeoutMs). */
+    signal?: AbortSignal;
+    /** Timeout in milliseconds for the fetch. Default: 15000. */
+    timeoutMs?: number;
+}
+/** A single search result extracted from DDG HTML. */
+export interface SearchResult {
+    /** Cleaned URL of the search result. */
+    url: string;
+    /** Title of the search result. */
+    title: string;
+    /** Snippet/description of the search result. */
+    snippet: string;
+}
+/**
+ * Detect whether DDG returned a CAPTCHA/bot-detection page.
+ *
+ * Checks for "anomaly-modal" (DDG's CAPTCHA container) and the
+ * "bots use DuckDuckGo too" message.
+ *
+ * @param html - Raw HTML response body from DDG
+ * @returns True if the page is a CAPTCHA challenge
+ */
+export declare function isDuckDuckGoCaptcha(html: string): boolean;
+/**
+ * Clean a DuckDuckGo result URL.
+ *
+ * DDG wraps result URLs in redirect links like:
+ *   `//duckduckgo.com/l/?uddg=ENCODED_URL&rut=...`
+ *
+ * This function:
+ * 1. Extracts the real URL from the `uddg` query parameter
+ * 2. Handles protocol-relative `//` URLs by prepending `https:`
+ * 3. Passes normal URLs through unchanged
+ *
+ * @param url - URL from a DDG search result
+ * @returns Cleaned URL pointing to the actual destination
+ */
+export declare function cleanDuckDuckGoUrl(url: string): string;
+/**
+ * Extract search results from DuckDuckGo HTML response.
+ *
+ * Parses DDG's HTML structure:
+ * - `class="result__url"` href for URLs (primary)
+ * - `class="result__a"` for titles (and fallback URLs)
+ * - `class="result__snippet"` for snippets
+ *
+ * If no `result__url` matches are found, falls back to `result__a` hrefs.
+ * Filters by domain using URL hostname parsing to prevent substring spoofing
+ * (e.g., "nytimes.com.evil.com" won't match "nytimes.com").
+ *
+ * @param html - Raw HTML response from DDG
+ * @param domainFilter - Optional domain to filter results by
+ * @returns Array of extracted search results
+ */
+export declare function extractUrlsFromDuckDuckGoHtml(html: string, domainFilter?: string): SearchResult[];
+/**
+ * Search DuckDuckGo via its HTML endpoint.
+ *
+ * Fetches `https://html.duckduckgo.com/html/?q=QUERY`, optionally
+ * prepending `site:domain` when domainFilter is set. Returns an empty
+ * array on CAPTCHA, error, or non-OK response.
+ *
+ * @param options - Search options including query, domain filter, and limits
+ * @returns Array of search results (empty on failure)
+ */
+export declare function searchDuckDuckGo(options: DuckDuckGoSearchOptions): Promise<SearchResult[]>;
+//# sourceMappingURL=duckduckgo-search.d.ts.map

package/dist/shared/duckduckgo-search.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"duckduckgo-search.d.ts","sourceRoot":"","sources":["../../src/shared/duckduckgo-search.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAQH,4CAA4C;AAC5C,MAAM,WAAW,uBAAuB;IACtC,2BAA2B;IAC3B,KAAK,EAAE,MAAM,CAAA;IACb,mEAAmE;IACnE,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,wDAAwD;IACxD,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,6DAA6D;IAC7D,MAAM,CAAC,EAAE,WAAW,CAAA;IACpB,6DAA6D;IAC7D,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAED,sDAAsD;AACtD,MAAM,WAAW,YAAY;IAC3B,wCAAwC;IACxC,GAAG,EAAE,MAAM,CAAA;IACX,kCAAkC;IAClC,KAAK,EAAE,MAAM,CAAA;IACb,gDAAgD;IAChD,OAAO,EAAE,MAAM,CAAA;CAChB;AAmBD;;;;;;;;GAQG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAGzD;AAMD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAyBtD;AAMD;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,6BAA6B,CAAC,IAAI,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,YAAY,EAAE,CAoEjG;AAqBD;;;;;;;;;GASG;AACH,wBAAsB,gBAAgB,CAAC,OAAO,EAAE,uBAAuB,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAmDhG"}

package/dist/shared/duckduckgo-search.js

@@ -0,0 +1,218 @@
+/**
+ * DuckDuckGo HTML search utility.
+ *
+ * Scrapes DDG's HTML endpoint (no API key required) to extract
+ * search result URLs, titles, and snippets. Includes CAPTCHA detection,
+ * DDG redirect URL cleaning, and domain-based filtering.
+ *
+ * Used by DuckDuckGoSearchSource and future news sources for
+ * `site:domain.com` style queries.
+ */
+import { decodeHtmlEntities } from "./html-utils.js";
+// ============================================================================
+// Constants
+// ============================================================================
+/** DDG HTML search endpoint. */
+const DDG_HTML_URL = "https://html.duckduckgo.com/html/";
+/** Default timeout for the search fetch in milliseconds. */
+const DEFAULT_TIMEOUT_MS = 15000;
+/** Default maximum number of results to return. */
+const DEFAULT_MAX_RESULTS = 10;
+// ============================================================================
+// CAPTCHA Detection
+// ============================================================================
+/**
+ * Detect whether DDG returned a CAPTCHA/bot-detection page.
+ *
+ * Checks for "anomaly-modal" (DDG's CAPTCHA container) and the
+ * "bots use DuckDuckGo too" message.
+ *
+ * @param html - Raw HTML response body from DDG
+ * @returns True if the page is a CAPTCHA challenge
+ */
+export function isDuckDuckGoCaptcha(html) {
+    const lower = html.toLowerCase();
+    return lower.includes("anomaly-modal") || lower.includes("bots use duckduckgo too");
+}
+// ============================================================================
+// URL Cleaning
+// ============================================================================
+/**
+ * Clean a DuckDuckGo result URL.
+ *
+ * DDG wraps result URLs in redirect links like:
+ *   `//duckduckgo.com/l/?uddg=ENCODED_URL&rut=...`
+ *
+ * This function:
+ * 1. Extracts the real URL from the `uddg` query parameter
+ * 2. Handles protocol-relative `//` URLs by prepending `https:`
+ * 3. Passes normal URLs through unchanged
+ *
+ * @param url - URL from a DDG search result
+ * @returns Cleaned URL pointing to the actual destination
+ */
+export function cleanDuckDuckGoUrl(url) {
+    // Handle DDG redirect URLs — only extract uddg from known DDG redirect paths
+    if (url.includes("uddg=")) {
+        const normalizedUrl = url.startsWith("//") ? `https:${url}` : url;
+        try {
+            const parsed = new URL(normalizedUrl);
+            const hostname = parsed.hostname.toLowerCase();
+            const isDDG = hostname === "duckduckgo.com" || hostname === "www.duckduckgo.com";
+            if (isDDG && parsed.pathname.startsWith("/l/")) {
+                const uddg = parsed.searchParams.get("uddg");
+                if (uddg) {
+                    return uddg;
+                }
+            }
+        }
+        catch {
+            // Fall through to other checks
+        }
+    }
+    // Handle protocol-relative URLs
+    if (url.startsWith("//")) {
+        return `https:${url}`;
+    }
+    return url;
+}
+// ============================================================================
+// HTML Extraction
+// ============================================================================
+/**
+ * Extract search results from DuckDuckGo HTML response.
+ *
+ * Parses DDG's HTML structure:
+ * - `class="result__url"` href for URLs (primary)
+ * - `class="result__a"` for titles (and fallback URLs)
+ * - `class="result__snippet"` for snippets
+ *
+ * If no `result__url` matches are found, falls back to `result__a` hrefs.
+ * Filters by domain using URL hostname parsing to prevent substring spoofing
+ * (e.g., "nytimes.com.evil.com" won't match "nytimes.com").
+ *
+ * @param html - Raw HTML response from DDG
+ * @param domainFilter - Optional domain to filter results by
+ * @returns Array of extracted search results
+ */
+export function extractUrlsFromDuckDuckGoHtml(html, domainFilter) {
+    // Extract result__url hrefs
+    const urlPattern = /class="result__url"\s+href="([^"]+)"/g;
+    const resultUrls = [];
+    let match;
+    match = urlPattern.exec(html);
+    while (match !== null) {
+        resultUrls.push(cleanDuckDuckGoUrl(decodeHtmlEntities(match[1])));
+        match = urlPattern.exec(html);
+    }
+    // Extract result__a titles and hrefs
+    const titlePattern = /class="result__a"\s+href="([^"]+)"[^>]*>([^<]*)</g;
+    const titles = [];
+    match = titlePattern.exec(html);
+    while (match !== null) {
+        titles.push({
+            href: cleanDuckDuckGoUrl(decodeHtmlEntities(match[1])),
+            title: decodeHtmlEntities(match[2]).trim(),
+        });
+        match = titlePattern.exec(html);
+    }
+    // Extract result__snippet text
+    const snippetPattern = /class="result__snippet"[^>]*>([\s\S]*?)<\/a>/g;
+    const snippets = [];
+    match = snippetPattern.exec(html);
+    while (match !== null) {
+        // Strip any inline HTML tags from snippet content
+        const rawSnippet = match[1].replace(/<[^>]+>/g, "");
+        snippets.push(decodeHtmlEntities(rawSnippet).trim());
+        match = snippetPattern.exec(html);
+    }
+    // Build results: prefer result__url, fall back to result__a hrefs
+    const useUrls = resultUrls.length > 0;
+    const primaryUrls = useUrls ? resultUrls : titles.map((t) => t.href);
+    const count = primaryUrls.length;
+    const results = [];
+    for (let i = 0; i < count; i++) {
+        const url = primaryUrls[i];
+        const title = titles[i]?.title ?? "";
+        const snippet = snippets[i] ?? "";
+        // Filter by domain using hostname parsing to prevent substring spoofing.
+        // Normalize to lowercase since URL.hostname is always lowercase.
+        if (domainFilter) {
+            const normalizedFilter = domainFilter.toLowerCase().trim();
+            try {
+                const hostname = new URL(url).hostname;
+                if (hostname !== normalizedFilter && !hostname.endsWith("." + normalizedFilter)) {
+                    continue;
+                }
+            }
+            catch {
+                // If URL can't be parsed, skip it when filtering
+                continue;
+            }
+        }
+        results.push({ url, title, snippet });
+    }
+    return results;
+}
+// ============================================================================
+// Search Function
+// ============================================================================
+/**
+ * Build the combined AbortSignal from a caller signal and a timeout.
+ *
+ * Uses `AbortSignal.any()` to combine both so that neither defeats the other.
+ */
+function buildSignal(callerSignal, timeoutMs) {
+    const timeout = timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    const timeoutSignal = AbortSignal.timeout(timeout);
+    if (callerSignal) {
+        return AbortSignal.any([callerSignal, timeoutSignal]);
+    }
+    return timeoutSignal;
+}
+/**
+ * Search DuckDuckGo via its HTML endpoint.
+ *
+ * Fetches `https://html.duckduckgo.com/html/?q=QUERY`, optionally
+ * prepending `site:domain` when domainFilter is set. Returns an empty
+ * array on CAPTCHA, error, or non-OK response.
+ *
+ * @param options - Search options including query, domain filter, and limits
+ * @returns Array of search results (empty on failure)
+ */
+export async function searchDuckDuckGo(options) {
+    const { query, domainFilter, maxResults = DEFAULT_MAX_RESULTS, signal: callerSignal, timeoutMs, } = options;
+    // Build the search query, prepending site: if domainFilter is set
+    const fullQuery = domainFilter ? `site:${domainFilter} ${query}` : query;
+    // Build the search URL
+    const searchUrl = `${DDG_HTML_URL}?q=${encodeURIComponent(fullQuery)}`;
+    // Build the abort signal combining caller signal with timeout
+    const signal = buildSignal(callerSignal, timeoutMs);
+    let response;
+    try {
+        response = await fetch(searchUrl, {
+            headers: {
+                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+            },
+            signal,
+        });
+    }
+    catch (error) {
+        // Re-throw abort/timeout so BaseResearchSource.lookup() can record telemetry
+        if (error instanceof DOMException &&
+            (error.name === "AbortError" || error.name === "TimeoutError")) {
+            throw error;
+        }
+        return [];
+    }
+    if (!response.ok) {
+        return [];
+    }
+    const html = await response.text();
+    if (isDuckDuckGoCaptcha(html)) {
+        return [];
+    }
+    const results = extractUrlsFromDuckDuckGoHtml(html, domainFilter);
+    return results.slice(0, maxResults);
+}
+//# sourceMappingURL=duckduckgo-search.js.map

package/dist/shared/duckduckgo-search.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"duckduckgo-search.js","sourceRoot":"","sources":["../../src/shared/duckduckgo-search.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AA8BpD,+EAA+E;AAC/E,YAAY;AACZ,+EAA+E;AAE/E,gCAAgC;AAChC,MAAM,YAAY,GAAG,mCAAmC,CAAA;AAExD,4DAA4D;AAC5D,MAAM,kBAAkB,GAAG,KAAK,CAAA;AAEhC,mDAAmD;AACnD,MAAM,mBAAmB,GAAG,EAAE,CAAA;AAE9B,+EAA+E;AAC/E,oBAAoB;AACpB,+EAA+E;AAE/E;;;;;;;;GAQG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAY;IAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAA;IAChC,OAAO,KAAK,CAAC,QAAQ,CAAC,eAAe,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,yBAAyB,CAAC,CAAA;AACrF,CAAC;AAED,+EAA+E;AAC/E,eAAe;AACf,+EAA+E;AAE/E;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAW;IAC5C,6EAA6E;IAC7E,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1B,MAAM,aAAa,GAAG,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAA;QACjE,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC,CAAA;YACrC,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAA;YAC9C,MAAM,KAAK,GAAG,QAAQ,KAAK,gBAAgB,IAAI,QAAQ,KAAK,oBAAoB,CAAA;YAChF,IAAI,KAAK,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC/C,MAAM,IAAI,GAAG,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;gBAC5C,IAAI,IAAI,EAAE,CAAC;oBACT,OAAO,IAAI,CAAA;gBACb,CAAC;YACH,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,+BAA+B;QACjC,CAAC;IACH,CAAC;IAED,gCAAgC;IAChC,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACzB,OAAO,SAAS,GAAG,EAAE,CAAA;IACvB,CAAC;IAED,OAAO,GAAG,CAAA;AACZ,CAAC;AAED,+EAA+E;AAC/E,kBAAkB;AAClB,+EAA+E;AAE/E;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,6BAA6B,CAAC,IAAY,EAAE,YAAqB;IAC/E,4BAA4B;IAC5B,MAAM,UAAU,GAAG,uCAAuC,CAAA;IAC1D,MAAM,UAAU,GAAa,EAAE,CAAA;IAC/B,IAAI,KAA6B,CAAA;IAEjC,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC7B,OAAO,KAAK,KAAK,IAAI,EAAE,CAAC;QACtB,UAAU,CAAC,IAAI,CAAC,kBAAkB,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QACjE,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC/B,CAAC;IAED,qCAAqC;IACrC,MAAM,YAAY,GAAG,mDAAmD,CAAA;IACxE,MAAM,MAAM,GAA2C,EAAE,CAAA;IAEzD,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC/B,OAAO,KAAK,KAAK,IAAI,EAAE,CAAC;QACtB,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,kBAAkB,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACtD,KAAK,EAAE,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE;SAC3C,CAAC,CAAA;QACF,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACjC,CAAC;IAED,+BAA+B;IAC/B,MAAM,cAAc,GAAG,+CAA+C,CAAA;IACtE,MAAM,QAAQ,GAAa,EAAE,CAAA;IAE7B,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACjC,OAAO,KAAK,KAAK,IAAI,EAAE,CAAC;QACtB,kDAAkD;QAClD,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAA;QACnD,QAAQ,CAAC,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;QACpD,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACnC,CAAC;IAED,kEAAkE;IAClE,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAA;IACrC,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;IACpE,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAA;IAEhC,MAAM,OAAO,GAAmB,EAAE,CAAA;IAElC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/B,MAAM,GAAG,GAAG,WAAW,CAAC,CAAC,CAAC,CAAA;QAC1B,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,EAAE,CAAA;QACpC,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;QAEjC,yEAAyE;QACzE,iEAAiE;QACjE,IAAI,YAAY,EAAE,CAAC;YACjB,MAAM,gBAAgB,GAAG,YAAY,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAA;YAC1D,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAA;gBACtC,IAAI,QAAQ,KAAK,gBAAgB,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,GAAG,gBAAgB,CAAC,EAAE,CAAC;oBAChF,SAAQ;gBACV,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,iDAAiD;gBACjD,SAAQ;YACV,CAAC;QACH,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAA;IACvC,CAAC;IAED,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,+EAA+E;AAC/E,kBAAkB;AAClB,+EAA+E;AAE/E;;;;GAIG;AACH,SAAS,WAAW,CAAC,YAA0B,EAAE,SAAkB;IACjE,MAAM,OAAO,GAAG,SAAS,IAAI,kBAAkB,CAAA;IAC/C,MAAM,aAAa,GAAG,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;IAElD,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC,CAAA;IACvD,CAAC;IACD,OAAO,aAAa,CAAA;AACtB,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,OAAgC;IACrE,MAAM,EACJ,KAAK,EACL,YAAY,EACZ,UAAU,GAAG,mBAAmB,EAChC,MAAM,EAAE,YAAY,EACpB,SAAS,GACV,GAAG,OAAO,CAAA;IAEX,kEAAkE;IAClE,MAAM,SAAS,GAAG,YAAY,CAAC,CAAC,CAAC,QAAQ,YAAY,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAA;IAExE,uBAAuB;IACvB,MAAM,SAAS,GAAG,GAAG,YAAY,MAAM,kBAAkB,CAAC,SAAS,CAAC,EAAE,CAAA;IAEtE,8DAA8D;IAC9D,MAAM,MAAM,GAAG,WAAW,CAAC,YAAY,EAAE,SAAS,CAAC,CAAA;IAEnD,IAAI,QAAkB,CAAA;IACtB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE;YAChC,OAAO,EAAE;gBACP,YAAY,EACV,iHAAiH;aACpH;YACD,MAAM;SACP,CAAC,CAAA;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,6EAA6E;QAC7E,IACE,KAAK,YAAY,YAAY;YAC7B,CAAC,KAAK,CAAC,IAAI,KAAK,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,CAAC,EAC9D,CAAC;YACD,MAAM,KAAK,CAAA;QACb,CAAC;QACD,OAAO,EAAE,CAAA;IACX,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,OAAO,EAAE,CAAA;IACX,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;IAElC,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9B,OAAO,EAAE,CAAA;IACX,CAAC;IAED,MAAM,OAAO,GAAG,6BAA6B,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;IAEjE,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAA;AACrC,CAAC"}

package/dist/shared/fetch-page.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Page fetching utility with browser-like headers and archive.org fallback.
+ *
+ * Provides a resilient page fetching pipeline:
+ * 1. Direct fetch with browser-like headers (Chrome UA, Accept text/html)
+ * 2. Block detection (hard HTTP blocks + soft body pattern matching)
+ * 3. Automatic archive.org fallback when blocked or on network error
+ * 4. Non-blocking HTTP errors (404, 500) return immediately without fallback
+ *
+ * Used by WebSearchBase when following links from search results.
+ */
+/** Options for fetching a page. */
+export interface FetchPageOptions {
+    /** URL to fetch. */
+    url: string;
+    /** AbortSignal from the caller (combined with timeoutMs). */
+    signal?: AbortSignal;
+    /** Total timeout budget in milliseconds (shared across direct + archive attempts). Default: 15000. */
+    timeoutMs?: number;
+    /** User-Agent header to send. Default: browser-like Chrome UA. */
+    userAgent?: string;
+    /** Whether to try archive.org when direct fetch is blocked. Default: true. */
+    archiveFallback?: boolean;
+}
+/** Result of a page fetch attempt. */
+export interface FetchPageResult {
+    /** Raw HTML content (empty string if fetch failed). */
+    content: string;
+    /** Final URL (may differ from input if archive.org was used). */
+    url: string;
+    /** How the content was obtained. */
+    fetchMethod: "direct" | "archive.org" | "none";
+    /** Error description when fetchMethod is "none". */
+    error?: string;
+}
+/**
+ * Fetch a page with browser-like headers and automatic archive.org fallback.
+ *
+ * Pipeline:
+ * 1. Direct fetch with browser-like headers
+ * 2. Block detection (hard HTTP status codes + soft body pattern matching)
+ * 3. If blocked and archiveFallback enabled, try archive.org
+ * 4. Non-blocking HTTP errors (404, 500) return "none" immediately
+ * 5. Network errors on direct fetch trigger archive fallback
+ *
+ * @param options - Fetch options including URL, signal, timeout, etc.
+ * @returns Result with content, final URL, and fetch method
+ */
+export declare function fetchPage(options: FetchPageOptions): Promise<FetchPageResult>;
+//# sourceMappingURL=fetch-page.d.ts.map

package/dist/shared/fetch-page.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch-page.d.ts","sourceRoot":"","sources":["../../src/shared/fetch-page.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,mCAAmC;AACnC,MAAM,WAAW,gBAAgB;IAC/B,oBAAoB;IACpB,GAAG,EAAE,MAAM,CAAA;IACX,6DAA6D;IAC7D,MAAM,CAAC,EAAE,WAAW,CAAA;IACpB,sGAAsG;IACtG,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,kEAAkE;IAClE,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,8EAA8E;IAC9E,eAAe,CAAC,EAAE,OAAO,CAAA;CAC1B;AAED,sCAAsC;AACtC,MAAM,WAAW,eAAe;IAC9B,uDAAuD;IACvD,OAAO,EAAE,MAAM,CAAA;IACf,iEAAiE;IACjE,GAAG,EAAE,MAAM,CAAA;IACX,oCAAoC;IACpC,WAAW,EAAE,QAAQ,GAAG,aAAa,GAAG,MAAM,CAAA;IAC9C,oDAAoD;IACpD,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AA2HD;;;;;;;;;;;;GAYG;AACH,wBAAsB,SAAS,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,CAAC,CAoGnF"}