@debriefer/sources 1.0.1

package/dist/structured/wikipedia.d.ts

@@ -0,0 +1,184 @@
+/**
+ * Generic Wikipedia source for encyclopedia content.
+ *
+ * Uses `wtf_wikipedia` to fetch and parse Wikipedia articles, producing clean
+ * plaintext with no citation markers, footnotes, or HTML artifacts.
+ *
+ * Domain-agnostic: consumers customize which sections to extract via the
+ * `sectionFilter` option. Default returns all sections. Common use cases:
+ * - Death research: filter for "Death", "Health", "Illness" sections
+ * - Biography research: filter for "Early life", "Personal life" sections
+ * - General research: return all sections (default)
+ *
+ * Handles disambiguation pages by trying alternate titles with common suffixes.
+ */
+import { BaseResearchSource, ReliabilityTier, type BaseSourceOptions, type ResearchSubject, type RawFinding } from "@debriefer/core";
+/** Metadata about a Wikipedia article section */
+export interface WikipediaSection {
+    /** Section index within the article */
+    index: number;
+    /** Section title (e.g., "Early life", "Death") */
+    title: string;
+    /** Depth level (0 = top-level, 1 = subsection, etc.) */
+    depth: number;
+}
+/**
+ * Function that filters Wikipedia sections to determine which to include.
+ * Receives all sections and returns the ones that should be extracted.
+ */
+export type SectionFilter = (sections: WikipediaSection[]) => WikipediaSection[];
+/**
+ * Async section filter that receives all sections and the full article text.
+ * Returns a promise resolving to the sections to include.
+ * Takes precedence over the sync `sectionFilter` when both are provided.
+ */
+export type AsyncSectionFilter = (sections: WikipediaSection[], articleText: string) => Promise<WikipediaSection[]>;
+/** Options for the Wikipedia source */
+export interface WikipediaOptions extends BaseSourceOptions {
+    /**
+     * Custom section filter. Receives all article sections, returns the ones
+     * to extract. Default: return all sections.
+     *
+     * @example
+     * ```typescript
+     * // Only extract death-related sections
+     * sectionFilter: (sections) => sections.filter(s =>
+     *   /death|illness|health|assassination/i.test(s.title)
+     * )
+     * ```
+     */
+    sectionFilter?: SectionFilter;
+    /**
+     * Async section filter. Receives all sections and the full article text,
+     * returns a promise of which sections to include. Takes precedence over
+     * the sync `sectionFilter`. Useful for AI-based section selection.
+     *
+     * @example
+     * ```typescript
+     * asyncSectionFilter: async (sections, articleText) => {
+     *   const selected = await geminiSelectSections(sections, articleText)
+     *   return selected
+     * }
+     * ```
+     */
+    asyncSectionFilter?: AsyncSectionFilter;
+    /**
+     * Whether to include the article introduction (section 0).
+     * Default: true.
+     */
+    includeIntro?: boolean;
+    /**
+     * Whether to handle disambiguation pages by trying alternate titles.
+     * Default: true.
+     */
+    handleDisambiguation?: boolean;
+    /**
+     * Alternate title suffixes to try if the article is a disambiguation page
+     * or not found. Default: ["_(actor)", "_(actress)"].
+     * Set to an empty array to disable alternate title attempts.
+     */
+    disambiguationSuffixes?: string[];
+    /**
+     * Validate that the fetched article matches the intended person.
+     * Receives the full article text and the subject. When provided and
+     * returns false, the source tries disambiguation suffixes before giving up.
+     *
+     * Supports both sync and async callbacks. An async callback (returning
+     * `Promise<boolean>`) is useful for AI-based validation (e.g., Gemini Flash
+     * date extraction) without blocking the event loop.
+     *
+     * @example
+     * ```typescript
+     * // Sync — simple birth-year check
+     * validatePerson: (articleText, subject) => {
+     *   const birthYear = subject.context?.birthYear as string
+     *   return birthYear ? articleText.includes(birthYear) : true
+     * }
+     *
+     * // Async — AI-based validation
+     * validatePerson: async (articleText, subject) => {
+     *   const dates = await extractDatesWithAI(articleText)
+     *   return dates.birthYear === subject.context?.birthYear
+     * }
+     * ```
+     */
+    validatePerson?: (articleText: string, subject: ResearchSubject) => boolean | Promise<boolean>;
+}
+/**
+ * Wikipedia source for encyclopedia article content.
+ *
+ * Fetches Wikipedia articles via `wtf_wikipedia`, extracts sections based
+ * on a configurable filter, and returns clean plaintext content as a RawFinding.
+ */
+export declare class WikipediaSource extends BaseResearchSource<ResearchSubject> {
+    readonly name = "Wikipedia";
+    readonly type = "wikipedia";
+    readonly reliabilityTier = ReliabilityTier.SECONDARY_COMPILATION;
+    readonly domain = "en.wikipedia.org";
+    readonly isFree = true;
+    readonly estimatedCostPerQuery = 0;
+    private sectionFilter;
+    private asyncSectionFilter?;
+    private includeIntro;
+    private handleDisambiguation;
+    private disambiguationSuffixes;
+    private validatePerson?;
+    constructor(options?: WikipediaOptions);
+    protected fetchResult(subject: ResearchSubject, _signal: AbortSignal): Promise<RawFinding | null>;
+    /**
+     * Build the search query for cache key generation.
+     * Includes option-derived key material so different WikipediaSource instances
+     * with different sectionFilter/includeIntro options don't collide in cache.
+     */
+    buildQuery(subject: ResearchSubject): string;
+    /**
+     * Fetch a Wikipedia document using wtf_wikipedia.
+     * Returns null if the article doesn't exist. Lets other errors propagate
+     * so BaseResearchSource.lookup() can record them via telemetry.
+     */
+    private fetchDocument;
+    /**
+     * Check if a document is a disambiguation page.
+     */
+    private isDisambig;
+    /**
+     * Try disambiguation suffixes to find a valid (non-disambiguation) article.
+     * Returns the first valid document found, or the provided fallback if none match.
+     */
+    private tryDisambiguationSuffixes;
+    /**
+     * Extract full plaintext from a document for validation and async filtering.
+     */
+    private getFullText;
+    /**
+     * Calculate content confidence based on text length and subject name presence.
+     * Returns a score between 0.3 and 0.9.
+     */
+    private calculateContentConfidence;
+}
+/**
+ * Create a Wikipedia source instance.
+ *
+ * @example
+ * ```typescript
+ * // Default: all sections
+ * const source = wikipedia()
+ *
+ * // Death research: only death-related sections
+ * const deathSource = wikipedia({
+ *   sectionFilter: (sections) => sections.filter(s =>
+ *     /death|illness|health|assassination|final years/i.test(s.title)
+ *   ),
+ * })
+ *
+ * // Biography research: personal life sections
+ * const bioSource = wikipedia({
+ *   sectionFilter: (sections) => sections.filter(s =>
+ *     /early life|personal|childhood|education|family/i.test(s.title)
+ *   ),
+ *   includeIntro: true,
+ * })
+ * ```
+ */
+export declare function wikipedia(options?: WikipediaOptions): WikipediaSource;
+//# sourceMappingURL=wikipedia.d.ts.map

package/dist/structured/wikipedia.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"wikipedia.d.ts","sourceRoot":"","sources":["../../src/structured/wikipedia.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAIH,OAAO,EACL,kBAAkB,EAClB,eAAe,EACf,KAAK,iBAAiB,EACtB,KAAK,eAAe,EACpB,KAAK,UAAU,EAChB,MAAM,iBAAiB,CAAA;AAYxB,iDAAiD;AACjD,MAAM,WAAW,gBAAgB;IAC/B,uCAAuC;IACvC,KAAK,EAAE,MAAM,CAAA;IACb,kDAAkD;IAClD,KAAK,EAAE,MAAM,CAAA;IACb,wDAAwD;IACxD,KAAK,EAAE,MAAM,CAAA;CACd;AAED;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,QAAQ,EAAE,gBAAgB,EAAE,KAAK,gBAAgB,EAAE,CAAA;AAEhF;;;;GAIG;AACH,MAAM,MAAM,kBAAkB,GAAG,CAC/B,QAAQ,EAAE,gBAAgB,EAAE,EAC5B,WAAW,EAAE,MAAM,KAChB,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAA;AAEhC,uCAAuC;AACvC,MAAM,WAAW,gBAAiB,SAAQ,iBAAiB;IACzD;;;;;;;;;;;OAWG;IACH,aAAa,CAAC,EAAE,aAAa,CAAA;IAE7B;;;;;;;;;;;;OAYG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAA;IAEvC;;;OAGG;IACH,YAAY,CAAC,EAAE,OAAO,CAAA;IAEtB;;;OAGG;IACH,oBAAoB,CAAC,EAAE,OAAO,CAAA;IAE9B;;;;OAIG;IACH,sBAAsB,CAAC,EAAE,MAAM,EAAE,CAAA;IAEjC;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,cAAc,CAAC,EAAE,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,KAAK,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;CAC/F;AAiBD;;;;;GAKG;AACH,qBAAa,eAAgB,SAAQ,kBAAkB,CAAC,eAAe,CAAC;IACtE,QAAQ,CAAC,IAAI,eAAc;IAC3B,QAAQ,CAAC,IAAI,eAAc;IAC3B,QAAQ,CAAC,eAAe,yCAAwC;IAChE,QAAQ,CAAC,MAAM,sBAAqB;IACpC,QAAQ,CAAC,MAAM,QAAO;IACtB,QAAQ,CAAC,qBAAqB,KAAI;IAElC,OAAO,CAAC,aAAa,CAAe;IACpC,OAAO,CAAC,kBAAkB,CAAC,CAAoB;IAC/C,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,oBAAoB,CAAS;IACrC,OAAO,CAAC,sBAAsB,CAAU;IACxC,OAAO,CAAC,cAAc,CAAC,CAGQ;gBAEnB,OAAO,GAAE,gBAAqB;cAU1B,WAAW,CACzB,OAAO,EAAE,eAAe,EAIxB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IAyG7B;;;;OAIG;IACM,UAAU,CAAC,OAAO,EAAE,eAAe,GAAG,MAAM;IAarD;;;;OAIG;YACW,aAAa;IAK3B;;OAEG;IACH,OAAO,CAAC,UAAU;IAIlB;;;OAGG;YACW,yBAAyB;IAcvC;;OAEG;IACH,OAAO,CAAC,WAAW;IAKnB;;;OAGG;IACH,OAAO,CAAC,0BAA0B;CAiCnC;AAMD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,SAAS,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,eAAe,CAErE"}

package/dist/structured/wikipedia.js

@@ -0,0 +1,275 @@
+/**
+ * Generic Wikipedia source for encyclopedia content.
+ *
+ * Uses `wtf_wikipedia` to fetch and parse Wikipedia articles, producing clean
+ * plaintext with no citation markers, footnotes, or HTML artifacts.
+ *
+ * Domain-agnostic: consumers customize which sections to extract via the
+ * `sectionFilter` option. Default returns all sections. Common use cases:
+ * - Death research: filter for "Death", "Health", "Illness" sections
+ * - Biography research: filter for "Early life", "Personal life" sections
+ * - General research: return all sections (default)
+ *
+ * Handles disambiguation pages by trying alternate titles with common suffixes.
+ */
+import wtf from "wtf_wikipedia";
+import { BaseResearchSource, ReliabilityTier, } from "@debriefer/core";
+// ============================================================================
+// Constants
+// ============================================================================
+const MIN_SECTION_LENGTH = 50;
+// ============================================================================
+// Default Section Filter
+// ============================================================================
+/**
+ * Default section filter: returns all sections.
+ */
+function defaultSectionFilter(sections) {
+    return sections;
+}
+// ============================================================================
+// Source Implementation
+// ============================================================================
+/**
+ * Wikipedia source for encyclopedia article content.
+ *
+ * Fetches Wikipedia articles via `wtf_wikipedia`, extracts sections based
+ * on a configurable filter, and returns clean plaintext content as a RawFinding.
+ */
+export class WikipediaSource extends BaseResearchSource {
+    name = "Wikipedia";
+    type = "wikipedia";
+    reliabilityTier = ReliabilityTier.SECONDARY_COMPILATION;
+    domain = "en.wikipedia.org";
+    isFree = true;
+    estimatedCostPerQuery = 0;
+    sectionFilter;
+    asyncSectionFilter;
+    includeIntro;
+    handleDisambiguation;
+    disambiguationSuffixes;
+    validatePerson;
+    constructor(options = {}) {
+        super({ rateLimitMs: 500, ...options });
+        this.sectionFilter = options.sectionFilter ?? defaultSectionFilter;
+        this.asyncSectionFilter = options.asyncSectionFilter;
+        this.includeIntro = options.includeIntro ?? true;
+        this.handleDisambiguation = options.handleDisambiguation ?? true;
+        this.disambiguationSuffixes = options.disambiguationSuffixes ?? ["_(actor)", "_(actress)"];
+        this.validatePerson = options.validatePerson;
+    }
+    async fetchResult(subject, 
+    // Note: wtf_wikipedia.fetch() does not accept an AbortSignal.
+    // The base class timeout/abort still applies to the overall lookup()
+    // call, but the underlying HTTP request cannot be cancelled mid-flight.
+    _signal) {
+        const baseTitle = subject.name.replace(/ /g, "_");
+        // Try the base title first
+        let doc = await this.fetchDocument(baseTitle);
+        // Handle disambiguation pages
+        if (this.handleDisambiguation && (!doc || this.isDisambig(doc))) {
+            doc = await this.tryDisambiguationSuffixes(baseTitle, doc);
+        }
+        // If we still have no valid document, return null
+        if (!doc || this.isDisambig(doc))
+            return null;
+        // Validate person if callback is provided.
+        // Track fullText so we can reuse it for asyncSectionFilter without recomputing.
+        let cachedFullText;
+        if (this.validatePerson) {
+            cachedFullText = this.getFullText(doc);
+            if (!(await this.validatePerson(cachedFullText, subject))) {
+                // Validation failed — try disambiguation suffixes if enabled
+                if (!this.handleDisambiguation)
+                    return null;
+                const altDoc = await this.tryDisambiguationSuffixes(baseTitle, null);
+                if (!altDoc || this.isDisambig(altDoc))
+                    return null;
+                // Validate the alternate document too
+                const altText = this.getFullText(altDoc);
+                if (!(await this.validatePerson(altText, subject)))
+                    return null;
+                doc = altDoc;
+                cachedFullText = altText;
+            }
+        }
+        const sections = doc.sections();
+        if (sections.length === 0)
+            return null;
+        // Reuse cached full text from validation, or compute once for async filter
+        const fullText = this.asyncSectionFilter ? (cachedFullText ?? this.getFullText(doc)) : undefined;
+        // Map wtf sections to WikipediaSection interface
+        const wikiSections = sections.map((s, i) => ({
+            index: i,
+            title: s.title() || "Introduction",
+            depth: s.depth(),
+        }));
+        // Apply section filter — async takes precedence over sync
+        const selectedSections = this.asyncSectionFilter
+            ? await this.asyncSectionFilter(wikiSections, fullText)
+            : this.sectionFilter(wikiSections);
+        // Build the set of section indices to extract
+        const indicesToExtract = new Set(selectedSections.map((s) => s.index));
+        // Always include intro if configured and not already in the filter result
+        if (this.includeIntro && !indicesToExtract.has(0)) {
+            indicesToExtract.add(0);
+        }
+        // If nothing to extract (filter returned empty and intro disabled), return null
+        if (indicesToExtract.size === 0)
+            return null;
+        // Extract text from selected sections
+        const sectionTexts = [];
+        const extractedTitles = [];
+        // Sort indices for consistent output order (article order)
+        const sortedIndices = [...indicesToExtract].sort((a, b) => a - b);
+        for (const idx of sortedIndices) {
+            const section = sections[idx];
+            if (!section)
+                continue;
+            const title = section.title() || "Introduction";
+            const text = section.text({});
+            if (text && text.length >= MIN_SECTION_LENGTH) {
+                sectionTexts.push(`[${title}] ${text}`);
+                extractedTitles.push(title);
+            }
+        }
+        if (sectionTexts.length === 0)
+            return null;
+        const combinedText = sectionTexts.join("\n\n");
+        const resolvedTitle = doc.title() || baseTitle.replace(/_/g, " ");
+        const resolvedUrl = `https://en.wikipedia.org/wiki/${encodeURIComponent(resolvedTitle.replace(/ /g, "_"))}`;
+        // Calculate confidence based on content quality
+        const confidence = this.calculateContentConfidence(combinedText, subject, sectionTexts.length);
+        return {
+            text: combinedText,
+            confidence,
+            costUsd: 0,
+            url: resolvedUrl,
+            publication: "Wikipedia",
+            articleTitle: resolvedTitle,
+            metadata: {
+                sectionCount: sectionTexts.length,
+                sectionTitles: extractedTitles,
+                textLength: combinedText.length,
+            },
+        };
+    }
+    /**
+     * Build the search query for cache key generation.
+     * Includes option-derived key material so different WikipediaSource instances
+     * with different sectionFilter/includeIntro options don't collide in cache.
+     */
+    buildQuery(subject) {
+        const parts = [subject.name];
+        if (this.asyncSectionFilter)
+            parts.push("sections:async");
+        else if (this.sectionFilter !== defaultSectionFilter)
+            parts.push("sections:custom");
+        if (this.includeIntro === false)
+            parts.push("no-intro");
+        if (this.validatePerson)
+            parts.push("validate:person");
+        if (!this.handleDisambiguation)
+            parts.push("disambig:off");
+        if (this.handleDisambiguation && this.disambiguationSuffixes.length > 0) {
+            parts.push(`suffixes:${this.disambiguationSuffixes.join(",")}`);
+        }
+        return parts.join("|");
+    }
+    /**
+     * Fetch a Wikipedia document using wtf_wikipedia.
+     * Returns null if the article doesn't exist. Lets other errors propagate
+     * so BaseResearchSource.lookup() can record them via telemetry.
+     */
+    async fetchDocument(title) {
+        const doc = await wtf.fetch(title);
+        return doc ?? null;
+    }
+    /**
+     * Check if a document is a disambiguation page.
+     */
+    isDisambig(doc) {
+        return doc.isDisambiguation();
+    }
+    /**
+     * Try disambiguation suffixes to find a valid (non-disambiguation) article.
+     * Returns the first valid document found, or the provided fallback if none match.
+     */
+    async tryDisambiguationSuffixes(baseTitle, fallback) {
+        for (const suffix of this.disambiguationSuffixes) {
+            const altTitle = baseTitle + suffix;
+            const altDoc = await this.fetchDocument(altTitle);
+            if (altDoc && !this.isDisambig(altDoc)) {
+                return altDoc;
+            }
+        }
+        return fallback;
+    }
+    /**
+     * Extract full plaintext from a document for validation and async filtering.
+     */
+    getFullText(doc) {
+        const sections = doc.sections();
+        return sections.map((s) => s.text({})).join("\n\n");
+    }
+    /**
+     * Calculate content confidence based on text length and subject name presence.
+     * Returns a score between 0.3 and 0.9.
+     */
+    calculateContentConfidence(text, subject, sectionCount) {
+        let confidence = 0.4;
+        // Name presence
+        if (text.toLowerCase().includes(subject.name.toLowerCase())) {
+            confidence += 0.1;
+        }
+        // Content length
+        if (text.length > 500) {
+            confidence += 0.2;
+        }
+        else if (text.length > 200) {
+            confidence += 0.1;
+        }
+        // If keywords are configured, delegate to the base class keyword-based
+        // confidence calculation instead of using our content heuristics.
+        // The base class checks for confidence === -1 as the delegation signal.
+        if (this.options.requiredKeywords && this.options.requiredKeywords.length > 0) {
+            return -1; // DELEGATE_TO_BASE_CLASS: base-source.ts:150 replaces with keyword confidence
+        }
+        // Section count bonus — use actual extracted section count, not regex on text
+        if (sectionCount > 1) {
+            confidence += Math.min(0.2, sectionCount * 0.05);
+        }
+        return Math.min(0.9, confidence);
+    }
+}
+// ============================================================================
+// Factory Function
+// ============================================================================
+/**
+ * Create a Wikipedia source instance.
+ *
+ * @example
+ * ```typescript
+ * // Default: all sections
+ * const source = wikipedia()
+ *
+ * // Death research: only death-related sections
+ * const deathSource = wikipedia({
+ *   sectionFilter: (sections) => sections.filter(s =>
+ *     /death|illness|health|assassination|final years/i.test(s.title)
+ *   ),
+ * })
+ *
+ * // Biography research: personal life sections
+ * const bioSource = wikipedia({
+ *   sectionFilter: (sections) => sections.filter(s =>
+ *     /early life|personal|childhood|education|family/i.test(s.title)
+ *   ),
+ *   includeIntro: true,
+ * })
+ * ```
+ */
+export function wikipedia(options) {
+    return new WikipediaSource(options);
+}
+//# sourceMappingURL=wikipedia.js.map

package/dist/structured/wikipedia.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"wikipedia.js","sourceRoot":"","sources":["../../src/structured/wikipedia.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,GAAG,MAAM,eAAe,CAAA;AAE/B,OAAO,EACL,kBAAkB,EAClB,eAAe,GAIhB,MAAM,iBAAiB,CAAA;AAExB,+EAA+E;AAC/E,YAAY;AACZ,+EAA+E;AAE/E,MAAM,kBAAkB,GAAG,EAAE,CAAA;AA6G7B,+EAA+E;AAC/E,yBAAyB;AACzB,+EAA+E;AAE/E;;GAEG;AACH,SAAS,oBAAoB,CAAC,QAA4B;IACxD,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED,+EAA+E;AAC/E,wBAAwB;AACxB,+EAA+E;AAE/E;;;;;GAKG;AACH,MAAM,OAAO,eAAgB,SAAQ,kBAAmC;IAC7D,IAAI,GAAG,WAAW,CAAA;IAClB,IAAI,GAAG,WAAW,CAAA;IAClB,eAAe,GAAG,eAAe,CAAC,qBAAqB,CAAA;IACvD,MAAM,GAAG,kBAAkB,CAAA;IAC3B,MAAM,GAAG,IAAI,CAAA;IACb,qBAAqB,GAAG,CAAC,CAAA;IAE1B,aAAa,CAAe;IAC5B,kBAAkB,CAAqB;IACvC,YAAY,CAAS;IACrB,oBAAoB,CAAS;IAC7B,sBAAsB,CAAU;IAChC,cAAc,CAGS;IAE/B,YAAY,UAA4B,EAAE;QACxC,KAAK,CAAC,EAAE,WAAW,EAAE,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC,CAAA;QACvC,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,oBAAoB,CAAA;QAClE,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,CAAA;QACpD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAA;QAChD,IAAI,CAAC,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,IAAI,IAAI,CAAA;QAChE,IAAI,CAAC,sBAAsB,GAAG,OAAO,CAAC,sBAAsB,IAAI,CAAC,UAAU,EAAE,YAAY,CAAC,CAAA;QAC1F,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,CAAA;IAC9C,CAAC;IAES,KAAK,CAAC,WAAW,CACzB,OAAwB;IACxB,8DAA8D;IAC9D,qEAAqE;IACrE,wEAAwE;IACxE,OAAoB;QAEpB,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;QAEjD,2BAA2B;QAC3B,IAAI,GAAG,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAA;QAE7C,8BAA8B;QAC9B,IAAI,IAAI,CAAC,oBAAoB,IAAI,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YAChE,GAAG,GAAG,MAAM,IAAI,CAAC,yBAAyB,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;QAC5D,CAAC;QAED,kDAAkD;QAClD,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,OAAO,IAAI,CAAA;QAE7C,2CAA2C;QAC3C,gFAAgF;QAChF,IAAI,cAAkC,CAAA;QACtC,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,cAAc,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAA;YACtC,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC,EAAE,CAAC;gBAC1D,6DAA6D;gBAC7D,IAAI,CAAC,IAAI,CAAC,oBAAoB;oBAAE,OAAO,IAAI,CAAA;gBAC3C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,yBAAyB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;gBACpE,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;oBAAE,OAAO,IAAI,CAAA;gBACnD,sCAAsC;gBACtC,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAA;gBACxC,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;oBAAE,OAAO,IAAI,CAAA;gBAC/D,GAAG,GAAG,MAAM,CAAA;gBACZ,cAAc,GAAG,OAAO,CAAA;YAC1B,CAAC;QACH,CAAC;QAED,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,EAAmB,CAAA;QAChD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAA;QAEtC,2EAA2E;QAC3E,MAAM,QAAQ,GAAG,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,cAAc,IAAI,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;QAEhG,iDAAiD;QACjD,MAAM,YAAY,GAAuB,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAc,EAAE,CAAS,EAAE,EAAE,CAAC,CAAC;YACpF,KAAK,EAAE,CAAC;YACR,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,cAAc;YAClC,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE;SACjB,CAAC,CAAC,CAAA;QAEH,0DAA0D;QAC1D,MAAM,gBAAgB,GAAG,IAAI,CAAC,kBAAkB;YAC9C,CAAC,CAAC,MAAM,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE,QAAS,CAAC;YACxD,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,CAAA;QAEpC,8CAA8C;QAC9C,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAA;QAEtE,0EAA0E;QAC1E,IAAI,IAAI,CAAC,YAAY,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YAClD,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;QACzB,CAAC;QAED,gFAAgF;QAChF,IAAI,gBAAgB,CAAC,IAAI,KAAK,CAAC;YAAE,OAAO,IAAI,CAAA;QAE5C,sCAAsC;QACtC,MAAM,YAAY,GAAa,EAAE,CAAA;QACjC,MAAM,eAAe,GAAa,EAAE,CAAA;QAEpC,2DAA2D;QAC3D,MAAM,aAAa,GAAG,CAAC,GAAG,gBAAgB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QAEjE,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;YAChC,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAA4B,CAAA;YACxD,IAAI,CAAC,OAAO;gBAAE,SAAQ;YAEtB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,EAAE,IAAI,cAAc,CAAA;YAC/C,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YAE7B,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,IAAI,kBAAkB,EAAE,CAAC;gBAC9C,YAAY,CAAC,IAAI,CAAC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC,CAAA;gBACvC,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC7B,CAAC;QACH,CAAC;QAED,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAA;QAE1C,MAAM,YAAY,GAAG,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAC9C,MAAM,aAAa,GAAG,GAAG,CAAC,KAAK,EAAE,IAAI,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;QACjE,MAAM,WAAW,GAAG,iCAAiC,kBAAkB,CAAC,aAAa,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,EAAE,CAAA;QAE3G,gDAAgD;QAChD,MAAM,UAAU,GAAG,IAAI,CAAC,0BAA0B,CAAC,YAAY,EAAE,OAAO,EAAE,YAAY,CAAC,MAAM,CAAC,CAAA;QAE9F,OAAO;YACL,IAAI,EAAE,YAAY;YAClB,UAAU;YACV,OAAO,EAAE,CAAC;YACV,GAAG,EAAE,WAAW;YAChB,WAAW,EAAE,WAAW;YACxB,YAAY,EAAE,aAAa;YAC3B,QAAQ,EAAE;gBACR,YAAY,EAAE,YAAY,CAAC,MAAM;gBACjC,aAAa,EAAE,eAAe;gBAC9B,UAAU,EAAE,YAAY,CAAC,MAAM;aAChC;SACF,CAAA;IACH,CAAC;IAED;;;;OAIG;IACM,UAAU,CAAC,OAAwB;QAC1C,MAAM,KAAK,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;QAC5B,IAAI,IAAI,CAAC,kBAAkB;YAAE,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;aACpD,IAAI,IAAI,CAAC,aAAa,KAAK,oBAAoB;YAAE,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;QACnF,IAAI,IAAI,CAAC,YAAY,KAAK,KAAK;YAAE,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;QACvD,IAAI,IAAI,CAAC,cAAc;YAAE,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;QACtD,IAAI,CAAC,IAAI,CAAC,oBAAoB;YAAE,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;QAC1D,IAAI,IAAI,CAAC,oBAAoB,IAAI,IAAI,CAAC,sBAAsB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxE,KAAK,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QACjE,CAAC;QACD,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACxB,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,aAAa,CAAC,KAAa;QACvC,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;QAClC,OAAQ,GAAgD,IAAI,IAAI,CAAA;IAClE,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,GAAsC;QACvD,OAAO,GAAG,CAAC,gBAAgB,EAAE,CAAA;IAC/B,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,yBAAyB,CACrC,SAAiB,EACjB,QAAkD;QAElD,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,sBAAsB,EAAE,CAAC;YACjD,MAAM,QAAQ,GAAG,SAAS,GAAG,MAAM,CAAA;YACnC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAA;YACjD,IAAI,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;gBACvC,OAAO,MAAM,CAAA;YACf,CAAC;QACH,CAAC;QACD,OAAO,QAAQ,CAAA;IACjB,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,GAAsC;QACxD,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,EAAmB,CAAA;QAChD,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAc,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IAClE,CAAC;IAED;;;OAGG;IACK,0BAA0B,CAChC,IAAY,EACZ,OAAwB,EACxB,YAAoB;QAEpB,IAAI,UAAU,GAAG,GAAG,CAAA;QAEpB,gBAAgB;QAChB,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;YAC5D,UAAU,IAAI,GAAG,CAAA;QACnB,CAAC;QAED,iBAAiB;QACjB,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YACtB,UAAU,IAAI,GAAG,CAAA;QACnB,CAAC;aAAM,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YAC7B,UAAU,IAAI,GAAG,CAAA;QACnB,CAAC;QAED,uEAAuE;QACvE,kEAAkE;QAClE,wEAAwE;QACxE,IAAI,IAAI,CAAC,OAAO,CAAC,gBAAgB,IAAI,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9E,OAAO,CAAC,CAAC,CAAA,CAAC,8EAA8E;QAC1F,CAAC;QAED,8EAA8E;QAC9E,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;YACrB,UAAU,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,GAAG,IAAI,CAAC,CAAA;QAClD,CAAC;QAED,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC,CAAA;IAClC,CAAC;CACF;AAED,+EAA+E;AAC/E,mBAAmB;AACnB,+EAA+E;AAE/E;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,UAAU,SAAS,CAAC,OAA0B;IAClD,OAAO,IAAI,eAAe,CAAC,OAAO,CAAC,CAAA;AACrC,CAAC"}

package/dist/web-search/base.d.ts

@@ -0,0 +1,128 @@
+/**
+ * Abstract base class for web search sources using the template method pattern.
+ *
+ * Subclasses only implement `performSearch()` — the base class handles the
+ * full pipeline: search → score/rank links → fetch pages → extract content →
+ * sanitize → combine with attribution.
+ *
+ * Used by Google, Bing, Brave, and DuckDuckGo search sources.
+ */
+import { BaseResearchSource, type BaseSourceOptions, type ResearchSubject, type RawFinding } from "@debriefer/core";
+/** A single result from a web search engine. */
+export interface WebSearchResult {
+    url: string;
+    title: string;
+    snippet: string;
+}
+/** Options for scoring and filtering search result links. */
+export interface LinkSelectionOptions {
+    /** Domain → 0-100 score. Adds to link score when hostname matches. */
+    domainScores?: Record<string, number>;
+    /** Keywords that boost a result's score when found in title+snippet. */
+    boostKeywords?: Array<{
+        keyword: string;
+        boost: number;
+    }>;
+    /** Keywords that penalize a result's score when found in title+snippet. */
+    penaltyKeywords?: Array<{
+        keyword: string;
+        penalty: number;
+    }>;
+    /** Domains to completely exclude from results. */
+    blockedDomains?: string[];
+}
+/** Options for WebSearchBase sources, combining base source and link selection options. */
+export interface WebSearchOptions extends BaseSourceOptions, LinkSelectionOptions {
+    /** Maximum number of search result pages to fetch. Default: 3. */
+    maxLinksToFollow?: number;
+    /** Minimum extracted text length in characters. Pages below this are filtered. Default: 200. */
+    minContentLength?: number;
+    /**
+     * Maximum cost in USD for link following per subject. When set, the source
+     * tracks cumulative fetch cost and stops following links when the budget
+     * is exhausted. Default: unlimited.
+     */
+    maxLinkCost?: number;
+    /**
+     * Custom link selector that filters/reorders search results before fetching.
+     * Receives ranked results and the subject, returns the results to follow.
+     * Useful for AI-assisted link selection (e.g., Claude ranking URLs by relevance).
+     * Applied after scoring/ranking but before the maxLinksToFollow limit.
+     */
+    linkSelector?: (results: WebSearchResult[], subject: ResearchSubject) => Promise<WebSearchResult[]> | WebSearchResult[];
+    /**
+     * Custom page fetcher that replaces the default fetch+readability pipeline.
+     * Useful for browser-based fetching (Playwright) or sites requiring
+     * authentication/fingerprinting. Returns extracted text or null on failure.
+     */
+    fetchPage?: (url: string, signal: AbortSignal) => Promise<string | null>;
+}
+/**
+ * Abstract base class for web search sources.
+ *
+ * Implements the template method pattern: subclasses provide `performSearch()`
+ * and this class handles the rest of the pipeline (scoring, fetching,
+ * extracting, sanitizing, combining).
+ */
+export declare abstract class WebSearchBase extends BaseResearchSource<ResearchSubject> {
+    protected readonly maxLinksToFollow: number;
+    protected readonly minContentLength: number;
+    protected readonly domainScores: Record<string, number>;
+    protected readonly boostKeywords: Array<{
+        keyword: string;
+        boost: number;
+    }>;
+    protected readonly penaltyKeywords: Array<{
+        keyword: string;
+        penalty: number;
+    }>;
+    protected readonly blockedDomains: string[];
+    private readonly maxLinkCost?;
+    private readonly linkSelector?;
+    private readonly customFetchPage?;
+    constructor(options?: WebSearchOptions);
+    /**
+     * Subclass-specific search API call. Returns raw search results.
+     *
+     * @param query - The search query string
+     * @param signal - Abort signal for cancellation
+     * @returns Array of search results (URL, title, snippet)
+     */
+    protected abstract performSearch(query: string, signal: AbortSignal): Promise<WebSearchResult[]>;
+    /**
+     * Full search pipeline: search → score → fetch → extract → combine.
+     *
+     * @param subject - The research subject
+     * @param signal - Abort signal for cancellation
+     * @returns RawFinding with combined text, or null if no content extracted
+     */
+    protected fetchResult(subject: ResearchSubject, signal: AbortSignal): Promise<RawFinding | null>;
+    /**
+     * Score and rank search results by relevance.
+     *
+     * Scoring:
+     * - Base: 50 - index (preserves search engine ordering)
+     * - + domainScores[domain] if hostname matches
+     * - + boost for each boostKeyword found in title+snippet
+     * - - penalty for each penaltyKeyword found in title+snippet
+     */
+    private scoreAndRank;
+    /**
+     * Check whether a URL should be excluded (blocked domain or unsafe URL).
+     * Blocks: non-http(s) schemes, localhost, private IP ranges, and user-specified domains.
+     */
+    protected isDomainBlocked(url: string): boolean;
+    /**
+     * Check if a hostname matches a domain (exact or subdomain match).
+     * Normalizes domain to lowercase since URL.hostname is always lowercase.
+     *
+     * "www.example.com" matches "example.com"
+     * "sub.example.com" matches "example.com"
+     * "example.com" matches "example.com"
+     * "notexample.com" does NOT match "example.com"
+     */
+    private hostnameMatchesDomain;
+    /** Check if hostname is in the 172.16.0.0–172.31.255.255 private range (RFC 1918). */
+    private isPrivate172;
+}
+//# sourceMappingURL=base.d.ts.map

package/dist/web-search/base.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"base.d.ts","sourceRoot":"","sources":["../../src/web-search/base.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EACL,kBAAkB,EAClB,KAAK,iBAAiB,EACtB,KAAK,eAAe,EACpB,KAAK,UAAU,EAChB,MAAM,iBAAiB,CAAA;AASxB,gDAAgD;AAChD,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAA;IACX,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,MAAM,CAAA;CAChB;AAED,6DAA6D;AAC7D,MAAM,WAAW,oBAAoB;IACnC,sEAAsE;IACtE,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACrC,wEAAwE;IACxE,aAAa,CAAC,EAAE,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IACzD,2EAA2E;IAC3E,eAAe,CAAC,EAAE,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IAC7D,kDAAkD;IAClD,cAAc,CAAC,EAAE,MAAM,EAAE,CAAA;CAC1B;AAED,2FAA2F;AAC3F,MAAM,WAAW,gBAAiB,SAAQ,iBAAiB,EAAE,oBAAoB;IAC/E,kEAAkE;IAClE,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,gGAAgG;IAChG,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;;;;OAKG;IACH,YAAY,CAAC,EAAE,CACb,OAAO,EAAE,eAAe,EAAE,EAC1B,OAAO,EAAE,eAAe,KACrB,OAAO,CAAC,eAAe,EAAE,CAAC,GAAG,eAAe,EAAE,CAAA;IACnD;;;;OAIG;IACH,SAAS,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAA;CACzE;AAeD;;;;;;GAMG;AACH,8BAAsB,aAAc,SAAQ,kBAAkB,CAAC,eAAe,CAAC;IAC7E,SAAS,CAAC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAA;IAC3C,SAAS,CAAC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAA;IAC3C,SAAS,CAAC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACvD,SAAS,CAAC,QAAQ,CAAC,aAAa,EAAE,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IAC3E,SAAS,CAAC,QAAQ,CAAC,eAAe,EAAE,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IAC/E,SAAS,CAAC,QAAQ,CAAC,cAAc,EAAE,MAAM,EAAE,CAAA;IAC3C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAQ;IACrC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAkC;IAChE,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAA+B;gBAEpD,OAAO,GAAE,gBAAqB;IAa1C;;;;;;OAMG;IACH,SAAS,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAEhG;;;;;;OAMG;cACa,WAAW,CACzB,OAAO,EAAE,eAAe,EACxB,MAAM,EAAE,WAAW,GAClB,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IA4G7B;;;;;;;;OAQG;IACH,OAAO,CAAC,YAAY;IA2CpB;;;OAGG;IACH,SAAS,CAAC,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IA0C/C;;;;;;;;OAQG;IACH,OAAO,CAAC,qBAAqB;IAK7B,sFAAsF;IACtF,OAAO,CAAC,YAAY;CAKrB"}