alexa-ai 2.1.1 → 2.1.2

package/src/services/WebAnswer.js

@@ -141,6 +141,12 @@ class WebAnswer {
     /**
      * The user turn sent to DeepAI.
      *
+     * With `results` (the engine's own web search) the model is a writer, not
+     * a researcher: it gets numbered search results as material and must
+     * build the report from them, citing the numbers. Without results it
+     * falls back to DeepAI's server-side search, which is unreliable on free
+     * models and frequently invents sources.
+     *
      * The long form hands the model a fill-in-the-blanks template rather than
      * a description of one: small models (`gpt-4o-mini`, `standard`) follow a
      * visible layout far more reliably than "write several sections".
@@ -148,14 +154,16 @@ class WebAnswer {
      * @param {string} question
      * @param {object} [opts]
      * @param {'short'|'long'} [opts.detail='long']
+     * @param {Array<{title?:string,url:string,description?:string,date?:string}>} [opts.results]
      * @param {string} [opts.language]        "Sinhala", "Tamil", … (default: the language of the query)
      * @param {string} [opts.instructions]    extra guidance ("focus on Sri Lanka")
      * @param {Date} [opts.now]
      * @returns {string}
      */
-    static prompt(question, { detail = 'long', language = '', instructions = '', now = new Date() } = {}) {
+    static prompt(question, { detail = 'long', results = null, language = '', instructions = '', now = new Date() } = {}) {
         const topic = String(question ?? '').trim();
         const today = WebAnswer._isoDate(now);
+        const grounded = Array.isArray(results) && results.length > 0;
         const extras = [];
         if (String(language ?? '').trim()) extras.push(`Write the entire answer in ${String(language).trim()}.`);
         if (String(instructions ?? '').trim()) extras.push(String(instructions).trim());
@@ -168,27 +176,43 @@ class WebAnswer {
             'Do not add notes about training data, knowledge cut-offs or being unable to browse the web, and do not ' +
             'describe yourself or your origins — just present the findings. Never reply with a one-word command.';
 
+        const material = grounded ? WebAnswer.formatResults(results) : '';
+        const opening = grounded
+            ? 'Below are web search results for the topic. Use them as your material: ' +
+              'build the answer from what they say, and put the result number in square brackets after each fact you take from one, like [2]. ' +
+              'You may add well-known background knowledge, but do not invent events, figures, dates or quotes.'
+            : 'Use your web search tool to research the topic below, then write';
+        const sourcesRule = grounded
+            ? 'Do NOT write a "Sources" list — the sources are attached automatically from the search results. Never write a URL.'
+            : 'End with a line that says exactly "Sources:" followed by the pages you used, one per line, written as "title (url)".';
+
         if (WebAnswer.detailOf(detail) === 'short') {
             return (
-                'Use your web search tool to look up the topic below, then answer directly in 2 to 4 sentences ' +
-                'with the most important current facts (figures, names, dates).\n\n' +
-                `Topic: ${topic}\nToday's date: ${today}\n\n` +
-                `Rules:\n1. ${formatting}\n` +
-                '2. End with a line that says exactly "Sources:" followed by the pages you used, one per line, ' +
-                'written as "title (url)".\n' +
-                `3. ${conduct}` +
+                (grounded
+                    ? `${opening}\n\nTopic: ${topic}\nToday's date: ${today}\n\nSearch results:\n${material}\n\n` +
+                      'Answer directly in 2 to 4 sentences with the most important current facts (figures, names, dates).\n\n'
+                    : `${opening.replace(', then write', '')}, then answer directly in 2 to 4 sentences ` +
+                      `with the most important current facts (figures, names, dates).\n\nTopic: ${topic}\nToday's date: ${today}\n\n`) +
+                `Rules:\n1. ${formatting}\n2. ${sourcesRule}\n3. ${conduct}` +
                 extra
             );
         }
 
+        const layoutSources = grounded ? '' : 'Sources:\n<Page title> (<url>)\n<Page title> (<url>)\n<Page title> (<url>)\n\n';
+        const layoutPoint = grounded
+            ? '<Two sentences with specific facts, names, figures and dates from the results.> [<result number>]'
+            : '<Two sentences with specific facts, names, figures and dates from your search.>';
+
         return (
-            'Use your web search tool to research the topic below, then write a detailed, well-organised report ' +
-            'on it for a WhatsApp reader.\n\n' +
-            `Topic: ${topic}\nToday's date: ${today}\n\n` +
+            (grounded
+                ? `${opening}\n\nTopic: ${topic}\nToday's date: ${today}\n\nSearch results:\n${material}\n\n` +
+                  'Write a detailed, well-organised report on the topic for a WhatsApp reader.\n\n'
+                : `${opening} a detailed, well-organised report on it for a WhatsApp reader.\n\n` +
+                  `Topic: ${topic}\nToday's date: ${today}\n\n`) +
             'Required layout — follow it exactly, replacing every <placeholder> (do not print the angle brackets):\n\n' +
             '<One or two sentences that directly answer or introduce the topic.>\n\n' +
             '*<Section heading 1>:*\n' +
-            '1. *<Headline>*: <Two sentences with specific facts, names, figures and dates from your search.>\n' +
+            `1. *<Headline>*: ${layoutPoint}\n` +
             '2. *<Headline>*: <…>\n' +
             '3. *<Headline>*: <…>\n' +
             '4. *<Headline>*: <…>\n\n' +
@@ -201,10 +225,7 @@ class WebAnswer {
             '1. *<Headline>*: <…>\n' +
             '2. *<Headline>*: <…>\n' +
             '3. *<Headline>*: <…>\n\n' +
-            'Sources:\n' +
-            '<Page title> (<url>)\n' +
-            '<Page title> (<url>)\n' +
-            '<Page title> (<url>)\n\n' +
+            layoutSources +
             'Rules:\n' +
             '1. Write 3 to 5 sections with 3 to 4 numbered points each — about 300 to 450 words in total. ' +
             'Never stop after one paragraph.\n' +
@@ -213,14 +234,122 @@ class WebAnswer {
             '*Recent Movement:* and *What Is Driving It:*; for a person or company, *Latest News:*, *Background:* ' +
             'and *Key Facts:*.\n' +
             '3. Every point must carry concrete, current information (numbers, names, places, dates) — no filler ' +
-            'and no repetition.\n' +
+            'and no repetition.' +
+            (grounded
+                ? ' If the results contain little about the topic, say so in the intro and cover what they do contain — never fill the gap with invented details.\n'
+                : '\n') +
             `4. ${formatting}\n` +
-            '5. The "Sources:" list comes last, one page per line, at least 3 pages if you can.\n' +
+            `5. ${sourcesRule}${grounded ? '' : ' List at least 3 pages if you can.'}\n` +
             `6. ${conduct}` +
             extra
         );
     }
 
+    /** Numbered search results as prompt material. */
+    static formatResults(results, { maxDescription = 300 } = {}) {
+        return (Array.isArray(results) ? results : [])
+            .map((r, i) => {
+                const title = String(r.title || '').trim() || WebAnswer._hostOf(r.url) || 'Untitled';
+                const date = r.date ? ` (${r.date})` : '';
+                const host = WebAnswer._hostOf(r.url);
+                const desc = String(r.description || '').replace(/\s+/g, ' ').trim();
+                const body = desc ? `\n   ${desc.length > maxDescription ? `${desc.slice(0, maxDescription - 1).trim()}…` : desc}` : '';
+                return `[${i + 1}] ${title}${date}${host ? ` — ${host}` : ''}${body}`;
+            })
+            .join('\n');
+    }
+
+    /**
+     * Plain list of the search results — the reply when the model is
+     * unavailable but the search worked.
+     */
+    static digest(results, { max = 6 } = {}) {
+        const items = (Array.isArray(results) ? results : []).slice(0, max);
+        if (!items.length) return '';
+        const lines = items.map((r, i) => {
+            const title = String(r.title || '').trim() || WebAnswer._hostOf(r.url) || 'Untitled';
+            const date = r.date ? ` _(${r.date})_` : '';
+            const desc = String(r.description || '').replace(/\s+/g, ' ').trim();
+            return `${i + 1}. *${title}*${date}${desc ? `: ${desc}` : ''}`;
+        });
+        return `Here is what I found:\n\n${lines.join('\n')}`;
+    }
+
+    /** @private */
+    static _hostOf(url) {
+        try {
+            // `URL` above is the regex source; use the WHATWG constructor.
+            return new globalThis.URL(String(url)).hostname.replace(/^www\./, '');
+        } catch {
+            return '';
+        }
+    }
+
+    /**
+     * Turn `[2]` / `[2, 3]` / `[2][3]` citation markers into nothing, and
+     * report which result numbers were actually cited (1-based).
+     * WhatsApp readers get clean prose; the numbers decide the order of the
+     * sources block.
+     */
+    static extractCitations(text, count) {
+        const cited = [];
+        const out = String(text ?? '')
+            // "[2]", "[2, 3]", "[2-4]", "[Source 2]", "(source: 2)", "(result 3)", "[refs 1, 2]"
+            .replace(/\s*[[(](?:(?:sources?|results?|refs?|references?)\s*:?\s*)?(\d{1,2}(?:\s*[,;–-]\s*\d{1,2})*)[\])]/gi, (m, list) => {
+                // "(2024)" / "(12)" style numbers in prose are not citations: only
+                // bare square brackets or an explicit "source/result" word count.
+                if (m.trim().startsWith('(') && !/^\(\s*(?:sources?|results?|refs?|references?)/i.test(m.trim())) return m;
+                for (const part of list.split(/\s*[,;]\s*/)) {
+                    const range = part.match(/^(\d{1,2})\s*[–-]\s*(\d{1,2})$/);
+                    const nums = range ? WebAnswer._range(Number(range[1]), Number(range[2])) : [Number(part)];
+                    for (const n of nums) if (n >= 1 && n <= count && !cited.includes(n)) cited.push(n);
+                }
+                return '';
+            })
+            .replace(/[ \t]+([.,;:!?])/g, '$1')
+            .replace(/[ \t]{2,}/g, ' ');
+        return { text: out, cited };
+    }
+
+    /**
+     * Grounded replies must not contain URLs the model typed itself. A URL
+     * that matches one of the search results becomes a citation marker for
+     * it; any other URL is removed (with its "(see …)" wrapper).
+     */
+    static stripUrls(text, results) {
+        const keys = new Map();
+        (Array.isArray(results) ? results : []).forEach((r, i) => {
+            if (r && r.url) keys.set(WebAnswer.urlKey(WebAnswer.cleanUrl(r.url) || r.url), i + 1);
+        });
+        const swap = (url) => {
+            const n = keys.get(WebAnswer.urlKey(WebAnswer.cleanUrl(url) || url));
+            return n ? ` [${n}]` : '';
+        };
+        return String(text ?? '')
+            // "(see https://…)", "(source: https://…, https://…)", "(https://…)"
+            .replace(new RegExp(`\\s*\\((?:[^()\\n]{0,20}?:?\\s*)?(${URL}(?:\\s*[,;]\\s*${URL})*)\\s*\\)`, 'gi'), (_m, urls) =>
+                String(urls)
+                    .split(/\s*[,;]\s*/)
+                    .map(swap)
+                    .join('')
+            )
+            // "Title (https://…)" style leftovers and bare URLs; sentence punctuation after the URL survives
+            .replace(new RegExp(`\\s*(?:\\b(?:at|see|via|from|source|link|read more(?: at)?):?\\s+)?(${URL})`, 'gi'), (_m, url) => {
+                const trail = url.match(/[.,;:!?]+$/);
+                return swap(trail ? url.slice(0, -trail[0].length) : url) + (trail ? trail[0] : '');
+            })
+            .replace(/[ \t]+([.,;:!?])/g, '$1')
+            .replace(/[ \t]{2,}/g, ' ');
+    }
+
+    /** @private */
+    static _range(a, b) {
+        const out = [];
+        if (b < a) [a, b] = [b, a];
+        for (let n = a; n <= b && out.length < 30; n++) out.push(n);
+        return out;
+    }
+
     /**
      * Split a formatted reply into the prose answer and the sources it listed.
      *
@@ -339,14 +468,16 @@ class WebAnswer {
      * @param {string} question
      * @param {number} words   how long the first attempt was
      */
-    static expandPrompt(question, words) {
+    static expandPrompt(question, words, { grounded = false } = {}) {
         return (
             `Your reply was only ${words} words and did not follow the required layout. Rewrite it in full now for ` +
             `the topic "${String(question ?? '').trim()}": one or two intro sentences, then 3 to 5 sections — each a ` +
             'bold *Heading:* line followed by 3 to 4 numbered points written as *Headline*: two sentences of specific, ' +
-            'current facts — at least 300 words in total, then the "Sources:" list with one "title (url)" per line. ' +
-            'Use your web search tool for current details. Output only the report: no apology, no preamble, no notes ' +
-            'about your abilities or training data.'
+            'current facts — at least 300 words in total' +
+            (grounded
+                ? ', built from the search results above with the result number in square brackets after each fact. Do not write URLs or a Sources list. '
+                : ', then the "Sources:" list with one "title (url)" per line. Use your web search tool for current details. ') +
+            'Output only the report: no apology, no preamble, no notes about your abilities or training data.'
         );
     }
 

package/src/services/WebSearch.js

@@ -0,0 +1,428 @@
+'use strict';
+
+/**
+ * WebSearch
+ * ---------
+ * The engine's own web search, used by `AlexaAI.searchWeb()`.
+ *
+ * WHY THIS EXISTS
+ * ---------------
+ * DeepAI's chat endpoint accepts `web_access_enabled` / `search` flags, but on
+ * the models available to free keys the search runs unreliably — often not at
+ * all. Observed live with `gpt-4o-mini`: two consecutive requests about the
+ * same topic produced two contradictory "reports", one without any sources
+ * and one whose sources were invented URLs. A model cannot be asked to be its
+ * own source of truth.
+ *
+ * So the engine searches first, using free public endpoints that need no API
+ * key, and gives the results to the model as material to write from. The
+ * URLs the bot shows come from these results, never from the model.
+ *
+ * Providers (all run in parallel, each with its own timeout):
+ *
+ *   bing         general web results   bing.com/search?format=rss
+ *   bing-news    recent news           bing.com/news RSS
+ *   wikipedia    background            en.wikipedia.org API
+ *   google-news  recent news           news.google.com RSS
+ *   duckduckgo   general web results   lite.duckduckgo.com, then html.duckduckgo.com
+ *
+ * Verified live on 2026-09-06: the four feeds above answered from a
+ * data-centre IP; DuckDuckGo's html endpoint served a bot challenge while
+ * the lite endpoint answered, hence the order.
+ *
+ * A host application with a proper search API (Brave, Serper, Tavily, …)
+ * can replace the built-ins with `webSearchProvider: async (query) => results`
+ * in the constructor options, or pass `results` straight into `searchWeb()`.
+ *
+ * Every provider is best-effort: a failure or an empty page yields no results
+ * from that provider and never throws out of `search()`.
+ */
+
+const PROVIDERS = ['bing', 'bing-news', 'wikipedia', 'google-news', 'duckduckgo'];
+
+/**
+ * Hosts that are redirectors or ads, never a source. `news.google.com`
+ * article links are kept: they are real (redirecting) links to the story.
+ */
+const JUNK_HOST = /(?:^|\.)(?:duckduckgo\.com|bing\.com|googleadservices\.com|doubleclick\.net)$|^(?:www\.)?google\.com$/i;
+
+/** DuckDuckGo's "bots use DuckDuckGo too" interstitial. */
+const DDG_CHALLENGE = /bots use duckduckgo|anomaly-modal|class="anomaly/i;
+
+const ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ', ndash: '–', mdash: '—', hellip: '…', rsquo: '’', lsquo: '‘', rdquo: '”', ldquo: '“' };
+
+class WebSearch {
+    /**
+     * @param {object} [config]
+     * @param {boolean} [config.webSearch=true]
+     * @param {string[]} [config.webSearchProviders]
+     * @param {number} [config.webSearchTimeout=8000]     per provider, ms
+     * @param {number} [config.webSearchResults=8]        results handed to the model
+     * @param {Function} [config.webSearchProvider]       custom `(query, opts) => results`
+     * @param {string} [config.userAgent]
+     * @param {object} [config.logger]
+     * @param {Function} [config.fetch]                   injectable for tests
+     */
+    constructor(config = {}) {
+        this.enabled = config.webSearch !== false;
+        this.providers = Array.isArray(config.webSearchProviders) && config.webSearchProviders.length ? config.webSearchProviders : PROVIDERS;
+        this.timeout = Number.isFinite(config.webSearchTimeout) ? config.webSearchTimeout : 8000;
+        this.maxResults = Number.isFinite(config.webSearchResults) ? config.webSearchResults : 8;
+        this.custom = typeof config.webSearchProvider === 'function' ? config.webSearchProvider : null;
+        this.userAgent =
+            config.userAgent ||
+            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36';
+        this.log = config.logger || console;
+        this.debug = Boolean(config.debug);
+        this._fetch = config.fetch || ((...args) => fetch(...args));
+    }
+
+    static get PROVIDERS() {
+        return [...PROVIDERS];
+    }
+
+    /**
+     * @param {string} query
+     * @param {object} [opts]
+     * @param {string[]} [opts.providers]
+     * @param {number} [opts.maxResults]
+     * @param {AbortSignal} [opts.signal]
+     * @returns {Promise<{results: Array<{title:string|null,url:string,description:string|null,date:string|null,provider:string}>, providers: string[], errors: Array<{provider:string,message:string}>}>}
+     */
+    async search(query, opts = {}) {
+        const q = String(query ?? '').trim();
+        const limit = Number.isFinite(opts.maxResults) ? Math.max(1, opts.maxResults) : this.maxResults;
+        const empty = { results: [], providers: [], errors: [] };
+        if (!q || !this.enabled) return empty;
+
+        if (this.custom) {
+            try {
+                const list = await this.custom(q, { maxResults: limit, signal: opts.signal });
+                const results = WebSearch.normalise(list, 'custom').slice(0, limit);
+                return { results, providers: results.length ? ['custom'] : [], errors: [] };
+            } catch (err) {
+                return { ...empty, errors: [{ provider: 'custom', message: err.message }] };
+            }
+        }
+
+        const names = (Array.isArray(opts.providers) && opts.providers.length ? opts.providers : this.providers).filter((n) =>
+            PROVIDERS.includes(n)
+        );
+        const settled = await Promise.allSettled(names.map((name) => this._run(name, q, opts.signal)));
+
+        const perProvider = [];
+        const providers = [];
+        const errors = [];
+        settled.forEach((outcome, i) => {
+            const name = names[i];
+            if (outcome.status === 'fulfilled' && outcome.value.length) {
+                perProvider.push(outcome.value);
+                providers.push(name);
+            } else if (outcome.status === 'rejected') {
+                errors.push({ provider: name, message: outcome.reason?.message || String(outcome.reason) });
+                if (this.debug) this.log.debug?.(`[AlexaAI] web search ${name} failed: ${errors[errors.length - 1].message}`);
+            }
+        });
+
+        return { results: WebSearch.interleave(perProvider, limit), providers, errors };
+    }
+
+    /** @private */
+    async _run(name, query, signal) {
+        switch (name) {
+            case 'bing':
+                return WebSearch.parseRss(
+                    await this._get(`https://www.bing.com/search?q=${encodeURIComponent(query)}&format=rss&setlang=en`, signal),
+                    'bing'
+                );
+            case 'duckduckgo':
+                return this._duckduckgo(query, signal);
+            case 'bing-news':
+                return WebSearch.parseRss(
+                    await this._get(`https://www.bing.com/news/search?q=${encodeURIComponent(query)}&format=rss&setlang=en`, signal),
+                    'bing-news'
+                );
+            case 'google-news':
+                return WebSearch.parseRss(
+                    await this._get(
+                        `https://news.google.com/rss/search?q=${encodeURIComponent(query)}&hl=en-US&gl=US&ceid=US:en`,
+                        signal
+                    ),
+                    'google-news'
+                );
+            case 'wikipedia':
+                return WebSearch.parseWikipedia(
+                    await this._get(
+                        'https://en.wikipedia.org/w/api.php?action=query&list=search&format=json&utf8=1&srlimit=3' +
+                            `&srprop=snippet%7Ctimestamp&srsearch=${encodeURIComponent(query)}`,
+                        signal,
+                        { 'user-agent': 'alexa-ai (https://github.com/AlexaInc/deepai)', accept: 'application/json' }
+                    )
+                );
+            default:
+                return [];
+        }
+    }
+
+    /** @private lite endpoint first, html endpoint when it yields nothing. */
+    async _duckduckgo(query, signal) {
+        const q = encodeURIComponent(query);
+        let results = [];
+        try {
+            results = WebSearch.parseDuckDuckGo(await this._get(`https://lite.duckduckgo.com/lite/?q=${q}&kl=wt-wt`, signal));
+        } catch (err) {
+            if (this.debug) this.log.debug?.(`[AlexaAI] duckduckgo lite failed: ${err.message}`);
+        }
+        if (!results.length) {
+            results = WebSearch.parseDuckDuckGo(await this._get(`https://html.duckduckgo.com/html/?q=${q}&kl=wt-wt`, signal));
+        }
+        return results;
+    }
+
+    /** @private GET with a timeout; rejects on HTTP errors. */
+    async _get(url, signal, headers = {}) {
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), this.timeout);
+        if (signal) {
+            if (signal.aborted) controller.abort();
+            else signal.addEventListener?.('abort', () => controller.abort(), { once: true });
+        }
+        try {
+            const response = await this._fetch(url, {
+                method: 'GET',
+                redirect: 'follow',
+                signal: controller.signal,
+                headers: {
+                    'user-agent': this.userAgent,
+                    accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+                    'accept-language': 'en-US,en;q=0.8',
+                    ...headers,
+                },
+            });
+            const body = await response.text();
+            if (response.status > 299) throw new Error(`HTTP ${response.status}`);
+            return body;
+        } catch (err) {
+            if (err.name === 'AbortError') throw new Error(`timed out after ${this.timeout}ms`);
+            throw err;
+        } finally {
+            clearTimeout(timer);
+        }
+    }
+
+    // ------------------------------------------------------------ parsers --
+
+    /** html.duckduckgo.com and lite.duckduckgo.com result pages. */
+    static parseDuckDuckGo(html) {
+        const page = String(html ?? '');
+        const results = [];
+        const seen = new Set();
+        if (DDG_CHALLENGE.test(page)) return results;
+
+        // html endpoint: <a class="result__a" href="…">Title</a> … <a class="result__snippet" …>snippet</a>
+        // lite endpoint: <a rel="nofollow" href="…" class='result-link'>Title</a> … <td class='result-snippet'>snippet</td>
+        const anchor = /<a\b[^>]*class=["'](?:result__a|result-link)["'][^>]*href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>|<a\b[^>]*href=["']([^"']+)["'][^>]*class=["'](?:result__a|result-link)["'][^>]*>([\s\S]*?)<\/a>/gi;
+        const matches = [];
+        let m;
+        while ((m = anchor.exec(page))) {
+            matches.push({ href: m[1] || m[3], title: m[2] || m[4], index: m.index, end: anchor.lastIndex });
+        }
+
+        matches.forEach((hit, i) => {
+            const url = WebSearch.unwrapRedirect(hit.href);
+            if (!url || WebSearch.isJunk(url)) return;
+            const key = url.replace(/\/+$/, '').toLowerCase();
+            if (seen.has(key)) return;
+            seen.add(key);
+
+            // The snippet sits between this anchor and the next one.
+            const segment = page.slice(hit.end, matches[i + 1] ? matches[i + 1].index : hit.end + 4000);
+            const snippet =
+                segment.match(/<a\b[^>]*class=["']result__snippet["'][^>]*>([\s\S]*?)<\/a>/i) ||
+                segment.match(/<td\b[^>]*class=["']result-snippet["'][^>]*>([\s\S]*?)<\/td>/i);
+
+            results.push({
+                title: WebSearch.text(hit.title) || null,
+                url,
+                description: snippet ? WebSearch.text(snippet[1]) || null : null,
+                date: null,
+                provider: 'duckduckgo',
+            });
+        });
+        return results;
+    }
+
+    /** RSS 2.0 from Bing News / Google News. */
+    static parseRss(xml, provider = 'rss') {
+        const feed = String(xml ?? '');
+        const results = [];
+        const items = feed.match(/<item\b[\s\S]*?<\/item>/gi) || [];
+        for (const item of items) {
+            let url = WebSearch.tag(item, 'link') || (item.match(/<link\b[^>]*href=["']([^"']+)["']/i) || [])[1] || '';
+            url = WebSearch.unwrapRedirect(WebSearch.decodeEntities(url).trim());
+            if (!url || !/^https?:\/\//i.test(url) || WebSearch.isJunk(url)) continue;
+
+            let title = WebSearch.text(WebSearch.tag(item, 'title'));
+            const source = WebSearch.text(WebSearch.tag(item, 'source') || WebSearch.tag(item, 'News:Source'));
+            // Google News: "Headline - Publisher"
+            if (source && title && title.toLowerCase().endsWith(` - ${source.toLowerCase()}`)) {
+                title = title.slice(0, -(source.length + 3)).trim();
+            }
+            let description = WebSearch.text(WebSearch.tag(item, 'description'));
+            if (description && title && description.toLowerCase().startsWith(title.toLowerCase())) {
+                // Google's description is the headline again plus the publisher.
+                const rest = description.slice(title.length).replace(/^[\s\-–—|:]+/, '').trim();
+                description = rest && rest.toLowerCase() !== (source || '').toLowerCase() ? rest : null;
+            }
+            if (source && !description) description = source;
+            else if (source && description && !description.toLowerCase().includes(source.toLowerCase())) description = `${source}: ${description}`;
+
+            results.push({
+                title: title || null,
+                url,
+                description: description || null,
+                date: WebSearch.isoDate(WebSearch.tag(item, 'pubDate')),
+                provider,
+            });
+        }
+        return results;
+    }
+
+    /** MediaWiki `list=search` JSON. */
+    static parseWikipedia(json) {
+        let data = json;
+        if (typeof json === 'string') {
+            try {
+                data = JSON.parse(json);
+            } catch {
+                return [];
+            }
+        }
+        const hits = data?.query?.search;
+        if (!Array.isArray(hits)) return [];
+        return hits
+            .filter((h) => h && typeof h.title === 'string' && !/\(disambiguation\)$/i.test(h.title))
+            .map((h) => ({
+                title: `${h.title} - Wikipedia`,
+                url: `https://en.wikipedia.org/wiki/${encodeURIComponent(h.title.replace(/ /g, '_'))}`,
+                description: WebSearch.text(h.snippet) || null,
+                date: WebSearch.isoDate(h.timestamp),
+                provider: 'wikipedia',
+            }));
+    }
+
+    // ------------------------------------------------------------ helpers --
+
+    /** Round-robin across providers so news does not crowd out background, capped at `limit`. */
+    static interleave(lists, limit) {
+        const out = [];
+        const seen = new Set();
+        const queues = lists.map((l) => [...l]);
+        while (out.length < limit && queues.some((q) => q.length)) {
+            for (const q of queues) {
+                while (q.length) {
+                    const item = q.shift();
+                    const key = WebSearch.urlKey(item.url);
+                    const titleKey = item.title ? item.title.toLowerCase().replace(/\s+-\s+[^-]+$/, '').trim() : null;
+                    if (seen.has(key) || (titleKey && seen.has(`t:${titleKey}`))) continue;
+                    seen.add(key);
+                    if (titleKey) seen.add(`t:${titleKey}`);
+                    out.push(item);
+                    break;
+                }
+                if (out.length >= limit) break;
+            }
+        }
+        return out;
+    }
+
+    /** Accept caller-supplied results in loose shapes. */
+    static normalise(list, provider = 'custom') {
+        if (!Array.isArray(list)) return [];
+        return list
+            .map((r) => {
+                if (typeof r === 'string') return { title: null, url: r, description: null, date: null, provider };
+                if (!r || typeof r !== 'object') return null;
+                const url = r.url || r.link || r.href;
+                if (typeof url !== 'string' || !/^https?:\/\//i.test(url)) return null;
+                return {
+                    title: r.title || r.name || null,
+                    url,
+                    description: r.description || r.snippet || r.content || r.summary || null,
+                    date: WebSearch.isoDate(r.date || r.published || r.pubDate || r.publishedAt) || null,
+                    provider: r.provider || provider,
+                };
+            })
+            .filter(Boolean);
+    }
+
+    /** `//duckduckgo.com/l/?uddg=<url>` and `bing.com/news/apiclick.aspx?…&url=<url>` → the real URL. */
+    static unwrapRedirect(href) {
+        let url = String(href ?? '').trim();
+        if (!url) return '';
+        if (url.startsWith('//')) url = `https:${url}`;
+        try {
+            const u = new URL(url);
+            if (/(?:^|\.)duckduckgo\.com$/i.test(u.hostname)) {
+                const inner = u.searchParams.get('uddg') || u.searchParams.get('u3');
+                if (inner) return WebSearch.unwrapRedirect(inner);
+            }
+            if (/(?:^|\.)bing\.com$/i.test(u.hostname)) {
+                const inner = u.searchParams.get('url') || u.searchParams.get('r');
+                if (inner) return WebSearch.unwrapRedirect(inner);
+            }
+            return u.href;
+        } catch {
+            return '';
+        }
+    }
+
+    static isJunk(url) {
+        try {
+            return JUNK_HOST.test(new URL(url).hostname);
+        } catch {
+            return true;
+        }
+    }
+
+    static urlKey(url) {
+        return String(url ?? '')
+            .trim()
+            .toLowerCase()
+            .replace(/^https?:\/\//, '')
+            .replace(/^www\./, '')
+            .replace(/[#?].*$/, '')
+            .replace(/\/+$/, '');
+    }
+
+    /** First `<tag>…</tag>` in a block, CDATA unwrapped, raw. */
+    static tag(block, name) {
+        const re = new RegExp(`<${name}(?:\\s[^>]*)?>([\\s\\S]*?)<\\/${name}>`, 'i');
+        const m = String(block ?? '').match(re);
+        if (!m) return '';
+        return m[1].replace(/^\s*<!\[CDATA\[([\s\S]*?)\]\]>\s*$/, '$1');
+    }
+
+    /** Entity-decode, strip tags, collapse whitespace. Handles double-encoded HTML in RSS descriptions. */
+    static text(raw) {
+        let s = WebSearch.decodeEntities(String(raw ?? ''));
+        s = s.replace(/<[^>]+>/g, ' ');
+        s = WebSearch.decodeEntities(s);
+        return s.replace(/\s+/g, ' ').replace(/\s+([…,.;:!?])/g, '$1').trim();
+    }
+
+    static decodeEntities(s) {
+        return String(s ?? '')
+            .replace(/&#x([0-9a-f]+);/gi, (_m, h) => String.fromCodePoint(parseInt(h, 16)))
+            .replace(/&#(\d+);/g, (_m, d) => String.fromCodePoint(Number(d)))
+            .replace(/&([a-z]+);/gi, (m, name) => ENTITIES[name.toLowerCase()] ?? m);
+    }
+
+    static isoDate(value) {
+        if (!value) return null;
+        const d = new Date(String(value).trim());
+        return Number.isNaN(d.getTime()) ? null : d.toISOString().slice(0, 10);
+    }
+}
+
+module.exports = WebSearch;