@yeaft/webchat-agent 0.1.664 → 0.1.666

package/unify/memory/consolidate.js

@@ -1,22 +1,19 @@
 /**
- * consolidate.js — Consolidate = compact + extract (one LLM call)
+ * consolidate.js — Hot-window budget partitioning utilities.
  *
- * Triggered when hot_tokens > MESSAGE_TOKEN_BUDGET.
- * One LLM call does two things simultaneously:
- *   1. Generate compact summary → append to compact.md ("short-term memory")
- *   2. Extract memory entries → write to entries/ ("long-term memory")
+ * Reduced surface (PR-B rip): the legacy LLM-driven consolidate() pipeline
+ * (compact summary + entries-store extraction) has been retired. The only
+ * survivors are the pure functions used by the compact orchestrator:
  *
- * After consolidation:
- *   - Processed messages moved from messages/ to cold/
- *   - index.md + scopes.md updated
+ *   - shouldConsolidate(store, budget) — decide when to compact
+ *   - partitionMessages(messages, budget) — split hot messages into
+ *     toArchive / toKeep based on token budget
  *
- * Reference: yeaft-unify-core-systems.md §3.1, §4.2
- *            yeaft-unify-design.md §6.1
+ * Memory extraction is now owned by Dream V2 (per-group diff -> triage ->
+ * merge by target scope -> apply via segment-store + summary-store).
+ * Conversation summarisation lives in compact/orchestrator.js's hooks.
  */
 
-import { extractMemories } from './extract.js';
-import { pickEffort } from '../effort.js';
-
 // ─── Constants ──────────────────────────────────────────────────
 
 /** Default MESSAGE_TOKEN_BUDGET (context * 4%, default ~8192). */
@@ -28,8 +25,6 @@ export const COMPACT_KEEP_RATIO = 0.4;
 /** Minimum messages to keep hot (newest). */
 const MIN_KEEP_MESSAGES = 3;
 
-// ─── Consolidate ────────────────────────────────────────────────
-
 /**
  * Check if consolidation should be triggered.
  *
@@ -81,113 +76,3 @@ export function partitionMessages(messages, budget = DEFAULT_MESSAGE_TOKEN_BUDGE
     toKeep: messages.slice(keepStart),
   };
 }
-
-/**
- * Generate a compact summary of messages.
- *
- * @param {object[]} messages — messages to summarize
- * @param {object} adapter — LLM adapter with .call()
- * @param {object} config — { model }
- * @returns {Promise<string>} — compact summary text
- */
-async function generateSummary(messages, adapter, config) {
-  const conversation = messages.map(m => {
-    const prefix = m.role === 'user' ? 'User' : m.role === 'assistant' ? 'Assistant' : m.role;
-    return `[${prefix}]: ${(m.content || '').slice(0, 500)}`;
-  }).join('\n\n');
-
-  const system = 'You are a conversation summarizer. Summarize the conversation concisely in 2-3 paragraphs, preserving key decisions, facts, and context. Write in the same language as the conversation.';
-
-  try {
-    const result = await adapter.call({
-      model: config.model,
-      system,
-      messages: [{ role: 'user', content: `Summarize this conversation:\n\n${conversation}` }],
-      maxTokens: 1024,
-      // task-327c: consolidate is a high-complexity side-query; flag as
-      // 'max' effort so supported models use extended thinking / reasoning.
-      // Router/adapter silently drops the param for models that don't
-      // support thinking, or when UNIFY_THINKING_V1 is off.
-      effort: pickEffort({ scenario: 'consolidate' }),
-    });
-    return result.text.trim();
-  } catch {
-    // Fallback: simple concatenation of first/last messages
-    const first = messages[0]?.content?.slice(0, 200) || '';
-    const last = messages[messages.length - 1]?.content?.slice(0, 200) || '';
-    return `[Auto-summary failed] Started with: ${first}... Ended with: ${last}`;
-  }
-}
-
-/**
- * Run the full Consolidate pipeline.
- *
- * 1. Partition messages (what to archive vs keep)
- * 2. Generate compact summary (LLM call)
- * 3. Extract memory entries (LLM call)
- * 4. Move archived messages to cold/
- * 5. Update compact.md, index.md, scopes.md
- *
- * @param {{
- *   conversationStore: import('../conversation/persist.js').ConversationStore,
- *   memoryStore: import('./store.js').MemoryStore,
- *   adapter: object,
- *   config: object,
- *   budget?: number
- * }} params
- * @returns {Promise<{ compactSummary: string, extractedEntries: string[], archivedCount: number }>}
- */
-export async function consolidate({ conversationStore, memoryStore, adapter, config, budget = DEFAULT_MESSAGE_TOKEN_BUDGET }) {
-  // Load all hot messages
-  const messages = conversationStore.loadAll();
-
-  if (messages.length <= MIN_KEEP_MESSAGES) {
-    return { compactSummary: '', extractedEntries: [], archivedCount: 0 };
-  }
-
-  // Step 1: Partition
-  const { toArchive, toKeep } = partitionMessages(messages, budget);
-
-  if (toArchive.length === 0) {
-    return { compactSummary: '', extractedEntries: [], archivedCount: 0 };
-  }
-
-  // Step 2: Generate compact summary
-  const compactSummary = await generateSummary(toArchive, adapter, config);
-
-  // Step 3: Extract memory entries
-  const extracted = await extractMemories({ messages: toArchive, adapter, config });
-
-  // Step 4: Move archived messages to cold
-  const archiveIds = toArchive.map(m => m.id).filter(Boolean);
-  conversationStore.moveToColdBatch(archiveIds);
-
-  // Step 5a: Update compact.md
-  if (compactSummary) {
-    conversationStore.updateCompactSummary(compactSummary);
-  }
-
-  // Step 5b: Write extracted memory entries
-  const entryNames = [];
-  for (const entry of extracted) {
-    const slug = memoryStore.writeEntry(entry);
-    entryNames.push(slug);
-  }
-
-  // Step 5c: Update index.md
-  const lastMsg = toKeep[toKeep.length - 1];
-  conversationStore.updateIndex({
-    lastMessageId: lastMsg?.id || null,
-  });
-
-  // Step 5d: Rebuild scopes.md
-  if (entryNames.length > 0) {
-    memoryStore.rebuildScopes();
-  }
-
-  return {
-    compactSummary,
-    extractedEntries: entryNames,
-    archivedCount: archiveIds.length,
-  };
-}

package/unify/prompts.js

@@ -368,27 +368,11 @@ export function buildSystemPrompt({
   }
 
   // ─── 6. Memory Section ─────────────────────────────────
+  // FTS5 pre-flow recall + AMS snapshot are concatenated upstream by the
+  // engine into a single `memoryInjection` block. The legacy entries-based
+  // memory.profile / memory.entries shape was retired in the H2-AMS rip.
   if (memoryInjection && memoryInjection.trim()) {
-    // New path (task-287): prebuilt injection from memory/layout.buildMemoryInjection()
-    // Contains index.md + user-preferences.md + optional project header excerpt.
     parts.push(memoryInjection.trim());
-  } else if (memory && (memory.profile || (memory.entries && memory.entries.length > 0))) {
-    // Legacy path — kept for callers (tests, CLI) that have not migrated yet.
-    const memoryParts = [lang.memoryHeader];
-
-    if (memory.profile) {
-      memoryParts.push(`${lang.profileHeader}\n${memory.profile}`);
-    }
-
-    if (memory.entries && memory.entries.length > 0) {
-      const entryLines = memory.entries.map(e => {
-        const tags = (e.tags && e.tags.length > 0) ? ` [${e.tags.join(', ')}]` : '';
-        return `- **${e.name}** (${e.kind}): ${e.content}${tags}`;
-      });
-      memoryParts.push(`${lang.recalledHeader}\n${entryLines.join('\n')}`);
-    }
-
-    parts.push(memoryParts.join('\n\n'));
   }
 
   // ─── 7. Compact Summary Section ────────────────────────

package/unify/session.js

@@ -5,7 +5,7 @@
  *
  * Wires all subsystems together:
  *   initYeaftDir → loadConfig → createTrace → createLLMAdapter →
- *   ConversationStore → MemoryStore → SkillManager → MCPManager →
+ *   ConversationStore → SkillManager → MCPManager →
  *   ToolRegistry → Engine → Session
  *
  * The ~/.yeaft/ directory is the agent's persistent workspace.
@@ -18,8 +18,6 @@ import { loadConfig, loadMCPConfig } from './config.js';
 import { createTrace } from './debug-trace.js';
 import { createLLMAdapter } from './llm/adapter.js';
 import { ConversationStore } from './conversation/persist.js';
-import { MemoryStore } from './memory/store.js';
-import { openMemoryShardStore } from './memory/shard-store.js';
 import { SkillManager, createSkillManager } from './skills.js';
 import { MCPManager } from './mcp.js';
 import { createFullRegistry } from './tools/index.js';
@@ -68,7 +66,6 @@ import { existsSync as existsSyncSafe, readFileSync as readFileSyncSafe } from '
  * @property {import('./llm/adapter.js').LLMAdapter} adapter — The LLM adapter
  * @property {object} config — Resolved configuration
  * @property {ConversationStore} conversationStore — Conversation persistence
- * @property {MemoryStore} memoryStore — Memory persistence
  * @property {SkillManager} skillManager — Skill manager
  * @property {MCPManager} mcpManager — MCP manager
  * @property {import('./tools/registry.js').ToolRegistry} toolRegistry — Tool registry
@@ -169,19 +166,6 @@ export async function loadSession(options = {}) {
 
   // ─── 5. Create stores ──────────────────────────────────
   const conversationStore = new ConversationStore(yeaftDir);
-  const memoryStore = new MemoryStore(yeaftDir);
-
-  // ─── 5-shard. Open R6 memory shard store (task-334f) ──────
-  //     VP-level memory shard store rooted at ~/.yeaft/memory/vp/default.
-  //     The 'default' VP matches the single-user Unify mode; R6 multi-VP
-  //     callers open per-VP stores via openMemoryShardStore() directly.
-  const memoryShardDir = join(yeaftDir, 'memory', 'vp', 'default');
-  let memoryShardStore = null;
-  try {
-    memoryShardStore = openMemoryShardStore(memoryShardDir, 'vp');
-  } catch (err) {
-    console.warn(`[Yeaft] Failed to open R6 memory shard store: ${err?.message || err}`);
-  }
 
   // ─── 5-fts. (GC.1) Open SegmentIndex for FTS pre-flow ────
   //     When config.memoryV2 is on, build a SQLite FTS5 index over
@@ -290,8 +274,6 @@ export async function loadSession(options = {}) {
     trace,
     config,
     conversationStore,
-    memoryStore,
-    memoryShardStore,
     memoryIndex,
     amsRegistry,
     toolRegistry,
@@ -363,8 +345,6 @@ export async function loadSession(options = {}) {
     adapter,
     config,
     conversationStore,
-    memoryStore,
-    memoryShardStore,
     dreamScheduler,
     skillManager,
     mcpManager,

package/unify/stop-hooks.js

@@ -3,17 +3,17 @@
  *
  * Runs after each query loop completes:
  *   1. Persist messages to conversation/messages/
- *   2. Consolidate check (compact + extract) — only when budget exceeded
+ *
+ * Consolidation (compact orchestrator) is driven by the engine itself
+ * via `#maybeConsolidate`; the legacy LLM-driven `consolidate()` plus
+ * entries-store extraction was retired in the H2-AMS rip.
  *
  * Dream V2 owns all background memory maintenance (scope summaries +
- * memory writes via dream-v2/session-wiring.js → createV2DreamScheduler);
- * the legacy `memory/dream.js` gate that used to fire here was retired
- * alongside recall-r6.
+ * memory writes via dream-v2/session-wiring.js → createV2DreamScheduler).
  *
  * Reference: yeaft-unify-core-systems.md §4.4
  */
 
-import { shouldConsolidate, consolidate } from './memory/consolidate.js';
 import { isPermissionError } from './init.js';
 
 /** Track whether we've already warned about permission issues in stop hooks. */
@@ -26,7 +26,6 @@ let _permissionWarned = false;
  *   yeaftDir: string,
  *   mode: string,
  *   conversationStore: import('./conversation/persist.js').ConversationStore,
- *   memoryStore: import('./memory/store.js').MemoryStore,
  *   adapter: object,
  *   config: object,
  *   primaryModel?: string,
@@ -42,7 +41,6 @@ export async function runStopHooks(context) {
     yeaftDir,
     mode,
     conversationStore,
-    memoryStore,
     adapter,
     config,
     primaryModel,
@@ -60,7 +58,6 @@ export async function runStopHooks(context) {
 
   const result = {
     messagesPersisted: 0,
-    consolidated: false,
     errors: [],
   };
 
@@ -133,41 +130,9 @@ export async function runStopHooks(context) {
     }
   }
 
-  // 2. Consolidate check (non-blocking, but awaited for correctness)
-  try {
-    if (conversationStore && memoryStore && adapter) {
-      if (shouldConsolidate(conversationStore, config.messageTokenBudget)) {
-        const consolidated = await consolidate({
-          conversationStore,
-          memoryStore,
-          adapter,
-          config,
-          budget: config.messageTokenBudget,
-        });
-        result.consolidated = true;
-        trace?.logEvent({
-          eventType: 'consolidate',
-          eventData: {
-            archivedCount: consolidated.archivedCount,
-            extractedEntries: consolidated.extractedEntries.length,
-          },
-        });
-      }
-    }
-  } catch (err) {
-    if (isPermissionError(err)) {
-      if (!_permissionWarned) {
-        result.errors.push('Cannot write to ~/.yeaft/ — consolidation skipped');
-        _permissionWarned = true;
-      }
-    } else {
-      result.errors.push(`Consolidate failed: ${err.message}`);
-    }
-  }
-
-  // 3. Dream V2 owns background scope-memory maintenance via the session
-  //    dream scheduler (createV2DreamScheduler). No legacy dream gate is
-  //    invoked here; the scheduler decides when to run on its own cadence.
+  // 2. Consolidation is owned by the engine (#maybeConsolidate → compact
+  //    orchestrator). Dream V2 owns background scope-memory maintenance
+  //    via the session dream scheduler (createV2DreamScheduler).
 
   return result;
 }
@@ -175,6 +140,5 @@ export async function runStopHooks(context) {
 /**
  * @typedef {Object} StopHookResult
  * @property {number} messagesPersisted — how many messages were persisted
- * @property {boolean} consolidated — whether consolidation ran
  * @property {string[]} errors — any non-fatal errors
  */

package/unify/tools/index.js

@@ -18,11 +18,6 @@ import exitWorktree from './exit-worktree.js';
 
 // --- P0 Core tools ---
 import askUser from './ask-user.js';
-import memoryRead from './memory-read.js';
-import memoryWrite from './memory-write.js';
-import memorySearch, { memorySearchAlias } from './memory-search.js';
-import memoryQuery from './memory-query.js';
-import memoryTrace from './memory-trace.js';
 import openSourceMessage from './open-source-message.js';
 import webSearch from './web-search.js';
 import webFetch from './web-fetch.js';
@@ -89,12 +84,6 @@ export const allTools = [
 
   // P0 Core
   askUser,
-  memoryRead,
-  memoryWrite,
-  memorySearch,
-  memorySearchAlias,
-  memoryQuery,
-  memoryTrace,
   openSourceMessage,
   webSearch,
   webFetch,

package/unify/tools/web-search.js

@@ -1,8 +1,24 @@
 /**
  * web-search.js — Web search tool.
  *
- * Delegates to an external search API or LLM-based web search.
- * Supports configurable search providers via Yeaft config.
+ * Strategy (in order):
+ *   1. Tavily API (default; configured via ~/.yeaft/config.json → search.tavilyApiKey)
+ *   2. Generic searchApiUrl (legacy; user-supplied JSON-returning endpoint)
+ *   3. HTML-scrape fallback: DuckDuckGo lite then Bing
+ *      (works on residential IPs; cloud IPs are usually flagged as bots)
+ *
+ * Config shape in ~/.yeaft/config.json:
+ *   {
+ *     "search": {
+ *       "tavilyApiKey": "tvly-...",
+ *       "searchApiUrl": "https://...",   // optional, alternative JSON endpoint
+ *       "disableHtmlFallback": false      // optional, opt-out of scraping
+ *     }
+ *   }
+ *
+ * The result is JSON-stringified so the LLM can parse it. We intentionally
+ * keep the output shape consistent across providers: { provider, query,
+ * answer?, results: [{title, url, snippet}] }.
  */
 
 import { defineTool } from './types.js';
@@ -36,44 +52,204 @@ Guidelines:
   isReadOnly: () => true,
   async execute(input, ctx) {
     const { query, limit = 5 } = input;
-    if (!query) return JSON.stringify({ error: 'query is required' });
-
-    try {
-      // Check if adapter supports web search natively (some LLM providers have built-in search)
-      const adapter = ctx?.adapter;
-      if (adapter && typeof adapter.webSearch === 'function') {
-        const results = await adapter.webSearch(query, limit);
-        return JSON.stringify(results, null, 2);
-      }
-
-      // Check config for search API endpoint
-      const searchUrl = ctx?.config?.searchApiUrl;
-      if (searchUrl) {
-        const url = new URL(searchUrl);
-        url.searchParams.set('q', query);
-        url.searchParams.set('limit', String(limit));
-
-        const response = await fetch(url.toString(), {
-          signal: ctx?.signal,
-          headers: { 'User-Agent': 'Yeaft/1.0' },
-        });
-
-        if (!response.ok) {
-          return JSON.stringify({ error: `Search API returned ${response.status}: ${response.statusText}` });
-        }
-
-        const data = await response.json();
-        return JSON.stringify(data, null, 2);
-      }
-
-      // Fallback: no search provider configured
-      return JSON.stringify({
-        error: 'No web search provider configured.',
-        hint: 'Configure searchApiUrl in ~/.yeaft/config.json or use an LLM provider with built-in search.',
-      });
-    } catch (err) {
-      if (err.name === 'AbortError') return JSON.stringify({ error: 'Search cancelled' });
-      return JSON.stringify({ error: `Web search failed: ${err.message}` });
+    if (!query || typeof query !== 'string') {
+      return JSON.stringify({ error: 'query is required' });
+    }
+
+    const search = ctx?.config?.search || {};
+    const signal = ctx?.signal;
+    const errors = [];
+
+    // 1. Tavily — default, fast, structured.
+    if (search.tavilyApiKey) {
+      const r = await tryTavily(query, limit, search.tavilyApiKey, signal);
+      if (r.ok) return JSON.stringify(r.data, null, 2);
+      errors.push(`tavily: ${r.error}`);
+    }
+
+    // 2. Generic JSON endpoint (legacy escape hatch — SearXNG, custom proxy, etc).
+    const genericUrl = search.searchApiUrl || ctx?.config?.searchApiUrl;
+    if (genericUrl) {
+      const r = await tryGenericApi(query, limit, genericUrl, signal);
+      if (r.ok) return JSON.stringify(r.data, null, 2);
+      errors.push(`searchApiUrl: ${r.error}`);
+    }
+
+    // 3. HTML-scrape fallback. Often blocked on cloud IPs; useful for
+    //    self-hosted / residential setups with no API key.
+    if (!search.disableHtmlFallback) {
+      const r = await tryHtmlScrape(query, limit, signal);
+      if (r.ok) return JSON.stringify(r.data, null, 2);
+      errors.push(`html: ${r.error}`);
     }
+
+    return JSON.stringify({
+      error: 'No web search backend succeeded.',
+      attempted: errors,
+      hint: 'Set search.tavilyApiKey in ~/.yeaft/config.json (free tier: https://tavily.com).',
+    });
   },
 });
+
+// ─── Backend implementations ────────────────────────────────────────
+
+async function tryTavily(query, limit, apiKey, signal) {
+  try {
+    const res = await fetch('https://api.tavily.com/search', {
+      method: 'POST',
+      signal,
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        api_key: apiKey,
+        query,
+        max_results: Math.max(1, Math.min(limit, 10)),
+        include_answer: true,
+        search_depth: 'basic',
+      }),
+    });
+    if (!res.ok) {
+      const text = await res.text().catch(() => '');
+      return { ok: false, error: `${res.status} ${res.statusText} ${text.slice(0, 200)}` };
+    }
+    const data = await res.json();
+    return {
+      ok: true,
+      data: {
+        provider: 'tavily',
+        query,
+        answer: data.answer || null,
+        results: (data.results || []).slice(0, limit).map((r) => ({
+          title: r.title,
+          url: r.url,
+          snippet: r.content,
+          score: r.score,
+        })),
+      },
+    };
+  } catch (err) {
+    if (err?.name === 'AbortError') return { ok: false, error: 'cancelled' };
+    return { ok: false, error: err.message || String(err) };
+  }
+}
+
+async function tryGenericApi(query, limit, urlStr, signal) {
+  try {
+    const url = new URL(urlStr);
+    url.searchParams.set('q', query);
+    url.searchParams.set('limit', String(limit));
+    const res = await fetch(url.toString(), {
+      signal,
+      headers: { 'User-Agent': 'Yeaft/1.0', Accept: 'application/json' },
+    });
+    if (!res.ok) return { ok: false, error: `${res.status} ${res.statusText}` };
+    const data = await res.json();
+    return { ok: true, data: { provider: 'generic', query, ...data } };
+  } catch (err) {
+    if (err?.name === 'AbortError') return { ok: false, error: 'cancelled' };
+    return { ok: false, error: err.message || String(err) };
+  }
+}
+
+/**
+ * HTML-scrape fallback. Tries DuckDuckGo's lite HTML endpoint first
+ * (smaller markup, but more aggressive bot detection on cloud IPs),
+ * then Bing. We intentionally keep the regex-based parsers minimal —
+ * they break less than full DOM selectors when sites tweak markup.
+ */
+async function tryHtmlScrape(query, limit, signal) {
+  const ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+  const headers = { 'User-Agent': ua, 'Accept-Language': 'en-US,en;q=0.9' };
+
+  try {
+    const ddg = await fetch(`https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`, { signal, headers });
+    if (ddg.ok) {
+      const html = await ddg.text();
+      const results = parseDdgHtml(html, limit);
+      if (results.length) return { ok: true, data: { provider: 'duckduckgo-html', query, results } };
+    }
+  } catch (err) {
+    if (err?.name === 'AbortError') return { ok: false, error: 'cancelled' };
+  }
+
+  try {
+    const bing = await fetch(`https://www.bing.com/search?q=${encodeURIComponent(query)}`, { signal, headers });
+    if (bing.ok) {
+      const html = await bing.text();
+      const results = parseBingHtml(html, limit);
+      if (results.length) return { ok: true, data: { provider: 'bing-html', query, results } };
+    }
+  } catch (err) {
+    if (err?.name === 'AbortError') return { ok: false, error: 'cancelled' };
+  }
+
+  return { ok: false, error: 'all HTML scrape backends returned 0 results (likely bot-blocked)' };
+}
+
+/**
+ * Parse DDG lite HTML. Each result is wrapped in
+ *   <a class="result__a" href="…">title</a>
+ *   <a class="result__snippet">snippet</a>
+ * Hash classes are not used here, so plain regex is fine.
+ */
+function parseDdgHtml(html, limit) {
+  const results = [];
+  const linkRe = /<a[^>]+class="result__a"[^>]+href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/g;
+  const snippetRe = /<a[^>]+class="result__snippet"[^>]*>([\s\S]*?)<\/a>/g;
+  const links = [...html.matchAll(linkRe)];
+  const snippets = [...html.matchAll(snippetRe)];
+  for (let i = 0; i < links.length && results.length < limit; i++) {
+    const url = decodeDdgUrl(links[i][1]);
+    const title = stripTags(links[i][2]).trim();
+    const snippet = snippets[i] ? stripTags(snippets[i][1]).trim() : '';
+    if (url && title) results.push({ title, url, snippet });
+  }
+  return results;
+}
+
+/**
+ * DDG often wraps outbound URLs in `/l/?uddg=…` redirects. Unwrap.
+ */
+function decodeDdgUrl(href) {
+  try {
+    if (href.startsWith('//')) href = 'https:' + href;
+    const u = new URL(href, 'https://duckduckgo.com');
+    const target = u.searchParams.get('uddg');
+    return target ? decodeURIComponent(target) : u.toString();
+  } catch {
+    return href;
+  }
+}
+
+/**
+ * Parse Bing search HTML. Result blocks: <li class="b_algo"> with
+ * <h2><a href="…">title</a></h2> and <p>snippet</p>. The class names
+ * have been stable for years; if Bing rotates them this will fail
+ * gracefully (no results extracted) and we'll surface the error upstream.
+ */
+function parseBingHtml(html, limit) {
+  const results = [];
+  const blockRe = /<li[^>]+class="[^"]*\bb_algo\b[^"]*"[^>]*>([\s\S]*?)<\/li>/g;
+  for (const m of html.matchAll(blockRe)) {
+    if (results.length >= limit) break;
+    const block = m[1];
+    const linkM = block.match(/<h2[^>]*>\s*<a[^>]+href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/);
+    if (!linkM) continue;
+    const url = linkM[1];
+    const title = stripTags(linkM[2]).trim();
+    const pM = block.match(/<p[^>]*>([\s\S]*?)<\/p>/);
+    const snippet = pM ? stripTags(pM[1]).trim() : '';
+    if (url && title) results.push({ title, url, snippet });
+  }
+  return results;
+}
+
+function stripTags(s) {
+  return s
+    .replace(/<[^>]+>/g, '')
+    .replace(/&amp;/g, '&')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/&nbsp;/g, ' ');
+}