@yeaft/webchat-agent 0.1.663 → 0.1.665

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yeaft/webchat-agent",
-  "version": "0.1.663",
+  "version": "0.1.665",
   "description": "Remote agent for Yeaft WebChat — connects worker machines to the central server",
   "main": "index.js",
   "type": "module",

package/unify/index.js

@@ -26,7 +26,6 @@ export { MemoryStore, parseEntry, serializeEntry, MEMORY_KINDS } from './memory/
 
 // Phase 5: Advanced features
 export { KINDS, KIND_PRIORITY, KIND_DESCRIPTIONS, IMPORTANCE_LEVELS, validateEntry, parseScopePath, getAncestorScopes, areScopesRelated } from './memory/types.js';
-export { scanEntries, scoreEntry, findStaleEntries, findDuplicateGroups, summarizeScan } from './memory/scan.js';
 export { runStopHooks } from './stop-hooks.js';
 export { MCPManager, createMCPManager } from './mcp.js';
 export { SkillManager, createSkillManager, parseSkill, serializeSkill } from './skills.js';

package/unify/init.js

@@ -79,7 +79,6 @@ const SUBDIRS = [
   'conversation/blobs',
   'memory/entries',
   'tasks',
-  'dream',
   'skills',
 ];
 

package/unify/tools/web-search.js

@@ -1,8 +1,24 @@
 /**
  * web-search.js — Web search tool.
  *
- * Delegates to an external search API or LLM-based web search.
- * Supports configurable search providers via Yeaft config.
+ * Strategy (in order):
+ *   1. Tavily API (default; configured via ~/.yeaft/config.json → search.tavilyApiKey)
+ *   2. Generic searchApiUrl (legacy; user-supplied JSON-returning endpoint)
+ *   3. HTML-scrape fallback: DuckDuckGo lite then Bing
+ *      (works on residential IPs; cloud IPs are usually flagged as bots)
+ *
+ * Config shape in ~/.yeaft/config.json:
+ *   {
+ *     "search": {
+ *       "tavilyApiKey": "tvly-...",
+ *       "searchApiUrl": "https://...",   // optional, alternative JSON endpoint
+ *       "disableHtmlFallback": false      // optional, opt-out of scraping
+ *     }
+ *   }
+ *
+ * The result is JSON-stringified so the LLM can parse it. We intentionally
+ * keep the output shape consistent across providers: { provider, query,
+ * answer?, results: [{title, url, snippet}] }.
  */
 
 import { defineTool } from './types.js';
@@ -36,44 +52,204 @@ Guidelines:
   isReadOnly: () => true,
   async execute(input, ctx) {
     const { query, limit = 5 } = input;
-    if (!query) return JSON.stringify({ error: 'query is required' });
-
-    try {
-      // Check if adapter supports web search natively (some LLM providers have built-in search)
-      const adapter = ctx?.adapter;
-      if (adapter && typeof adapter.webSearch === 'function') {
-        const results = await adapter.webSearch(query, limit);
-        return JSON.stringify(results, null, 2);
-      }
-
-      // Check config for search API endpoint
-      const searchUrl = ctx?.config?.searchApiUrl;
-      if (searchUrl) {
-        const url = new URL(searchUrl);
-        url.searchParams.set('q', query);
-        url.searchParams.set('limit', String(limit));
-
-        const response = await fetch(url.toString(), {
-          signal: ctx?.signal,
-          headers: { 'User-Agent': 'Yeaft/1.0' },
-        });
-
-        if (!response.ok) {
-          return JSON.stringify({ error: `Search API returned ${response.status}: ${response.statusText}` });
-        }
-
-        const data = await response.json();
-        return JSON.stringify(data, null, 2);
-      }
-
-      // Fallback: no search provider configured
-      return JSON.stringify({
-        error: 'No web search provider configured.',
-        hint: 'Configure searchApiUrl in ~/.yeaft/config.json or use an LLM provider with built-in search.',
-      });
-    } catch (err) {
-      if (err.name === 'AbortError') return JSON.stringify({ error: 'Search cancelled' });
-      return JSON.stringify({ error: `Web search failed: ${err.message}` });
+    if (!query || typeof query !== 'string') {
+      return JSON.stringify({ error: 'query is required' });
+    }
+
+    const search = ctx?.config?.search || {};
+    const signal = ctx?.signal;
+    const errors = [];
+
+    // 1. Tavily — default, fast, structured.
+    if (search.tavilyApiKey) {
+      const r = await tryTavily(query, limit, search.tavilyApiKey, signal);
+      if (r.ok) return JSON.stringify(r.data, null, 2);
+      errors.push(`tavily: ${r.error}`);
+    }
+
+    // 2. Generic JSON endpoint (legacy escape hatch — SearXNG, custom proxy, etc).
+    const genericUrl = search.searchApiUrl || ctx?.config?.searchApiUrl;
+    if (genericUrl) {
+      const r = await tryGenericApi(query, limit, genericUrl, signal);
+      if (r.ok) return JSON.stringify(r.data, null, 2);
+      errors.push(`searchApiUrl: ${r.error}`);
+    }
+
+    // 3. HTML-scrape fallback. Often blocked on cloud IPs; useful for
+    //    self-hosted / residential setups with no API key.
+    if (!search.disableHtmlFallback) {
+      const r = await tryHtmlScrape(query, limit, signal);
+      if (r.ok) return JSON.stringify(r.data, null, 2);
+      errors.push(`html: ${r.error}`);
     }
+
+    return JSON.stringify({
+      error: 'No web search backend succeeded.',
+      attempted: errors,
+      hint: 'Set search.tavilyApiKey in ~/.yeaft/config.json (free tier: https://tavily.com).',
+    });
   },
 });
+
+// ─── Backend implementations ────────────────────────────────────────
+
+async function tryTavily(query, limit, apiKey, signal) {
+  try {
+    const res = await fetch('https://api.tavily.com/search', {
+      method: 'POST',
+      signal,
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        api_key: apiKey,
+        query,
+        max_results: Math.max(1, Math.min(limit, 10)),
+        include_answer: true,
+        search_depth: 'basic',
+      }),
+    });
+    if (!res.ok) {
+      const text = await res.text().catch(() => '');
+      return { ok: false, error: `${res.status} ${res.statusText} ${text.slice(0, 200)}` };
+    }
+    const data = await res.json();
+    return {
+      ok: true,
+      data: {
+        provider: 'tavily',
+        query,
+        answer: data.answer || null,
+        results: (data.results || []).slice(0, limit).map((r) => ({
+          title: r.title,
+          url: r.url,
+          snippet: r.content,
+          score: r.score,
+        })),
+      },
+    };
+  } catch (err) {
+    if (err?.name === 'AbortError') return { ok: false, error: 'cancelled' };
+    return { ok: false, error: err.message || String(err) };
+  }
+}
+
+async function tryGenericApi(query, limit, urlStr, signal) {
+  try {
+    const url = new URL(urlStr);
+    url.searchParams.set('q', query);
+    url.searchParams.set('limit', String(limit));
+    const res = await fetch(url.toString(), {
+      signal,
+      headers: { 'User-Agent': 'Yeaft/1.0', Accept: 'application/json' },
+    });
+    if (!res.ok) return { ok: false, error: `${res.status} ${res.statusText}` };
+    const data = await res.json();
+    return { ok: true, data: { provider: 'generic', query, ...data } };
+  } catch (err) {
+    if (err?.name === 'AbortError') return { ok: false, error: 'cancelled' };
+    return { ok: false, error: err.message || String(err) };
+  }
+}
+
+/**
+ * HTML-scrape fallback. Tries DuckDuckGo's lite HTML endpoint first
+ * (smaller markup, but more aggressive bot detection on cloud IPs),
+ * then Bing. We intentionally keep the regex-based parsers minimal —
+ * they break less than full DOM selectors when sites tweak markup.
+ */
+async function tryHtmlScrape(query, limit, signal) {
+  const ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+  const headers = { 'User-Agent': ua, 'Accept-Language': 'en-US,en;q=0.9' };
+
+  try {
+    const ddg = await fetch(`https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`, { signal, headers });
+    if (ddg.ok) {
+      const html = await ddg.text();
+      const results = parseDdgHtml(html, limit);
+      if (results.length) return { ok: true, data: { provider: 'duckduckgo-html', query, results } };
+    }
+  } catch (err) {
+    if (err?.name === 'AbortError') return { ok: false, error: 'cancelled' };
+  }
+
+  try {
+    const bing = await fetch(`https://www.bing.com/search?q=${encodeURIComponent(query)}`, { signal, headers });
+    if (bing.ok) {
+      const html = await bing.text();
+      const results = parseBingHtml(html, limit);
+      if (results.length) return { ok: true, data: { provider: 'bing-html', query, results } };
+    }
+  } catch (err) {
+    if (err?.name === 'AbortError') return { ok: false, error: 'cancelled' };
+  }
+
+  return { ok: false, error: 'all HTML scrape backends returned 0 results (likely bot-blocked)' };
+}
+
+/**
+ * Parse DDG lite HTML. Each result is wrapped in
+ *   <a class="result__a" href="…">title</a>
+ *   <a class="result__snippet">snippet</a>
+ * Hash classes are not used here, so plain regex is fine.
+ */
+function parseDdgHtml(html, limit) {
+  const results = [];
+  const linkRe = /<a[^>]+class="result__a"[^>]+href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/g;
+  const snippetRe = /<a[^>]+class="result__snippet"[^>]*>([\s\S]*?)<\/a>/g;
+  const links = [...html.matchAll(linkRe)];
+  const snippets = [...html.matchAll(snippetRe)];
+  for (let i = 0; i < links.length && results.length < limit; i++) {
+    const url = decodeDdgUrl(links[i][1]);
+    const title = stripTags(links[i][2]).trim();
+    const snippet = snippets[i] ? stripTags(snippets[i][1]).trim() : '';
+    if (url && title) results.push({ title, url, snippet });
+  }
+  return results;
+}
+
+/**
+ * DDG often wraps outbound URLs in `/l/?uddg=…` redirects. Unwrap.
+ */
+function decodeDdgUrl(href) {
+  try {
+    if (href.startsWith('//')) href = 'https:' + href;
+    const u = new URL(href, 'https://duckduckgo.com');
+    const target = u.searchParams.get('uddg');
+    return target ? decodeURIComponent(target) : u.toString();
+  } catch {
+    return href;
+  }
+}
+
+/**
+ * Parse Bing search HTML. Result blocks: <li class="b_algo"> with
+ * <h2><a href="…">title</a></h2> and <p>snippet</p>. The class names
+ * have been stable for years; if Bing rotates them this will fail
+ * gracefully (no results extracted) and we'll surface the error upstream.
+ */
+function parseBingHtml(html, limit) {
+  const results = [];
+  const blockRe = /<li[^>]+class="[^"]*\bb_algo\b[^"]*"[^>]*>([\s\S]*?)<\/li>/g;
+  for (const m of html.matchAll(blockRe)) {
+    if (results.length >= limit) break;
+    const block = m[1];
+    const linkM = block.match(/<h2[^>]*>\s*<a[^>]+href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/);
+    if (!linkM) continue;
+    const url = linkM[1];
+    const title = stripTags(linkM[2]).trim();
+    const pM = block.match(/<p[^>]*>([\s\S]*?)<\/p>/);
+    const snippet = pM ? stripTags(pM[1]).trim() : '';
+    if (url && title) results.push({ title, url, snippet });
+  }
+  return results;
+}
+
+function stripTags(s) {
+  return s
+    .replace(/<[^>]+>/g, '')
+    .replace(/&amp;/g, '&')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/&nbsp;/g, ' ');
+}

package/unify/memory/scan.js

@@ -1,273 +0,0 @@
-/**
- * scan.js — Memory header scanning and scope/tag matching
- *
- * Fast in-memory scanning of entry frontmatter for:
- * - Scope tree traversal
- * - Tag overlap scoring
- * - Kind-based filtering
- * - Stale entry detection (for Dream)
- *
- * Reference: yeaft-unify-core-systems.md §3.3, yeaft-unify-design.md §5.1
- */
-
-import { KINDS, KIND_PRIORITY, IMPORTANCE_WEIGHT, getAncestorScopes } from './types.js';
-
-// ─── Scan Results ──────────────────────────────────────────
-
-/**
- * @typedef {Object} ScanResult
- * @property {object[]} entries — all parsed entries
- * @property {Map<string, number>} scopeCount — scope → entry count
- * @property {Map<string, number>} kindCount — kind → entry count
- * @property {Map<string, Set<string>>} tagIndex — tag → set of entry names
- * @property {number} totalEntries — total count
- */
-
-/**
- * Scan all entries from a MemoryStore and build indexes.
- *
- * @param {import('./store.js').MemoryStore} memoryStore
- * @returns {ScanResult}
- */
-export function scanEntries(memoryStore) {
-  const entries = memoryStore.listEntries();
-
-  const scopeCount = new Map();
-  const kindCount = new Map();
-  const tagIndex = new Map();
-
-  for (const entry of entries) {
-    // Scope count
-    const scope = entry.scope || 'global';
-    scopeCount.set(scope, (scopeCount.get(scope) || 0) + 1);
-
-    // Kind count
-    const kind = entry.kind || 'fact';
-    kindCount.set(kind, (kindCount.get(kind) || 0) + 1);
-
-    // Tag index
-    const tags = entry.tags || [];
-    for (const tag of tags) {
-      const lowerTag = tag.toLowerCase();
-      if (!tagIndex.has(lowerTag)) tagIndex.set(lowerTag, new Set());
-      tagIndex.get(lowerTag).add(entry.name);
-    }
-  }
-
-  return {
-    entries,
-    scopeCount,
-    kindCount,
-    tagIndex,
-    totalEntries: entries.length,
-  };
-}
-
-// ─── Scoring Functions ─────────────────────────────────────
-
-/**
- * Score an entry for relevance to a query context.
- *
- * Scoring factors:
- *   - Scope match: exact=5, parent/child=3, global=1
- *   - Tag overlap: 2 per matching tag
- *   - Kind priority: see KIND_PRIORITY
- *   - Importance weight: see IMPORTANCE_WEIGHT
- *   - Frequency bonus: log2(frequency)
- *   - Recency bonus: entries updated in last 7 days get +2
- *
- * @param {object} entry — memory entry
- * @param {{ scope?: string, tags?: string[], preferKinds?: string[] }} context
- * @returns {number} — relevance score
- */
-export function scoreEntry(entry, context = {}) {
-  let score = 0;
-
-  // Scope match
-  if (context.scope && entry.scope) {
-    if (entry.scope === context.scope) {
-      score += 5; // exact match
-    } else {
-      const ancestors = getAncestorScopes(context.scope);
-      if (ancestors.includes(entry.scope)) {
-        score += 3; // ancestor match
-      } else if (entry.scope.startsWith(context.scope + '/')) {
-        score += 3; // descendant match
-      } else if (entry.scope === 'global') {
-        score += 1; // global fallback
-      }
-    }
-  }
-
-  // Tag overlap
-  if (context.tags && context.tags.length > 0 && entry.tags) {
-    const entryTags = new Set(entry.tags.map(t => t.toLowerCase()));
-    for (const tag of context.tags) {
-      if (entryTags.has(tag.toLowerCase())) {
-        score += 2;
-      }
-    }
-  }
-
-  // Kind priority
-  const kindPriority = KIND_PRIORITY[entry.kind] || 0;
-  score += kindPriority * 0.5;
-
-  // Preferred kinds bonus
-  if (context.preferKinds && context.preferKinds.includes(entry.kind)) {
-    score += 2;
-  }
-
-  // Importance weight
-  const impWeight = IMPORTANCE_WEIGHT[entry.importance] || IMPORTANCE_WEIGHT.normal;
-  score += impWeight * 0.5;
-
-  // Frequency bonus (logarithmic)
-  const freq = entry.frequency || 1;
-  score += Math.log2(Math.max(freq, 1));
-
-  // Recency bonus
-  if (entry.updated_at) {
-    const daysSince = (Date.now() - new Date(entry.updated_at).getTime()) / (1000 * 60 * 60 * 24);
-    if (daysSince <= 7) score += 2;
-    else if (daysSince <= 30) score += 1;
-  }
-
-  return score;
-}
-
-// ─── Stale Detection (for Dream) ────────────────────────────
-
-/**
- * Find entries that are potentially stale.
- *
- * Stale criteria:
- * - context entries older than 30 days
- * - entries never recalled (frequency = 1) and older than 60 days
- * - relation entries older than 90 days
- *
- * @param {object[]} entries
- * @returns {object[]} — stale entries
- */
-export function findStaleEntries(entries) {
-  const now = Date.now();
-  const stale = [];
-
-  for (const entry of entries) {
-    const updatedAt = entry.updated_at ? new Date(entry.updated_at).getTime() : 0;
-    const daysSince = (now - updatedAt) / (1000 * 60 * 60 * 24);
-
-    let isStale = false;
-
-    // Context entries become stale fast
-    if (entry.kind === 'context' && daysSince > 30) {
-      isStale = true;
-    }
-
-    // Entries never recalled and old
-    if ((entry.frequency || 1) <= 1 && daysSince > 60) {
-      isStale = true;
-    }
-
-    // Relations are volatile
-    if (entry.kind === 'relation' && daysSince > 90) {
-      isStale = true;
-    }
-
-    if (isStale) {
-      stale.push({ ...entry, _daysSinceUpdate: Math.round(daysSince) });
-    }
-  }
-
-  return stale;
-}
-
-// ─── Duplicate Detection (for Dream Merge) ──────────────────
-
-/**
- * Find groups of entries that are potentially duplicates.
- * Entries are grouped if they share ≥2 tags AND the same kind.
- *
- * @param {object[]} entries
- * @returns {object[][]} — groups of potentially duplicate entries
- */
-export function findDuplicateGroups(entries) {
-  const groups = [];
-  const visited = new Set();
-
-  for (let i = 0; i < entries.length; i++) {
-    if (visited.has(i)) continue;
-
-    const group = [entries[i]];
-    const eTags = new Set((entries[i].tags || []).map(t => t.toLowerCase()));
-
-    for (let j = i + 1; j < entries.length; j++) {
-      if (visited.has(j)) continue;
-      if (entries[i].kind !== entries[j].kind) continue;
-
-      const jTags = new Set((entries[j].tags || []).map(t => t.toLowerCase()));
-      let overlap = 0;
-      for (const tag of eTags) {
-        if (jTags.has(tag)) overlap++;
-      }
-
-      if (overlap >= 2) {
-        group.push(entries[j]);
-        visited.add(j);
-      }
-    }
-
-    if (group.length > 1) {
-      visited.add(i);
-      groups.push(group);
-    }
-  }
-
-  return groups;
-}
-
-// ─── Stats Summary ──────────────────────────────────────────
-
-/**
- * Generate a text summary of memory state (for Dream prompts).
- *
- * @param {ScanResult} scan
- * @returns {string}
- */
-export function summarizeScan(scan) {
-  const lines = [];
-
-  lines.push(`Total entries: ${scan.totalEntries}`);
-
-  // Kind breakdown
-  const kindLines = [];
-  for (const kind of KINDS) {
-    const count = scan.kindCount.get(kind) || 0;
-    if (count > 0) kindLines.push(`${kind}: ${count}`);
-  }
-  if (kindLines.length > 0) {
-    lines.push(`Kinds: ${kindLines.join(', ')}`);
-  }
-
-  // Scope breakdown (top 10)
-  const scopeEntries = [...scan.scopeCount.entries()]
-    .sort((a, b) => b[1] - a[1])
-    .slice(0, 10);
-  if (scopeEntries.length > 0) {
-    lines.push('Top scopes:');
-    for (const [scope, count] of scopeEntries) {
-      lines.push(`  ${scope}: ${count}`);
-    }
-  }
-
-  // Tag cloud (top 20)
-  const tagEntries = [...scan.tagIndex.entries()]
-    .map(([tag, names]) => [tag, names.size])
-    .sort((a, b) => b[1] - a[1])
-    .slice(0, 20);
-  if (tagEntries.length > 0) {
-    lines.push(`Top tags: ${tagEntries.map(([t, c]) => `${t}(${c})`).join(', ')}`);
-  }
-
-  return lines.join('\n');
-}