onbuzz 5.0.2 → 5.3.0

package/src/services/documents/documentStore.js

@@ -0,0 +1,451 @@
+/**
+ * DocumentStore — per-agent persistent store for large externalized content
+ * (web fetches first; any oversized tool output later).
+ *
+ * Why this exists:
+ *   Large tool outputs (a fetched web page) used to be dumped verbatim into
+ *   the conversation — bloating the context window and then getting swept by
+ *   compaction (so the data was both expensive AND lost). Instead we persist
+ *   the full content to disk here, return only a compact handle to the model,
+ *   and index the content for semantic recall (see DocumentIndexer).
+ *
+ * NO-DUPLICATE GUARANTEE:
+ *   `docId` is a DETERMINISTIC hash of the CANONICALIZED source — never a
+ *   timestamp/random id. So re-fetching the same URL maps to the same docId.
+ *   Content `sha256` is the change-detector:
+ *     - unchanged → no-op reuse (just refresh fetchedAt/expiresAt)
+ *     - changed   → rewrite file in place, bump `version`
+ *   The vector layer keys chunks `${docId}#${i}` and upserts by id, so it
+ *   replaces rather than appends. Two fetches → one document.
+ *
+ * Layout (per agent):
+ *   <baseDir>/<agentId>/documents/<docId>.md      — full content
+ *   <baseDir>/<agentId>/documents/manifest.json   — { version, agentId, docs }
+ *
+ * Failure policy: writes are atomic (tmp + rename). Methods validate inputs
+ * and throw TypeError on misuse; they do not swallow fs errors (callers
+ * decide how to degrade).
+ */
+
+import { promises as fsPromises } from 'fs';
+import path from 'path';
+import crypto from 'crypto';
+import { getUserDataPaths } from '../../utilities/userDataDir.js';
+import { retentionScore } from '../embeddings/vectorStore/scoring.js';
+import { touch } from '../embeddings/utilities/decayingCounter.js';
+
+/** Manifest schema version — bump if the on-disk shape changes. */
+export const MANIFEST_VERSION = 1;
+
+/** Default ceiling on stored docs per agent before LRU eviction kicks in. */
+export const DEFAULT_MAX_DOCS_PER_AGENT = 200;
+
+/** Query params stripped during canonicalization so trackers don't fork keys. */
+const TRACKING_PARAMS = new Set([
+  'utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
+  'fbclid', 'gclid', 'gbraid', 'wbraid', 'mc_eid', 'mc_cid', 'igshid',
+  'ref', 'ref_src', 'ref_url', 'spm',
+]);
+
+/**
+ * Canonicalize a source string so trivially-different URLs collapse to one
+ * key. URLs: lowercase scheme+host, drop default port, drop fragment, strip
+ * tracking params, sort remaining params, trim a trailing slash. Non-URL
+ * strings: trimmed + lowercased (deterministic fallback).
+ *
+ * @param {string} source
+ * @returns {string}
+ */
+export function canonicalizeSource(source) {
+  const raw = String(source ?? '').trim();
+  if (!raw) return '';
+
+  let u;
+  try {
+    u = new URL(raw);
+  } catch {
+    return raw.toLowerCase();
+  }
+
+  u.protocol = u.protocol.toLowerCase();
+  u.hostname = u.hostname.toLowerCase();
+  u.hash = '';
+  // Note: the URL constructor already drops default ports (:80/:443), so no
+  // manual normalization is needed here.
+
+  for (const key of [...u.searchParams.keys()]) {
+    if (TRACKING_PARAMS.has(key.toLowerCase())) u.searchParams.delete(key);
+  }
+  u.searchParams.sort();
+
+  if (u.pathname.length > 1 && u.pathname.endsWith('/')) {
+    u.pathname = u.pathname.slice(0, -1);
+  }
+
+  return u.toString();
+}
+
+/**
+ * Deterministic document id from a source. Same source (modulo
+ * canonicalization) always yields the same id.
+ *
+ * @param {string} source
+ * @returns {string} e.g. "doc-1a2b3c4d5e6f7a8b"
+ */
+export function computeDocId(source) {
+  const canonical = canonicalizeSource(source);
+  const hash = crypto.createHash('sha256').update(canonical).digest('hex').slice(0, 16);
+  return `doc-${hash}`;
+}
+
+/** Filesystem-safe agent id (defensive — agent ids are controlled, but still). */
+function sanitizeAgentId(agentId) {
+  return String(agentId).replace(/[^A-Za-z0-9._-]/g, '_');
+}
+
+function sha256Hex(text) {
+  return crypto.createHash('sha256').update(text, 'utf8').digest('hex');
+}
+
+function countLines(text) {
+  if (text.length === 0) return 0;
+  return text.split('\n').length;
+}
+
+export class DocumentStore {
+  /**
+   * @param {object} [opts]
+   * @param {string} [opts.baseDir]  Root for per-agent dirs. Defaults to
+   *   <userDataDir>/state/agents.
+   * @param {object} [opts.fs]       fs.promises-like (readFile/writeFile/
+   *   rename/mkdir/unlink/readdir). Injectable for tests.
+   * @param {() => number} [opts.now] Clock (ms). Injectable for TTL tests.
+   * @param {number|null} [opts.maxDocsPerAgent] Optional hard ceiling on stored
+   *   docs per agent. Default `null` = NO cap (keep everything searchable so the
+   *   far past can auto-resurface via recall — see AR2). When set, it acts as a
+   *   last-resort retention ceiling: the lowest-retention docs are evicted.
+   * @param {number|null} [opts.defaultTtlMs] Default doc lifetime; null = no expiry.
+   * @param {object} [opts.logger]
+   */
+  constructor({
+    baseDir = null,
+    fs = fsPromises,
+    now = () => Date.now(),
+    maxDocsPerAgent = null,
+    defaultTtlMs = null,
+    logger = null,
+  } = {}) {
+    this._baseDir = baseDir || getUserDataPaths().agents;
+    this._fs = fs;
+    this._now = now;
+    // AR2 (auto-resurfacing): default to NO cap — keep every document
+    // searchable so the far past can auto-resurface via recall. A cap is
+    // opt-in; when set it acts as a last-resort retention ceiling.
+    this._maxDocs = maxDocsPerAgent == null ? null : Math.max(1, maxDocsPerAgent);
+    this._defaultTtlMs = defaultTtlMs;
+    this._logger = logger;
+  }
+
+  _agentDir(agentId) {
+    return path.join(this._baseDir, sanitizeAgentId(agentId), 'documents');
+  }
+
+  _manifestPath(agentId) {
+    return path.join(this._agentDir(agentId), 'manifest.json');
+  }
+
+  _docPath(agentId, docId) {
+    return path.join(this._agentDir(agentId), `${docId}.md`);
+  }
+
+  async _ensureDir(agentId) {
+    await this._fs.mkdir(this._agentDir(agentId), { recursive: true });
+  }
+
+  async _loadManifest(agentId) {
+    try {
+      const raw = await this._fs.readFile(this._manifestPath(agentId), 'utf8');
+      const parsed = JSON.parse(raw);
+      if (!parsed || typeof parsed !== 'object' || typeof parsed.docs !== 'object' || parsed.docs === null) {
+        return { version: MANIFEST_VERSION, agentId, docs: {} };
+      }
+      return { version: MANIFEST_VERSION, agentId, docs: parsed.docs };
+    } catch {
+      // Missing or corrupt → fresh manifest. The doc files are the source of
+      // truth for content; a lost manifest just means we re-index on next put.
+      return { version: MANIFEST_VERSION, agentId, docs: {} };
+    }
+  }
+
+  async _saveManifest(agentId, manifest) {
+    await this._ensureDir(agentId);
+    const target = this._manifestPath(agentId);
+    const tmp = `${target}.tmp`;
+    await this._fs.writeFile(tmp, JSON.stringify(manifest, null, 2), 'utf8');
+    await this._fs.rename(tmp, target);
+  }
+
+  async _writeDoc(agentId, docId, content) {
+    await this._ensureDir(agentId);
+    const target = this._docPath(agentId, docId);
+    const tmp = `${target}.tmp`;
+    await this._fs.writeFile(tmp, content, 'utf8');
+    await this._fs.rename(tmp, target);
+  }
+
+  async _deleteDocFile(agentId, docId) {
+    try {
+      await this._fs.unlink(this._docPath(agentId, docId));
+    } catch (err) {
+      if (err && err.code !== 'ENOENT') {
+        this._logger?.warn?.(`[documentStore] failed to delete ${docId}: ${err.message}`);
+      }
+    }
+  }
+
+  /**
+   * Store (or refresh) a document. Returns the manifest entry augmented with
+   * `{ path, deduped, changed, evicted }`.
+   *
+   * @param {string} agentId
+   * @param {object} doc
+   * @param {string} doc.source         Origin (URL). Drives the deterministic id.
+   * @param {string} doc.content        Full text to persist.
+   * @param {string} [doc.title]
+   * @param {string} [doc.contentType='text/markdown']
+   * @param {number|null} [doc.ttlMs]   Overrides defaultTtlMs for this doc.
+   */
+  async put(agentId, { source, content, title = '', contentType = 'text/markdown', ttlMs, pinned } = {}) {
+    if (!agentId) throw new TypeError('DocumentStore.put: agentId required');
+    if (typeof source !== 'string' || source.trim() === '') {
+      throw new TypeError('DocumentStore.put: source must be a non-empty string');
+    }
+    if (typeof content !== 'string') {
+      throw new TypeError('DocumentStore.put: content must be a string');
+    }
+
+    const canonicalSource = canonicalizeSource(source);
+    const docId = computeDocId(source);
+    const sha256 = sha256Hex(content);
+    const bytes = Buffer.byteLength(content, 'utf8');
+    const lines = countLines(content);
+
+    const manifest = await this._loadManifest(agentId);
+    const existing = manifest.docs[docId] || null;
+
+    const nowMs = this._now();
+    const nowIso = new Date(nowMs).toISOString();
+    const effectiveTtl = ttlMs === undefined ? this._defaultTtlMs : ttlMs;
+    const expiresAt = (effectiveTtl && effectiveTtl > 0)
+      ? new Date(nowMs + effectiveTtl).toISOString()
+      : null;
+
+    // Unchanged content → no-op reuse: just refresh access/expiry.
+    if (existing && existing.sha256 === sha256) {
+      existing.fetchedAt = nowIso;
+      existing.expiresAt = expiresAt;
+      if (title) existing.title = title;
+      if (pinned !== undefined) existing.pinned = !!pinned;
+      manifest.docs[docId] = existing;
+      await this._saveManifest(agentId, manifest);
+      return { ...existing, path: this._docPath(agentId, docId), deduped: true, changed: false, evicted: [] };
+    }
+
+    // New or changed → (re)write the file in place.
+    await this._writeDoc(agentId, docId, content);
+
+    const entry = {
+      docId,
+      source,
+      canonicalSource,
+      title,
+      contentType,
+      sha256,
+      bytes,
+      lines,
+      version: existing ? (existing.version || 1) + 1 : 1,
+      fetchedAt: nowIso,
+      createdAt: existing?.createdAt || nowIso,
+      expiresAt,
+      // Retention: pinned never evicts; accessStat is the decayed access
+      // counter (bumped via recordAccess on recall). Preserved across re-puts.
+      pinned: pinned !== undefined ? !!pinned : (existing?.pinned ?? false),
+      accessStat: existing?.accessStat || null,
+    };
+    manifest.docs[docId] = entry;
+
+    const evicted = await this._evictIfNeeded(agentId, manifest, docId);
+    await this._saveManifest(agentId, manifest);
+
+    return { ...entry, path: this._docPath(agentId, docId), deduped: false, changed: true, evicted };
+  }
+
+  /**
+   * Retention-based eviction (aging-LFU): when over the cap, evict the
+   * LOWEST-retention docs — usage-with-decay × recency — so an old-but-still-
+   * used doc outlives a newer-but-untouched one. Ties break by `fetchedAt`
+   * (older first), so with no access signal it degrades gracefully to LRU.
+   *
+   * Never evicts: the just-written doc (`keepDocId`) or any `pinned` doc. If
+   * everything over the cap is pinned, we keep more than the cap (pins win).
+   *
+   * Mutates `manifest.docs`; returns evicted entries so the caller can purge
+   * their vectors.
+   * @private
+   */
+  async _evictIfNeeded(agentId, manifest, keepDocId) {
+    // No cap (AR2 default) → never evict; everything stays searchable.
+    if (this._maxDocs == null) return [];
+    const ids = Object.keys(manifest.docs);
+    if (ids.length <= this._maxDocs) return [];
+
+    const now = this._now();
+    const ranked = ids
+      .filter((id) => id !== keepDocId && !manifest.docs[id].pinned)
+      .map((id) => manifest.docs[id])
+      .map((e) => ({
+        e,
+        // Map our entry fields onto the scorer: fetchedAt is the freshness
+        // signal (last fetch), accessStat carries last-access + decayed count.
+        score: retentionScore(
+          { accessStat: e.accessStat, updatedAt: e.fetchedAt, createdAt: e.createdAt },
+          { now },
+        ),
+      }))
+      .sort((a, b) => (a.score - b.score) || String(a.e.fetchedAt).localeCompare(String(b.e.fetchedAt)));
+
+    const overBy = ids.length - this._maxDocs;
+    const toEvict = ranked.slice(0, overBy).map((r) => r.e);
+
+    const evicted = [];
+    for (const entry of toEvict) {
+      await this._deleteDocFile(agentId, entry.docId);
+      delete manifest.docs[entry.docId];
+      evicted.push(entry);
+    }
+    return evicted;
+  }
+
+  /**
+   * Register an access to a doc — bumps its decayed access counter (the
+   * aging-LFU "reinforcement" signal) and resets its last-access time. Called
+   * when a doc is recalled. No-op if the doc is unknown.
+   * @returns {Promise<object|null>} the updated entry, or null.
+   */
+  async recordAccess(agentId, docId) {
+    const manifest = await this._loadManifest(agentId);
+    const entry = manifest.docs[docId];
+    if (!entry) return null;
+    entry.accessStat = touch(entry.accessStat, this._now());
+    manifest.docs[docId] = entry;
+    await this._saveManifest(agentId, manifest);
+    return entry;
+  }
+
+  /**
+   * Batch variant of recordAccess — bump several docs in ONE manifest
+   * load+save (used by recall, which touches every hit). Unknown ids are
+   * skipped. @returns {Promise<object[]>} the updated entries.
+   */
+  async recordAccessMany(agentId, docIds) {
+    if (!Array.isArray(docIds) || docIds.length === 0) return [];
+    const manifest = await this._loadManifest(agentId);
+    const now = this._now();
+    const updated = [];
+    for (const docId of new Set(docIds)) {
+      const entry = manifest.docs[docId];
+      if (!entry) continue;
+      entry.accessStat = touch(entry.accessStat, now);
+      updated.push(entry);
+    }
+    if (updated.length > 0) await this._saveManifest(agentId, manifest);
+    return updated;
+  }
+
+  /**
+   * Pin (or unpin) a doc so it is never evicted regardless of retention.
+   * @returns {Promise<object|null>} the updated entry, or null.
+   */
+  async pin(agentId, docId, pinned = true) {
+    const manifest = await this._loadManifest(agentId);
+    const entry = manifest.docs[docId];
+    if (!entry) return null;
+    entry.pinned = !!pinned;
+    manifest.docs[docId] = entry;
+    await this._saveManifest(agentId, manifest);
+    return entry;
+  }
+
+  /** @returns {Promise<object|null>} manifest entry or null. */
+  async get(agentId, docId) {
+    const manifest = await this._loadManifest(agentId);
+    return manifest.docs[docId] || null;
+  }
+
+  /** @returns {Promise<object[]>} all manifest entries, newest first. */
+  async list(agentId) {
+    const manifest = await this._loadManifest(agentId);
+    return Object.values(manifest.docs)
+      .sort((a, b) => String(b.fetchedAt).localeCompare(String(a.fetchedAt)));
+  }
+
+  /**
+   * Read full content, or a 1-based inclusive line range (mirrors code-map
+   * read-range semantics so handles can point the agent at exact lines).
+   *
+   * @returns {Promise<{docId, content, startLine, endLine, totalLines}|null>}
+   *   null when the doc is unknown.
+   */
+  async read(agentId, docId, { startLine, endLine } = {}) {
+    const entry = await this.get(agentId, docId);
+    if (!entry) return null;
+
+    let content;
+    try {
+      content = await this._fs.readFile(this._docPath(agentId, docId), 'utf8');
+    } catch (err) {
+      if (err && err.code === 'ENOENT') return null;
+      throw err;
+    }
+
+    if (startLine == null && endLine == null) {
+      return { docId, content, startLine: 1, endLine: countLines(content), totalLines: countLines(content) };
+    }
+
+    const allLines = content.split('\n');
+    const total = allLines.length;
+    const from = Math.max(1, startLine ?? 1);
+    const to = Math.min(total, endLine ?? total);
+    const slice = from > to ? '' : allLines.slice(from - 1, to).join('\n');
+    return { docId, content: slice, startLine: from, endLine: to, totalLines: total };
+  }
+
+  /** Remove one document (file + manifest entry). @returns the removed entry or null. */
+  async remove(agentId, docId) {
+    const manifest = await this._loadManifest(agentId);
+    const entry = manifest.docs[docId];
+    if (!entry) return null;
+    await this._deleteDocFile(agentId, docId);
+    delete manifest.docs[docId];
+    await this._saveManifest(agentId, manifest);
+    return entry;
+  }
+
+  /** Remove all expired documents. @returns {Promise<object[]>} removed entries. */
+  async sweepExpired(agentId) {
+    const manifest = await this._loadManifest(agentId);
+    const nowIso = new Date(this._now()).toISOString();
+    const removed = [];
+    for (const entry of Object.values(manifest.docs)) {
+      if (entry.expiresAt && entry.expiresAt <= nowIso) {
+        await this._deleteDocFile(agentId, entry.docId);
+        delete manifest.docs[entry.docId];
+        removed.push(entry);
+      }
+    }
+    if (removed.length > 0) await this._saveManifest(agentId, manifest);
+    return removed;
+  }
+}
+
+export default DocumentStore;

package/src/services/embeddings/__tests__/autoRecall.test.js

@@ -152,6 +152,90 @@ describe('recallForPrompt — threshold + topK + budget', () => {
   });
 });
 
+describe('recallForPrompt — relevance gate (AR3 auto-resurfacing)', () => {
+  // Hit whose RANKING score has been damped (old item: recency factor, or a
+  // hybrid RRF score on a different scale) but whose raw cosine `similarity`
+  // is high. The far past must still resurface — gate on relevance, not score.
+  function hitWithSim(id, score, similarity, text) {
+    return { id, score, similarity, metadata: { text, title: `t-${id}`, createdAt: '2026-01-01T00:00:00Z' } };
+  }
+
+  test('high-similarity but low (recency/RRF-damped) score still surfaces', async () => {
+    const deps = makeDeps({
+      memoryHits: [hitWithSim('old', 0.03, 0.92, 'far past but on-topic')],
+      extraConfig: { threshold: 0.78 },
+    });
+    const r = await recallForPrompt(deps, 'a', 'query');
+    expect(r.items.map(i => i.id)).toContain('old');
+  });
+
+  test('low similarity is dropped even when its score is high', async () => {
+    const deps = makeDeps({
+      memoryHits: [hitWithSim('offtopic', 0.99, 0.40, 'unrelated')],
+      extraConfig: { threshold: 0.78 },
+    });
+    const r = await recallForPrompt(deps, 'a', 'query');
+    expect(r.items).toEqual([]);
+  });
+
+  test('survivors are still ranked by score, not by similarity', async () => {
+    const deps = makeDeps({
+      memoryHits: [
+        hitWithSim('a', 0.05, 0.95, 'old high-relevance'),  // top relevance, low score
+        hitWithSim('b', 0.90, 0.80, 'fresh'),               // lower relevance, high score
+      ],
+      extraConfig: { threshold: 0.78, topK: 5 },
+    });
+    const r = await recallForPrompt(deps, 'a', 'query');
+    expect(r.items.map(i => i.id)).toEqual(['b', 'a']); // both pass gate; b leads by score
+  });
+
+  test('back-compat: hit with only score (no similarity) is gated on score', async () => {
+    const deps = makeDeps({
+      memoryHits: [memHit('m1', 0.9, 'cosine-as-score'), memHit('m2', 0.4, 'below')],
+      extraConfig: { threshold: 0.78 },
+    });
+    const r = await recallForPrompt(deps, 'a', 'query');
+    expect(r.items.map(i => i.id)).toEqual(['m1']);
+  });
+});
+
+describe('recallForPrompt — telemetry stats (memory-path view)', () => {
+  test('stats funnel: considered → passed → injected → tokens', async () => {
+    const text = 'word '.repeat(20); // ~25 tokens
+    const deps = makeDeps({
+      memoryHits: [
+        memHit('m1', 0.95, text),  // passes
+        memHit('m2', 0.90, text),  // passes
+        memHit('m3', 0.20, 'lo'),  // below gate
+      ],
+      extraConfig: { threshold: 0.5, topK: 5, maxTokens: 400 },
+    });
+    const r = await recallForPrompt(deps, 'a', 'query');
+    expect(r.stats.considered).toBe(3);
+    expect(r.stats.passed).toBe(2);
+    expect(r.stats.injected).toBe(2);
+    expect(r.stats.tokens).toBeGreaterThan(0);
+  });
+
+  test('stats present (all zero) when no candidates surface', async () => {
+    const deps = makeDeps({ memoryHits: [], reminisceHits: [] });
+    const r = await recallForPrompt(deps, 'a', 'query');
+    expect(r.stats).toEqual({ considered: 0, passed: 0, injected: 0, tokens: 0 });
+  });
+
+  test('injected < passed when token budget caps additions', async () => {
+    const text = 'a'.repeat(100); // ~25 tokens each
+    const deps = makeDeps({
+      memoryHits: [memHit('m1', 0.95, text), memHit('m2', 0.94, text), memHit('m3', 0.93, text)],
+      extraConfig: { threshold: 0.5, topK: 10, maxTokens: 50 },
+    });
+    const r = await recallForPrompt(deps, 'a', 'query');
+    expect(r.stats.passed).toBe(3);
+    expect(r.stats.injected).toBeLessThan(r.stats.passed);
+  });
+});
+
 describe('recallForPrompt — multi-surface', () => {
   test('merges results from memory + reminisce, sorted by score', async () => {
     const deps = makeDeps({
@@ -235,3 +319,64 @@ describe('formatRecallSection', () => {
     expect(out).toContain('line one line two line three');
   });
 });
+
+describe('recallForPrompt — documents surface (offloaded web/file content)', () => {
+  // DocumentIndexer hits are flat: { docId, source(url), title, chunkIndex,
+  // lineStart, lineEnd, score, text }.
+  function docHit({ docId, chunkIndex = 0, score, text, title, url }) {
+    return { docId, chunkIndex, score, text, title, source: url, lineStart: 1, lineEnd: 3 };
+  }
+  function makeDocDeps(hits) {
+    const documents = { isEnabled: true, search: jest.fn(async () => ({ enabled: true, hits })) };
+    return {
+      deps: {
+        getIndexers: () => ({ documents }),
+        getConfig: () => ({ autoRecall: { enabled: true, threshold: 0.5, topK: 3, maxTokens: 400, surfaces: ['documents'] } }),
+      },
+      documents,
+    };
+  }
+
+  test('document hits normalise to id docId#chunkIndex with text + title', async () => {
+    const { deps } = makeDocDeps([
+      docHit({ docId: 'doc-abc', chunkIndex: 2, score: 0.9, text: 'the rate limit is 100/min', title: 'API Docs', url: 'https://x/api' }),
+    ]);
+    const r = await recallForPrompt(deps, 'a', 'what are the rate limits?');
+    expect(r.enabled).toBe(true);
+    expect(r.items).toHaveLength(1);
+    expect(r.items[0]).toMatchObject({ id: 'doc-abc#2', source: 'documents', text: 'the rate limit is 100/min', title: 'API Docs' });
+  });
+
+  test('title falls back to the URL when the page had no title', async () => {
+    const { deps } = makeDocDeps([
+      docHit({ docId: 'doc-xyz', score: 0.9, text: 'body', title: undefined, url: 'https://x/page' }),
+    ]);
+    const r = await recallForPrompt(deps, 'a', 'query');
+    expect(r.items[0].title).toBe('https://x/page');
+  });
+
+  test('missing chunkIndex/text/title/url all use safe fallbacks', async () => {
+    // Bare hit: only docId + score → exercises every fallback in the block.
+    const documents = { isEnabled: true, search: jest.fn(async () => ({ enabled: true, hits: [{ docId: 'doc-n', score: 0.9 }] })) };
+    const deps = {
+      getIndexers: () => ({ documents }),
+      getConfig: () => ({ autoRecall: { enabled: true, threshold: 0.5, topK: 3, maxTokens: 400, surfaces: ['documents'] } }),
+    };
+    const r = await recallForPrompt(deps, 'a', 'q');
+    expect(r.items[0].id).toBe('doc-n#0'); // chunkIndex ?? 0
+    expect(r.items[0].text).toBe('');      // text || ''
+    expect(r.items[0].title).toBe('');     // title || source || ''
+  });
+
+  test('default surfaces include documents (config omits surfaces)', async () => {
+    const documents = { isEnabled: true, search: jest.fn(async () => ({ enabled: true, hits: [docHit({ docId: 'd', score: 0.9, text: 'hit', title: 'T', url: 'u' })] })) };
+    const deps = {
+      getIndexers: () => ({ documents }),
+      // No `surfaces` key → recallForPrompt must default to include 'documents'.
+      getConfig: () => ({ autoRecall: { enabled: true, threshold: 0.5, topK: 3, maxTokens: 400 } }),
+    };
+    const r = await recallForPrompt(deps, 'a', 'query');
+    expect(documents.search).toHaveBeenCalled();
+    expect(r.items[0].id).toBe('d#0');
+  });
+});