onbuzz 5.0.2 → 5.3.0

package/src/services/documents/__tests__/documentStore.test.js

@@ -0,0 +1,507 @@
+/**
+ * Tests for DocumentStore — the per-agent persistent store for externalized
+ * large content. Covers the no-duplicate guarantee (deterministic id +
+ * sha256 change-detection), TTL/LRU eviction, ranged read, and the fs/clock
+ * error branches. Uses a real temp dir; injects clock + faulty-fs where a
+ * branch can't be reached with a healthy filesystem.
+ */
+import { describe, test, expect, beforeEach, afterEach, jest } from '@jest/globals';
+import { promises as fs } from 'fs';
+import os from 'os';
+import path from 'path';
+import {
+  DocumentStore,
+  canonicalizeSource,
+  computeDocId,
+  DEFAULT_MAX_DOCS_PER_AGENT,
+  MANIFEST_VERSION,
+} from '../documentStore.js';
+
+let tmp;
+beforeEach(async () => {
+  tmp = await fs.mkdtemp(path.join(os.tmpdir(), 'docstore-'));
+});
+afterEach(async () => {
+  await fs.rm(tmp, { recursive: true, force: true });
+});
+
+const AGENT = 'agent-test-1';
+
+function makeStore(opts = {}) {
+  return new DocumentStore({ baseDir: tmp, ...opts });
+}
+
+// ── canonicalizeSource ──────────────────────────────────────────────────
+describe('canonicalizeSource', () => {
+  test('collapses fragment, trailing slash, default port, tracking params', () => {
+    const a = canonicalizeSource('HTTPS://Example.com:443/Path/?utm_source=x&b=2&a=1#frag');
+    const b = canonicalizeSource('https://example.com/Path?a=1&b=2');
+    expect(a).toBe(b);
+  });
+
+  test('keeps non-default port and meaningful query', () => {
+    expect(canonicalizeSource('http://h.com:8080/p?q=1')).toContain(':8080');
+    expect(canonicalizeSource('http://h.com/p?q=1')).toContain('q=1');
+  });
+
+  test('non-URL source falls back to trimmed-lowercased', () => {
+    expect(canonicalizeSource('  Not A URL  ')).toBe('not a url');
+  });
+
+  test('empty / nullish → empty string', () => {
+    expect(canonicalizeSource('')).toBe('');
+    expect(canonicalizeSource(null)).toBe('');
+    expect(canonicalizeSource(undefined)).toBe('');
+  });
+
+  test('root path keeps its slash', () => {
+    expect(canonicalizeSource('https://example.com/')).toBe('https://example.com/');
+  });
+});
+
+// ── computeDocId ────────────────────────────────────────────────────────
+describe('computeDocId', () => {
+  test('same source (modulo canonicalization) → same id', () => {
+    expect(computeDocId('https://x.com/a#top')).toBe(computeDocId('https://x.com/a'));
+  });
+  test('different sources → different ids', () => {
+    expect(computeDocId('https://x.com/a')).not.toBe(computeDocId('https://x.com/b'));
+  });
+  test('id has the doc- prefix', () => {
+    expect(computeDocId('https://x.com')).toMatch(/^doc-[0-9a-f]{16}$/);
+  });
+});
+
+// ── put / dedup ─────────────────────────────────────────────────────────
+describe('put', () => {
+  test('stores a new document (changed, version 1, file on disk)', async () => {
+    const store = makeStore();
+    const r = await store.put(AGENT, { source: 'https://x.com/a', content: 'hello\nworld', title: 'A' });
+    expect(r.changed).toBe(true);
+    expect(r.deduped).toBe(false);
+    expect(r.version).toBe(1);
+    expect(r.bytes).toBe(11);
+    expect(r.lines).toBe(2);
+    expect(r.docId).toBe(computeDocId('https://x.com/a'));
+    const onDisk = await fs.readFile(r.path, 'utf8');
+    expect(onDisk).toBe('hello\nworld');
+  });
+
+  test('re-put with identical content is a no-op reuse (deduped, version stays)', async () => {
+    const store = makeStore({ now: () => 1000 });
+    const first = await store.put(AGENT, { source: 'https://x.com/a', content: 'same' });
+    const store2 = new DocumentStore({ baseDir: tmp, now: () => 5000 });
+    const second = await store2.put(AGENT, { source: 'https://x.com/a#frag', content: 'same', title: 'T' });
+    expect(second.deduped).toBe(true);
+    expect(second.changed).toBe(false);
+    expect(second.version).toBe(first.version); // unchanged
+    expect(second.docId).toBe(first.docId);     // canonicalization collapses #frag
+    expect(second.title).toBe('T');             // title refreshed on dedup
+    expect(second.fetchedAt).toBe(new Date(5000).toISOString()); // access refreshed
+  });
+
+  test('re-put with changed content bumps version and rewrites file', async () => {
+    const store = makeStore();
+    await store.put(AGENT, { source: 'https://x.com/a', content: 'v1' });
+    const r = await store.put(AGENT, { source: 'https://x.com/a', content: 'v2-different' });
+    expect(r.changed).toBe(true);
+    expect(r.version).toBe(2);
+    expect(await fs.readFile(r.path, 'utf8')).toBe('v2-different');
+  });
+
+  test('ttlMs sets expiresAt; absence leaves it null', async () => {
+    const store = makeStore({ now: () => 1000 });
+    const withTtl = await store.put(AGENT, { source: 'https://x.com/a', content: 'a', ttlMs: 500 });
+    expect(withTtl.expiresAt).toBe(new Date(1500).toISOString());
+    const noTtl = await store.put(AGENT, { source: 'https://x.com/b', content: 'b' });
+    expect(noTtl.expiresAt).toBeNull();
+  });
+
+  test('defaultTtlMs applies when ttlMs omitted', async () => {
+    const store = makeStore({ now: () => 0, defaultTtlMs: 100 });
+    const r = await store.put(AGENT, { source: 'https://x.com/a', content: 'a' });
+    expect(r.expiresAt).toBe(new Date(100).toISOString());
+  });
+
+  test('ttlMs:0 means no expiry (falsy guard)', async () => {
+    const store = makeStore({ now: () => 1000, defaultTtlMs: 5000 });
+    const r = await store.put(AGENT, { source: 'https://x.com/a', content: 'a', ttlMs: 0 });
+    expect(r.expiresAt).toBeNull();
+  });
+
+  test('dedup without a new title keeps the original title', async () => {
+    const store = makeStore();
+    await store.put(AGENT, { source: 'https://x.com/a', content: 'same', title: 'Original' });
+    const again = await store.put(AGENT, { source: 'https://x.com/a', content: 'same' }); // no title
+    expect(again.deduped).toBe(true);
+    expect(again.title).toBe('Original');
+  });
+
+  test('empty content is allowed (0 bytes, 0 lines)', async () => {
+    const store = makeStore();
+    const r = await store.put(AGENT, { source: 'https://x.com/a', content: '' });
+    expect(r.bytes).toBe(0);
+    expect(r.lines).toBe(0);
+  });
+
+  test('invalid inputs throw TypeError', async () => {
+    const store = makeStore();
+    await expect(store.put('', { source: 'https://x', content: 'a' })).rejects.toThrow(TypeError);
+    await expect(store.put(AGENT, { source: '', content: 'a' })).rejects.toThrow(TypeError);
+    await expect(store.put(AGENT, { source: '   ', content: 'a' })).rejects.toThrow(TypeError);
+    await expect(store.put(AGENT, { source: 'https://x', content: 123 })).rejects.toThrow(TypeError);
+  });
+
+  test('put with no doc object throws (default param branch)', async () => {
+    await expect(makeStore().put(AGENT)).rejects.toThrow(TypeError);
+  });
+
+  test('changed re-put over an entry missing version/createdAt uses fallbacks', async () => {
+    const store = makeStore({ now: () => 7000 });
+    const first = await store.put(AGENT, { source: 'https://x.com/a', content: 'v1' });
+    const mpath = store._manifestPath(AGENT);
+    const m = JSON.parse(await fs.readFile(mpath, 'utf8'));
+    delete m.docs[first.docId].version;
+    delete m.docs[first.docId].createdAt;
+    await fs.writeFile(mpath, JSON.stringify(m), 'utf8');
+    const r = await store.put(AGENT, { source: 'https://x.com/a', content: 'v2' });
+    expect(r.version).toBe(2); // (undefined || 1) + 1
+    expect(r.createdAt).toBe(new Date(7000).toISOString()); // || nowIso
+  });
+});
+
+// ── eviction ────────────────────────────────────────────────────────────
+describe('LRU eviction', () => {
+  test('evicts oldest when over cap; never the just-written doc', async () => {
+    let t = 1000;
+    const store = makeStore({ maxDocsPerAgent: 2, now: () => (t += 1000) });
+    const a = await store.put(AGENT, { source: 'https://x.com/a', content: 'a' }); // oldest
+    await store.put(AGENT, { source: 'https://x.com/b', content: 'b' });
+    const c = await store.put(AGENT, { source: 'https://x.com/c', content: 'c' }); // triggers evict
+
+    expect(c.evicted.map((e) => e.docId)).toEqual([a.docId]);
+    expect(await store.get(AGENT, a.docId)).toBeNull();
+    await expect(fs.readFile(a.path, 'utf8')).rejects.toThrow(); // file gone
+    expect(await store.get(AGENT, c.docId)).not.toBeNull();      // kept
+  });
+
+  test('no eviction when at/under cap', async () => {
+    const store = makeStore({ maxDocsPerAgent: 5 });
+    const r = await store.put(AGENT, { source: 'https://x.com/a', content: 'a' });
+    expect(r.evicted).toEqual([]);
+  });
+});
+
+// ── AR2: default = no cap (keep everything searchable / auto-resurfacing) ──
+describe('no-cap default (AR2)', () => {
+  test('default store keeps every doc — never evicts past any count', async () => {
+    const store = makeStore(); // no maxDocsPerAgent → unlimited
+    const ids = [];
+    for (let i = 0; i < 25; i++) {
+      const r = await store.put(AGENT, { source: `https://x.com/p${i}`, content: `c${i}` });
+      expect(r.evicted).toEqual([]); // nothing ever dropped
+      ids.push(r.docId);
+    }
+    expect(await store.list(AGENT)).toHaveLength(25);
+    // the very first doc is still present (the "far past" stays searchable)
+    expect(await store.get(AGENT, ids[0])).not.toBeNull();
+  });
+
+  test('explicit null cap behaves the same as the default', async () => {
+    const store = makeStore({ maxDocsPerAgent: null });
+    const a = await store.put(AGENT, { source: 'https://x.com/a', content: 'a' });
+    const b = await store.put(AGENT, { source: 'https://x.com/b', content: 'b' });
+    expect(b.evicted).toEqual([]);
+    expect(await store.get(AGENT, a.docId)).not.toBeNull();
+  });
+});
+
+// ── get / list ──────────────────────────────────────────────────────────
+describe('get / list', () => {
+  test('get returns entry or null', async () => {
+    const store = makeStore();
+    const r = await store.put(AGENT, { source: 'https://x.com/a', content: 'a' });
+    expect((await store.get(AGENT, r.docId)).docId).toBe(r.docId);
+    expect(await store.get(AGENT, 'doc-missing')).toBeNull();
+    expect(await store.get('agent-none', 'doc-x')).toBeNull();
+  });
+
+  test('list is newest-first', async () => {
+    let t = 0;
+    const store = makeStore({ now: () => (t += 1000) });
+    const a = await store.put(AGENT, { source: 'https://x.com/a', content: 'a' });
+    const b = await store.put(AGENT, { source: 'https://x.com/b', content: 'b' });
+    const ids = (await store.list(AGENT)).map((e) => e.docId);
+    expect(ids).toEqual([b.docId, a.docId]);
+  });
+});
+
+// ── read (full + ranged) ────────────────────────────────────────────────
+describe('read', () => {
+  const CONTENT = 'line1\nline2\nline3\nline4\nline5';
+
+  test('full read returns whole content + total lines', async () => {
+    const store = makeStore();
+    const r = await store.put(AGENT, { source: 'https://x.com/a', content: CONTENT });
+    const got = await store.read(AGENT, r.docId);
+    expect(got.content).toBe(CONTENT);
+    expect(got.totalLines).toBe(5);
+    expect(got.startLine).toBe(1);
+    expect(got.endLine).toBe(5);
+  });
+
+  test('ranged read is 1-based inclusive', async () => {
+    const store = makeStore();
+    const r = await store.put(AGENT, { source: 'https://x.com/a', content: CONTENT });
+    const got = await store.read(AGENT, r.docId, { startLine: 2, endLine: 3 });
+    expect(got.content).toBe('line2\nline3');
+    expect(got.startLine).toBe(2);
+    expect(got.endLine).toBe(3);
+  });
+
+  test('range clamps to bounds; from>to yields empty', async () => {
+    const store = makeStore();
+    const r = await store.put(AGENT, { source: 'https://x.com/a', content: CONTENT });
+    const clamped = await store.read(AGENT, r.docId, { startLine: 0, endLine: 999 });
+    expect(clamped.startLine).toBe(1);
+    expect(clamped.endLine).toBe(5);
+    const empty = await store.read(AGENT, r.docId, { startLine: 4, endLine: 2 });
+    expect(empty.content).toBe('');
+  });
+
+  test('only startLine given → reads to end; only endLine given → reads from 1', async () => {
+    const store = makeStore();
+    const r = await store.put(AGENT, { source: 'https://x.com/a', content: CONTENT });
+    const fromOnly = await store.read(AGENT, r.docId, { startLine: 4 });
+    expect(fromOnly.content).toBe('line4\nline5');
+    expect(fromOnly.endLine).toBe(5);
+    const toOnly = await store.read(AGENT, r.docId, { endLine: 2 });
+    expect(toOnly.content).toBe('line1\nline2');
+    expect(toOnly.startLine).toBe(1);
+  });
+
+  test('unknown doc → null', async () => {
+    const store = makeStore();
+    expect(await store.read(AGENT, 'doc-nope')).toBeNull();
+  });
+
+  test('entry present but file missing (ENOENT) → null', async () => {
+    const store = makeStore();
+    const r = await store.put(AGENT, { source: 'https://x.com/a', content: 'a' });
+    await fs.unlink(r.path); // delete file but keep manifest entry
+    expect(await store.read(AGENT, r.docId)).toBeNull();
+  });
+
+  test('non-ENOENT read error rethrows', async () => {
+    const store = makeStore();
+    const r = await store.put(AGENT, { source: 'https://x.com/a', content: 'a' });
+    // Faulty fs: throw EACCES only for the .md file, delegate manifest reads.
+    const faulty = {
+      ...fs,
+      readFile: (p, enc) => p.endsWith(`${r.docId}.md`)
+        ? Promise.reject(Object.assign(new Error('denied'), { code: 'EACCES' }))
+        : fs.readFile(p, enc),
+    };
+    const store2 = new DocumentStore({ baseDir: tmp, fs: faulty });
+    await expect(store2.read(AGENT, r.docId)).rejects.toThrow('denied');
+  });
+});
+
+// ── remove ──────────────────────────────────────────────────────────────
+describe('remove', () => {
+  test('removes file + manifest entry, returns entry', async () => {
+    const store = makeStore();
+    const r = await store.put(AGENT, { source: 'https://x.com/a', content: 'a' });
+    const removed = await store.remove(AGENT, r.docId);
+    expect(removed.docId).toBe(r.docId);
+    expect(await store.get(AGENT, r.docId)).toBeNull();
+    await expect(fs.readFile(r.path, 'utf8')).rejects.toThrow();
+  });
+
+  test('removing unknown doc → null', async () => {
+    const store = makeStore();
+    expect(await store.remove(AGENT, 'doc-nope')).toBeNull();
+  });
+
+  test('unlink ENOENT is silent (no warn), removal still completes', async () => {
+    const warn = jest.fn();
+    const r = await makeStore().put(AGENT, { source: 'https://x.com/a', content: 'a' });
+    const faulty = {
+      ...fs,
+      unlink: () => Promise.reject(Object.assign(new Error('gone'), { code: 'ENOENT' })),
+    };
+    const store = new DocumentStore({ baseDir: tmp, fs: faulty, logger: { warn } });
+    expect((await store.remove(AGENT, r.docId)).docId).toBe(r.docId);
+    expect(warn).not.toHaveBeenCalled();
+  });
+
+  test('unlink rejecting with a falsy error is tolerated (no warn)', async () => {
+    const warn = jest.fn();
+    const r = await makeStore().put(AGENT, { source: 'https://x.com/a', content: 'a' });
+    const faulty = { ...fs, unlink: () => Promise.reject(undefined) };
+    const store = new DocumentStore({ baseDir: tmp, fs: faulty, logger: { warn } });
+    expect((await store.remove(AGENT, r.docId)).docId).toBe(r.docId);
+    expect(warn).not.toHaveBeenCalled();
+  });
+
+  test('unlink non-ENOENT error is logged, removal still completes', async () => {
+    const warn = jest.fn();
+    const r = await makeStore().put(AGENT, { source: 'https://x.com/a', content: 'a' });
+    const faulty = {
+      ...fs,
+      unlink: () => Promise.reject(Object.assign(new Error('busy'), { code: 'EPERM' })),
+    };
+    const store = new DocumentStore({ baseDir: tmp, fs: faulty, logger: { warn } });
+    const removed = await store.remove(AGENT, r.docId);
+    expect(removed.docId).toBe(r.docId);
+    expect(warn).toHaveBeenCalled();
+    expect(await store.get(AGENT, r.docId)).toBeNull(); // manifest entry gone
+  });
+});
+
+// ── sweepExpired ────────────────────────────────────────────────────────
+describe('sweepExpired', () => {
+  test('removes expired, keeps live, returns removed', async () => {
+    let t = 1000;
+    const store = makeStore({ now: () => t });
+    const exp = await store.put(AGENT, { source: 'https://x.com/exp', content: 'e', ttlMs: 10 });
+    const live = await store.put(AGENT, { source: 'https://x.com/live', content: 'l', ttlMs: 100_000_000 });
+    t = 1_000_000; // far future: past exp (1010) but before live (100,001,000)
+    const removed = await store.sweepExpired(AGENT);
+    expect(removed.map((e) => e.docId)).toEqual([exp.docId]);
+    expect(await store.get(AGENT, exp.docId)).toBeNull();
+    expect(await store.get(AGENT, live.docId)).not.toBeNull();
+  });
+
+  test('nothing expired → empty array, no throw', async () => {
+    const store = makeStore({ now: () => 1000 });
+    await store.put(AGENT, { source: 'https://x.com/a', content: 'a' }); // no ttl
+    expect(await store.sweepExpired(AGENT)).toEqual([]);
+  });
+});
+
+// ── manifest resilience + defaults ──────────────────────────────────────
+describe('manifest resilience & defaults', () => {
+  test('corrupt manifest JSON → treated as fresh', async () => {
+    const store = makeStore();
+    const r = await store.put(AGENT, { source: 'https://x.com/a', content: 'a' });
+    await fs.writeFile(store._manifestPath(AGENT), '{ not json', 'utf8');
+    expect(await store.get(AGENT, r.docId)).toBeNull(); // fresh manifest
+  });
+
+  test('manifest with non-object docs → treated as fresh', async () => {
+    const store = makeStore();
+    await fs.mkdir(store._agentDir(AGENT), { recursive: true });
+    await fs.writeFile(store._manifestPath(AGENT), JSON.stringify({ docs: null }), 'utf8');
+    expect(await store.list(AGENT)).toEqual([]);
+  });
+
+  test('default baseDir resolves under state/agents when not injected', () => {
+    const store = new DocumentStore();
+    expect(store._agentDir(AGENT)).toMatch(/[\\/]state[\\/]agents[\\/]/);
+  });
+
+  test('exports sane constants', () => {
+    expect(MANIFEST_VERSION).toBe(1);
+    expect(DEFAULT_MAX_DOCS_PER_AGENT).toBeGreaterThan(0);
+  });
+});
+
+// ── retention-based eviction + pin + access ─────────────────────────────
+describe('retention eviction / pin / recordAccess', () => {
+  test('an old-but-accessed doc outlives a newer untouched one (retention > age)', async () => {
+    let t = 1000;
+    const store = makeStore({ maxDocsPerAgent: 2, now: () => t });
+    const a = await store.put(AGENT, { source: 'https://x.com/a', content: 'a' }); // oldest
+    t = 2000;
+    const b = await store.put(AGENT, { source: 'https://x.com/b', content: 'b' });
+    // Reinforce `a` so its retention beats the newer-but-untouched `b`.
+    t = 3000; await store.recordAccess(AGENT, a.docId);
+    t = 4000; await store.recordAccess(AGENT, a.docId);
+    t = 5000; await store.recordAccess(AGENT, a.docId);
+    t = 6000;
+    const c = await store.put(AGENT, { source: 'https://x.com/c', content: 'c' }); // over cap → evict 1
+
+    expect(c.evicted.map((e) => e.docId)).toEqual([b.docId]); // b evicted, not the older a
+    expect(await store.get(AGENT, a.docId)).not.toBeNull();   // accessed-old survives
+    expect(await store.get(AGENT, b.docId)).toBeNull();
+  });
+
+  test('equal-retention docs fall back to fetchedAt tie-break (degrades to LRU)', async () => {
+    // Fixed clock + no access ⇒ identical retention. Cap 2 + 3 docs ⇒ the
+    // eviction ranks TWO equal candidates, exercising the tie-break comparator.
+    const store = makeStore({ maxDocsPerAgent: 2, now: () => 5000 });
+    const a = await store.put(AGENT, { source: 'https://x.com/a', content: 'a' });
+    await store.put(AGENT, { source: 'https://x.com/b', content: 'b' });
+    await store.put(AGENT, { source: 'https://x.com/c', content: 'c' }); // evicts 1 of 3
+    expect(await store.list(AGENT)).toHaveLength(2);
+    expect(await store.get(AGENT, a.docId)).toBeNull(); // first-inserted evicted on tie
+  });
+
+  test('pinned docs are never evicted, even across multiple evictions', async () => {
+    let t = 1000;
+    const store = makeStore({ maxDocsPerAgent: 1, now: () => t });
+    const a = await store.put(AGENT, { source: 'https://x.com/a', content: 'a', pinned: true });
+    expect((await store.get(AGENT, a.docId)).pinned).toBe(true);
+    t = 2000; await store.put(AGENT, { source: 'https://x.com/b', content: 'b' });
+    t = 3000; await store.put(AGENT, { source: 'https://x.com/c', content: 'c' });
+    // a was the oldest and lowest-access, but pinned → survived both rounds.
+    expect(await store.get(AGENT, a.docId)).not.toBeNull();
+  });
+
+  test('all-pinned over cap keeps everything (pins win over the cap)', async () => {
+    let t = 0;
+    const store = makeStore({ maxDocsPerAgent: 1, now: () => (t += 1000) });
+    await store.put(AGENT, { source: 'https://x.com/a', content: 'a', pinned: true });
+    await store.put(AGENT, { source: 'https://x.com/b', content: 'b', pinned: true });
+    await store.put(AGENT, { source: 'https://x.com/c', content: 'c', pinned: true });
+    expect(await store.list(AGENT)).toHaveLength(3); // none evictable
+  });
+
+  test('pinned is preserved on dedup and overridable', async () => {
+    const store = makeStore();
+    const a = await store.put(AGENT, { source: 'https://x.com/a', content: 'same', pinned: true });
+    // dedup (same content), no pinned arg → stays pinned
+    await store.put(AGENT, { source: 'https://x.com/a', content: 'same' });
+    expect((await store.get(AGENT, a.docId)).pinned).toBe(true);
+    // dedup with explicit unpin
+    await store.put(AGENT, { source: 'https://x.com/a', content: 'same', pinned: false });
+    expect((await store.get(AGENT, a.docId)).pinned).toBe(false);
+  });
+
+  test('recordAccess bumps the decayed counter; unknown doc → null', async () => {
+    let t = 1000;
+    const store = makeStore({ now: () => t });
+    const a = await store.put(AGENT, { source: 'https://x.com/a', content: 'a' });
+    t = 2000; const e1 = await store.recordAccess(AGENT, a.docId);
+    t = 3000; const e2 = await store.recordAccess(AGENT, a.docId);
+    expect(e1.accessStat.v).toBeGreaterThan(0);
+    expect(e2.accessStat.v).toBeGreaterThan(e1.accessStat.v);
+    expect(e2.accessStat.t).toBe(3000);
+    expect(await store.recordAccess(AGENT, 'doc-nope')).toBeNull();
+  });
+
+  test('pin() toggles the flag; unknown doc → null', async () => {
+    const store = makeStore();
+    const a = await store.put(AGENT, { source: 'https://x.com/a', content: 'a' });
+    expect((await store.pin(AGENT, a.docId)).pinned).toBe(true);
+    expect((await store.pin(AGENT, a.docId, false)).pinned).toBe(false);
+    expect(await store.pin(AGENT, 'doc-nope')).toBeNull();
+  });
+
+  test('recordAccessMany bumps several docs in one pass; dedups + skips unknown', async () => {
+    const store = makeStore({ now: () => 5000 });
+    const a = await store.put(AGENT, { source: 'https://x.com/a', content: 'a' });
+    const b = await store.put(AGENT, { source: 'https://x.com/b', content: 'b' });
+    const updated = await store.recordAccessMany(AGENT, [a.docId, b.docId, a.docId, 'doc-nope']);
+    expect(updated.map((e) => e.docId).sort()).toEqual([a.docId, b.docId].sort());
+    expect((await store.get(AGENT, a.docId)).accessStat.v).toBeGreaterThan(0);
+    expect((await store.get(AGENT, b.docId)).accessStat.v).toBeGreaterThan(0);
+  });
+
+  test('recordAccessMany with empty / non-array → no-op []', async () => {
+    const store = makeStore();
+    expect(await store.recordAccessMany(AGENT, [])).toEqual([]);
+    expect(await store.recordAccessMany(AGENT, null)).toEqual([]);
+  });
+});

package/src/services/documents/documentMemory.js

@@ -0,0 +1,156 @@
+/**
+ * DocumentMemory — the coordination facade over DocumentStore (durable files
+ * + manifest) and DocumentIndexer (the `documents` vector surface).
+ *
+ * It exists so callers (the web tool, later any oversized-output producer)
+ * have ONE entry point and the store↔index stay consistent:
+ *   - remember():  persist + index in one call; purge vectors for any docs the
+ *                  store evicted (LRU); return a COMPACT handle (no payload).
+ *   - recall():    semantic/hybrid search over the agent's documents.
+ *   - forget():    remove file + vectors together.
+ *   - sweep():     drop expired docs (file + vectors).
+ *   - read()/list(): pass-throughs for handle hydration + the prompt index.
+ *
+ * Never throws on the index side — embeddings are best-effort; a doc is always
+ * persisted even if indexing is disabled/fails (recall just won't find it
+ * until re-indexed). `embeddingsEnabled` in the handle lets callers advise the
+ * user to configure embeddings.
+ */
+
+export class DocumentMemory {
+  /**
+   * @param {object} opts
+   * @param {import('./documentStore.js').DocumentStore} opts.store
+   * @param {import('../embeddings/indexers/documentIndexer.js').DocumentIndexer} opts.indexer
+   * @param {object} [opts.logger]
+   */
+  constructor({ store, indexer, logger = null }) {
+    if (!store) throw new TypeError('DocumentMemory requires a store');
+    if (!indexer) throw new TypeError('DocumentMemory requires an indexer');
+    this._store = store;
+    this._indexer = indexer;
+    this._logger = logger;
+  }
+
+  get embeddingsEnabled() {
+    return !!this._indexer.isEnabled;
+  }
+
+  /** Best-effort vector purge — index failures never break the store path. */
+  async _purge(agentId, docId) {
+    try {
+      await this._indexer.removeDocument(agentId, docId);
+    } catch (err) {
+      this._logger?.warn?.(`[documentMemory] vector purge failed for ${docId}: ${err?.message}`);
+    }
+  }
+
+  /**
+   * Persist + index a document, returning a compact handle (never the body).
+   *
+   * @param {string} agentId
+   * @param {object} doc  { source, content, title?, contentType?, ttlMs? }
+   * @returns {Promise<object>} handle:
+   *   { docId, source, title, bytes, lines, version, path, expiresAt,
+   *     deduped, changed, indexed, chunkCount, embeddingsEnabled }
+   */
+  async remember(agentId, doc) {
+    const put = await this._store.put(agentId, doc);
+
+    // Keep the index in lockstep with LRU eviction.
+    for (const evicted of put.evicted || []) {
+      await this._purge(agentId, evicted.docId);
+    }
+
+    let indexed = false;
+    let chunkCount = 0;
+    if (put.changed) {
+      // New or changed content → (re)index. indexDocument replaces this doc's
+      // existing chunks, so a shrunk doc never leaves orphans.
+      let res = { indexed: false, chunks: 0 };
+      try {
+        res = await this._indexer.indexDocument(agentId, {
+          docId: put.docId,
+          content: doc.content,
+          source: put.source,
+          title: put.title,
+        });
+      } catch (err) {
+        this._logger?.warn?.(`[documentMemory] index failed for ${put.docId}: ${err?.message}`);
+      }
+      indexed = !!res.indexed;
+      chunkCount = res.chunks || 0;
+    } else {
+      // Deduped (content unchanged) → existing vectors stand; skip re-embed.
+      indexed = this.embeddingsEnabled;
+    }
+
+    return {
+      docId: put.docId,
+      source: put.source,
+      title: put.title,
+      bytes: put.bytes,
+      lines: put.lines,
+      version: put.version,
+      path: put.path,
+      expiresAt: put.expiresAt,
+      deduped: put.deduped,
+      changed: put.changed,
+      indexed,
+      chunkCount,
+      embeddingsEnabled: this.embeddingsEnabled,
+    };
+  }
+
+  /**
+   * Semantic/hybrid recall over the agent's documents. Recalled docs are
+   * REINFORCED (their decayed access counter is bumped) so usage feeds the
+   * retention/eviction score — the aging-LFU "access strengthens" half. Bumps
+   * are batched into one manifest write and best-effort (never fail recall).
+   */
+  async recall(agentId, query, opts = {}) {
+    const res = await this._indexer.search(agentId, query, opts);
+    const docIds = [...new Set((res.hits || []).map((h) => h && h.docId).filter(Boolean))];
+    if (docIds.length > 0) {
+      try {
+        await this._store.recordAccessMany(agentId, docIds);
+      } catch (err) {
+        this._logger?.warn?.(`[documentMemory] access reinforcement failed: ${err?.message}`);
+      }
+    }
+    return res;
+  }
+
+  /** Pin (or unpin) a document so it is never evicted regardless of retention. */
+  async pin(agentId, docId, pinned = true) {
+    return this._store.pin(agentId, docId, pinned);
+  }
+
+  /** Remove a document everywhere (file + manifest + vectors). */
+  async forget(agentId, docId) {
+    const removed = await this._store.remove(agentId, docId);
+    await this._purge(agentId, docId);
+    return removed;
+  }
+
+  /** Drop all expired documents (file + vectors). @returns removed entries. */
+  async sweep(agentId) {
+    const removed = await this._store.sweepExpired(agentId);
+    for (const entry of removed) {
+      await this._purge(agentId, entry.docId);
+    }
+    return removed;
+  }
+
+  /** Read full or ranged content for handle hydration. */
+  async read(agentId, docId, range) {
+    return this._store.read(agentId, docId, range);
+  }
+
+  /** List manifest entries (newest-first) for the prompt docs index. */
+  async list(agentId) {
+    return this._store.list(agentId);
+  }
+}
+
+export default DocumentMemory;