onbuzz 4.9.13 → 4.10.0

package/src/services/embeddings/indexers/__tests__/reminisceIndexer.test.js

@@ -1,357 +1,356 @@
-/**
- * ReminisceIndexer tests.
- *
- * Strategy mirrors memoryIndexer.test.js: build a real EmbeddingService
- * + InMemoryJsonStore against a tmp dir, with a stub provider that
- * produces deterministic vectors. Exercises the full chunk → embed →
- * upsert → search → dedup path.
- *
- * Pins:
- *   - messageToText packing: "role: content" with optional tool args
- *   - Short messages produce 1 chunk; long messages produce many
- *   - IDs follow the `${messageId}#${chunkIndex}` convention
- *   - indexOne replaces previous chunks for the same message (no orphans
- *     when a message gets shorter)
- *   - search dedups by messageId, keeping highest-score chunk
- *   - deleteOne removes ALL chunks for a message
- *   - backfill drops + rebuilds, handles failures cleanly
- *   - Per-agent isolation
- *   - hybrid mode merges semantic + substring via RRF
- */
-
-import { describe, test, expect, jest, beforeEach, afterEach } from '@jest/globals';
-import os from 'os';
-import fsp from 'fs/promises';
-import path from 'path';
-import {
-  ReminisceIndexer,
-  messageToText,
-  estimateMessageTokens,
-} from '../reminisceIndexer.js';
-import { EmbeddingService } from '../../embeddingService.js';
-import { EmbeddingProvider } from '../../providerInterface.js';
-
-class StubProvider extends EmbeddingProvider {
-  constructor(dimensions = 8) {
-    super();
-    this._dimensions = dimensions;
-    this.calls = [];
-  }
-  getInfo() {
-    return {
-      name: 'azure', modelId: 'stub-rem', dimensions: this._dimensions,
-      maxBatchSize: 32, maxTokensPerText: 1000,
-      distinguishesQueryAndDocument: false, isLocal: false,
-    };
-  }
-  getModelFingerprint() { return 'stub-rem:v1'; }
-  async _embed(texts, kind) {
-    this.calls.push({ texts: [...texts], kind });
-    return texts.map(t => {
-      const idx = (t.charCodeAt(0) || 0) % this._dimensions;
-      const vec = Array(this._dimensions).fill(0);
-      vec[idx] = 1;
-      vec[(idx + 1) % this._dimensions] = (t.length % 7) * 0.1;
-      return vec;
-    });
-  }
-}
-
-function makeIndexer({ tmpDir, enabled = true, chunkMaxChars, chunkOverlapChars } = {}) {
-  const apiKeyManager = { getEmbeddingApiKey: jest.fn(() => 'stub-key') };
-  if (!enabled) {
-    const svc = new EmbeddingService({
-      apiKeyManager, baseUrl: 'https://x', stateDir: tmpDir,
-    });
-    return { indexer: new ReminisceIndexer({ embeddingService: svc, chunkMaxChars, chunkOverlapChars }), svc };
-  }
-  const provider = new StubProvider();
-  const svc = new EmbeddingService({
-    config: { provider: 'azure' },
-    apiKeyManager, baseUrl: 'https://x', stateDir: tmpDir,
-    providerOverrides: { azure: () => provider },
-  });
-  return {
-    indexer: new ReminisceIndexer({ embeddingService: svc, chunkMaxChars, chunkOverlapChars }),
-    svc, provider,
-  };
-}
-
-describe('messageToText', () => {
-  test('packs "role: content"', () => {
-    expect(messageToText({ role: 'user', content: 'hi' })).toBe('user: hi');
-  });
-
-  test('appends tool args on separate lines', () => {
-    const m = {
-      role: 'assistant',
-      content: 'running it now',
-      toolExecutions: [
-        { toolId: 'terminal', input: { command: 'ls -la' } },
-        { toolId: 'seek', input: 'TODO' },
-      ],
-    };
-    const text = messageToText(m);
-    expect(text).toContain('assistant: running it now');
-    expect(text).toContain('[tool: terminal]');
-    expect(text).toContain('"command":"ls -la"');
-    expect(text).toContain('[tool: seek]');
-    expect(text).toContain('TODO');
-  });
-
-  test('handles array-form content', () => {
-    const m = { role: 'user', content: [{ type: 'text', text: 'piece one' }, { type: 'text', text: ' piece two' }] };
-    expect(messageToText(m)).toBe('user: piece one piece two');
-  });
-
-  test('empty / null / no-content message returns empty string', () => {
-    // The role prefix is only emitted when there's content to attach;
-    // a content-less message produces no embeddable text.
-    expect(messageToText(null)).toBe('');
-    expect(messageToText({})).toBe('');
-    expect(messageToText({ role: 'user' })).toBe('');
-  });
-
-  test('tool results are NOT included (alignment with conversationQuery)', () => {
-    const m = {
-      role: 'assistant',
-      content: 'ok',
-      toolExecutions: [{ toolId: 'terminal', input: 'ls', output: 'a giant noisy blob that we do not embed' }],
-    };
-    expect(messageToText(m)).not.toContain('giant noisy blob');
-  });
-});
-
-describe('estimateMessageTokens', () => {
-  test('roughly chars/4', () => {
-    expect(estimateMessageTokens({ role: 'user', content: 'a'.repeat(40) })).toBeGreaterThanOrEqual(10);
-  });
-});
-
-describe('ReminisceIndexer — disabled path', () => {
-  let tmpDir;
-  beforeEach(async () => { tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'loxia-remidx-')); });
-  afterEach(async () => { await fsp.rm(tmpDir, { recursive: true, force: true }); });
-
-  test('isEnabled = false when service is disabled', () => {
-    const { indexer } = makeIndexer({ tmpDir, enabled: false });
-    expect(indexer.isEnabled).toBe(false);
-  });
-
-  test('indexOne returns {indexed:false, reason:"disabled"}', async () => {
-    const { indexer } = makeIndexer({ tmpDir, enabled: false });
-    const r = await indexer.indexOne('a1', { id: 'm1', role: 'user', content: 'hi' });
-    expect(r).toEqual({ indexed: false, reason: 'disabled' });
-  });
-
-  test('search returns enabled:false', async () => {
-    const { indexer } = makeIndexer({ tmpDir, enabled: false });
-    const r = await indexer.search('a1', 'hello');
-    expect(r).toEqual({ enabled: false, hits: [] });
-  });
-
-  test('backfill returns ok:false / reason:"disabled"', async () => {
-    const { indexer } = makeIndexer({ tmpDir, enabled: false });
-    const r = await indexer.backfill('a1', [{ id: 'm1', role: 'user', content: 'hi' }]);
-    expect(r).toEqual({ ok: false, reason: 'disabled' });
-  });
-});
-
-describe('ReminisceIndexer — single-message round-trip', () => {
-  let tmpDir, indexer;
-  beforeEach(async () => {
-    tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'loxia-remidx-'));
-    ({ indexer } = makeIndexer({ tmpDir }));
-  });
-  afterEach(async () => { await fsp.rm(tmpDir, { recursive: true, force: true }); });
-
-  test('short message → 1 chunk, retrievable by id', async () => {
-    const m = { id: 'msg-abc', role: 'user', content: 'telegram dedup discussion', createdAt: '2026-05-26T00:00:00Z' };
-    const r = await indexer.indexOne('a1', m);
-    expect(r.indexed).toBe(true);
-    expect(r.chunks).toBe(1);
-
-    const s = await indexer.search('a1', 'telegram dedup');
-    expect(s.enabled).toBe(true);
-    expect(s.hits.find(h => h.messageId === 'msg-abc')).toBeDefined();
-  });
-
-  test('long message → multiple chunks, search dedups by messageId', async () => {
-    // Build a ~3000-char message
-    const long = ('Sentence about scheduling. '.repeat(100));
-    const m = { id: 'msg-long', role: 'assistant', content: long, createdAt: '2026-05-26T00:00:00Z' };
-    const { indexer: idx } = makeIndexer({ tmpDir, chunkMaxChars: 600, chunkOverlapChars: 100 });
-    const r = await idx.indexOne('a1', m);
-    expect(r.indexed).toBe(true);
-    expect(r.chunks).toBeGreaterThan(1);
-
-    const s = await idx.search('a1', 'scheduling');
-    // Multiple chunks would otherwise produce multiple hits for same msg-long;
-    // dedup keeps a single row.
-    const msgLongHits = s.hits.filter(h => h.messageId === 'msg-long');
-    expect(msgLongHits).toHaveLength(1);
-  });
-
-  test('re-indexing a message that shrank removes orphan chunks', async () => {
-    const { indexer: idx, svc } = makeIndexer({ tmpDir, chunkMaxChars: 200, chunkOverlapChars: 40 });
-    // First version: 6 chunks
-    const long = 'A. '.repeat(200);   // 600 chars, multiple chunks
-    await idx.indexOne('a1', { id: 'm1', role: 'user', content: long });
-    const store1 = await svc.getStore({ agentId: 'a1', surface: 'reminisce' });
-    const before = (await store1.stats()).count;
-    expect(before).toBeGreaterThan(1);
-
-    // Re-index with much shorter content
-    await idx.indexOne('a1', { id: 'm1', role: 'user', content: 'tiny update' });
-    const after = (await store1.stats()).count;
-    expect(after).toBe(1);
-  });
-
-  test('deleteOne removes all chunks for the message', async () => {
-    const { indexer: idx, svc } = makeIndexer({ tmpDir, chunkMaxChars: 200, chunkOverlapChars: 40 });
-    await idx.indexOne('a1', { id: 'm1', role: 'user', content: 'A. '.repeat(200) });
-    await idx.indexOne('a1', { id: 'm2', role: 'assistant', content: 'kept' });
-    const store = await svc.getStore({ agentId: 'a1', surface: 'reminisce' });
-    const before = (await store.stats()).count;
-
-    const r = await idx.deleteOne('a1', 'm1');
-    expect(r.deleted).toBe(true);
-    expect(r.removed).toBeGreaterThan(0);
-
-    const after = (await store.stats()).count;
-    expect(after).toBe(1);   // only m2 remains
-  });
-
-  test('id format is `${messageId}#${chunkIndex}`', async () => {
-    const { indexer: idx, svc } = makeIndexer({ tmpDir, chunkMaxChars: 300, chunkOverlapChars: 50 });
-    await idx.indexOne('a1', { id: 'msg-x', role: 'user', content: 'A. '.repeat(300) });
-    const store = await svc.getStore({ agentId: 'a1', surface: 'reminisce' });
-    // Walk the store via internals — we just verify some id has the convention.
-    const sample = store._rows[0];
-    expect(sample.id).toMatch(/^msg-x#\d+$/);
-  });
-});
-
-describe('ReminisceIndexer — batched indexMany', () => {
-  let tmpDir, indexer, provider;
-  beforeEach(async () => {
-    tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'loxia-remidx-'));
-    ({ indexer, provider } = makeIndexer({ tmpDir }));
-  });
-  afterEach(async () => { await fsp.rm(tmpDir, { recursive: true, force: true }); });
-
-  test('indexMany batches all chunks into one embed call', async () => {
-    const msgs = [
-      { id: 'm1', role: 'user',      content: 'short one' },
-      { id: 'm2', role: 'assistant', content: 'short two' },
-      { id: 'm3', role: 'user',      content: 'short three' },
-    ];
-    const r = await indexer.indexMany('a1', msgs);
-    expect(r.indexedMessages).toBe(3);
-    expect(r.indexedChunks).toBeGreaterThanOrEqual(3);
-    expect(r.failed).toBe(0);
-    expect(provider.calls).toHaveLength(1);
-  });
-
-  test('skips messages with no id / no text', async () => {
-    const msgs = [
-      { id: 'm1', role: 'user', content: 'kept' },
-      { role: 'user', content: 'no id' },
-      { id: 'm-empty', role: 'user' },
-    ];
-    const r = await indexer.indexMany('a1', msgs);
-    expect(r.indexedMessages).toBe(1);
-    expect(r.skipped).toBe(2);
-  });
-
-  test('long messages within the batch are split into chunks', async () => {
-    const { indexer: idx, provider: p } = makeIndexer({
-      tmpDir, chunkMaxChars: 200, chunkOverlapChars: 40,
-    });
-    const msgs = [
-      { id: 'short', role: 'user', content: 'just a short one' },
-      { id: 'long', role: 'assistant', content: 'A. '.repeat(200) },   // ~600 chars → 3+ chunks
-    ];
-    const r = await idx.indexMany('a1', msgs);
-    expect(r.indexedMessages).toBe(2);
-    expect(r.indexedChunks).toBeGreaterThan(3);
-    expect(p.calls).toHaveLength(1);
-    expect(p.calls[0].texts.length).toBe(r.indexedChunks);
-  });
-
-  test('embed failure surfaces failed counts cleanly', async () => {
-    const apiKeyManager = { getEmbeddingApiKey: () => 'k' };
-    class FailingProvider extends StubProvider {
-      async _embed() { throw new Error('rate-limited'); }
-    }
-    const svc = new EmbeddingService({
-      config: { provider: 'azure' },
-      apiKeyManager, baseUrl: 'https://x', stateDir: tmpDir,
-      providerOverrides: { azure: () => new FailingProvider() },
-    });
-    const idx = new ReminisceIndexer({ embeddingService: svc });
-    const r = await idx.indexMany('a1', [
-      { id: 'm1', role: 'user', content: 'a' },
-      { id: 'm2', role: 'user', content: 'b' },
-    ]);
-    expect(r.indexedMessages).toBe(0);
-    expect(r.failed).toBe(2);
-    expect(r.error).toMatch(/rate-limited/);
-  });
-});
-
-describe('ReminisceIndexer — backfill', () => {
-  let tmpDir;
-  beforeEach(async () => { tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'loxia-remidx-')); });
-  afterEach(async () => { await fsp.rm(tmpDir, { recursive: true, force: true }); });
-
-  test('backfill drops existing index and reindexes the input set', async () => {
-    const { indexer } = makeIndexer({ tmpDir });
-    await indexer.indexOne('a1', { id: 'stale', role: 'user', content: 'old' });
-
-    const replacement = [
-      { id: 'new1', role: 'user', content: 'fresh one' },
-      { id: 'new2', role: 'assistant', content: 'fresh two' },
-    ];
-    const r = await indexer.backfill('a1', replacement);
-    expect(r.ok).toBe(true);
-    expect(r.indexedMessages).toBe(2);
-
-    const s = await indexer.search('a1', 'fresh');
-    expect(s.hits.find(h => h.messageId === 'stale')).toBeUndefined();
-    expect(s.hits.find(h => h.messageId === 'new1')).toBeDefined();
-  });
-});
-
-describe('ReminisceIndexer — per-agent isolation + hybrid', () => {
-  let tmpDir, indexer;
-  beforeEach(async () => {
-    tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'loxia-remidx-'));
-    ({ indexer } = makeIndexer({ tmpDir }));
-  });
-  afterEach(async () => { await fsp.rm(tmpDir, { recursive: true, force: true }); });
-
-  test('search on a1 does not see a2 messages', async () => {
-    await indexer.indexOne('a1', { id: 'm1', role: 'user', content: 'shared text alpha' });
-    await indexer.indexOne('a2', { id: 'm1', role: 'user', content: 'shared text beta' });
-    const r = await indexer.search('a1', 'shared');
-    expect(r.hits.every(h => (h.chunkText || '').includes('alpha'))).toBe(true);
-  });
-
-  test('hybrid search surfaces substring-only matches', async () => {
-    await indexer.indexOne('a1', {
-      id: 'm-needle',
-      role: 'assistant',
-      content: 'this message mentions UNIQUE_RARE_TOKEN explicitly somewhere in the middle',
-    });
-    const r = await indexer.search('a1', 'UNIQUE_RARE_TOKEN');
-    expect(r.hits.some(h => h.messageId === 'm-needle')).toBe(true);
-  });
-
-  test('hybrid:false skips substring fusion', async () => {
-    await indexer.indexOne('a1', { id: 'm1', role: 'user', content: 'something' });
-    const r = await indexer.search('a1', 'something', { hybrid: false });
-    expect(r.enabled).toBe(true);
-    expect(Array.isArray(r.hits)).toBe(true);
-  });
-});
+/**
+ * ReminisceIndexer tests.
+ *
+ * Strategy mirrors memoryIndexer.test.js: build a real EmbeddingService
+ * + InMemoryJsonStore against a tmp dir, with a stub provider that
+ * produces deterministic vectors. Exercises the full chunk → embed →
+ * upsert → search → dedup path.
+ *
+ * Pins:
+ *   - messageToText packing: "role: content" with optional tool args
+ *   - Short messages produce 1 chunk; long messages produce many
+ *   - IDs follow the `${messageId}#${chunkIndex}` convention
+ *   - indexOne replaces previous chunks for the same message (no orphans
+ *     when a message gets shorter)
+ *   - search dedups by messageId, keeping highest-score chunk
+ *   - deleteOne removes ALL chunks for a message
+ *   - backfill drops + rebuilds, handles failures cleanly
+ *   - Per-agent isolation
+ *   - hybrid mode merges semantic + substring via RRF
+ */
+
+import { describe, test, expect, jest, beforeEach, afterEach } from '@jest/globals';
+import os from 'os';
+import fsp from 'fs/promises';
+import path from 'path';
+import {
+  ReminisceIndexer,
+  messageToText,
+  estimateMessageTokens,
+} from '../reminisceIndexer.js';
+import { EmbeddingService } from '../../embeddingService.js';
+import { EmbeddingProvider } from '../../providerInterface.js';
+
+class StubProvider extends EmbeddingProvider {
+  constructor(dimensions = 8) {
+    super();
+    this._dimensions = dimensions;
+    this.calls = [];
+  }
+  getInfo() {
+    return {
+      name: 'azure', modelId: 'stub-rem', dimensions: this._dimensions,
+      maxBatchSize: 32, maxTokensPerText: 1000,
+      distinguishesQueryAndDocument: false, isLocal: false,
+    };
+  }
+  getModelFingerprint() { return 'stub-rem:v1'; }
+  async _embed(texts, kind) {
+    this.calls.push({ texts: [...texts], kind });
+    return texts.map(t => {
+      const idx = (t.charCodeAt(0) || 0) % this._dimensions;
+      const vec = Array(this._dimensions).fill(0);
+      vec[idx] = 1;
+      vec[(idx + 1) % this._dimensions] = (t.length % 7) * 0.1;
+      return vec;
+    });
+  }
+}
+
+function makeIndexer({ tmpDir, enabled = true, chunkMaxChars, chunkOverlapChars } = {}) {
+  const apiKeyManager = { getEmbeddingApiKey: jest.fn(() => 'stub-key') };
+  if (!enabled) {
+    const svc = new EmbeddingService({
+      apiKeyManager, baseUrl: 'https://x', stateDir: tmpDir,
+    });
+    return { indexer: new ReminisceIndexer({ embeddingService: svc, chunkMaxChars, chunkOverlapChars }), svc };
+  }
+  const provider = new StubProvider();
+  const svc = new EmbeddingService({
+    config: { provider: 'azure' },
+    apiKeyManager, baseUrl: 'https://x', stateDir: tmpDir,
+    providerOverrides: { azure: () => provider },
+  });
+  return {
+    indexer: new ReminisceIndexer({ embeddingService: svc, chunkMaxChars, chunkOverlapChars }),
+    svc, provider,
+  };
+}
+
+describe('messageToText', () => {
+  test('packs "role: content"', () => {
+    expect(messageToText({ role: 'user', content: 'hi' })).toBe('user: hi');
+  });
+
+  test('appends tool args on separate lines', () => {
+    const m = {
+      role: 'assistant',
+      content: 'running it now',
+      toolExecutions: [
+        { toolId: 'terminal', input: { command: 'ls -la' } },
+        { toolId: 'seek', input: 'TODO' },
+      ],
+    };
+    const text = messageToText(m);
+    expect(text).toContain('assistant: running it now');
+    expect(text).toContain('[tool: terminal]');
+    expect(text).toContain('"command":"ls -la"');
+    expect(text).toContain('[tool: seek]');
+    expect(text).toContain('TODO');
+  });
+
+  test('handles array-form content', () => {
+    const m = { role: 'user', content: [{ type: 'text', text: 'piece one' }, { type: 'text', text: ' piece two' }] };
+    expect(messageToText(m)).toBe('user: piece one piece two');
+  });
+
+  test('empty / null / no-content message returns empty string', () => {
+    // The role prefix is only emitted when there's content to attach;
+    // a content-less message produces no embeddable text.
+    expect(messageToText(null)).toBe('');
+    expect(messageToText({})).toBe('');
+    expect(messageToText({ role: 'user' })).toBe('');
+  });
+
+  test('tool results are NOT included (alignment with conversationQuery)', () => {
+    const m = {
+      role: 'assistant',
+      content: 'ok',
+      toolExecutions: [{ toolId: 'terminal', input: 'ls', output: 'a giant noisy blob that we do not embed' }],
+    };
+    expect(messageToText(m)).not.toContain('giant noisy blob');
+  });
+});
+
+describe('estimateMessageTokens', () => {
+  test('roughly chars/4', () => {
+    expect(estimateMessageTokens({ role: 'user', content: 'a'.repeat(40) })).toBeGreaterThanOrEqual(10);
+  });
+});
+
+describe('ReminisceIndexer — disabled path', () => {
+  let tmpDir;
+  beforeEach(async () => { tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'loxia-remidx-')); });
+  afterEach(async () => { await fsp.rm(tmpDir, { recursive: true, force: true }); });
+
+  test('isEnabled = false when service is disabled', () => {
+    const { indexer } = makeIndexer({ tmpDir, enabled: false });
+    expect(indexer.isEnabled).toBe(false);
+  });
+
+  test('indexOne returns {indexed:false, reason:"disabled"}', async () => {
+    const { indexer } = makeIndexer({ tmpDir, enabled: false });
+    const r = await indexer.indexOne('a1', { id: 'm1', role: 'user', content: 'hi' });
+    expect(r).toEqual({ indexed: false, reason: 'disabled' });
+  });
+
+  test('search returns enabled:false', async () => {
+    const { indexer } = makeIndexer({ tmpDir, enabled: false });
+    const r = await indexer.search('a1', 'hello');
+    expect(r).toEqual({ enabled: false, hits: [] });
+  });
+
+  test('backfill returns ok:false / reason:"disabled"', async () => {
+    const { indexer } = makeIndexer({ tmpDir, enabled: false });
+    const r = await indexer.backfill('a1', [{ id: 'm1', role: 'user', content: 'hi' }]);
+    expect(r).toEqual({ ok: false, reason: 'disabled' });
+  });
+});
+
+describe('ReminisceIndexer — single-message round-trip', () => {
+  let tmpDir, indexer;
+  beforeEach(async () => {
+    tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'loxia-remidx-'));
+    ({ indexer } = makeIndexer({ tmpDir }));
+  });
+  afterEach(async () => { await fsp.rm(tmpDir, { recursive: true, force: true }); });
+
+  test('short message → 1 chunk, retrievable by id', async () => {
+    const m = { id: 'msg-abc', role: 'user', content: 'telegram dedup discussion', createdAt: '2026-05-26T00:00:00Z' };
+    const r = await indexer.indexOne('a1', m);
+    expect(r.indexed).toBe(true);
+    expect(r.chunks).toBe(1);
+
+    const s = await indexer.search('a1', 'telegram dedup');
+    expect(s.enabled).toBe(true);
+    expect(s.hits.find(h => h.messageId === 'msg-abc')).toBeDefined();
+  });
+
+  test('long message → multiple chunks, search dedups by messageId', async () => {
+    // Build a ~3000-char message
+    const long = ('Sentence about scheduling. '.repeat(100));
+    const m = { id: 'msg-long', role: 'assistant', content: long, createdAt: '2026-05-26T00:00:00Z' };
+    const { indexer: idx } = makeIndexer({ tmpDir, chunkMaxChars: 600, chunkOverlapChars: 100 });
+    const r = await idx.indexOne('a1', m);
+    expect(r.indexed).toBe(true);
+    expect(r.chunks).toBeGreaterThan(1);
+
+    const s = await idx.search('a1', 'scheduling');
+    // Multiple chunks would otherwise produce multiple hits for same msg-long;
+    // dedup keeps a single row.
+    const msgLongHits = s.hits.filter(h => h.messageId === 'msg-long');
+    expect(msgLongHits).toHaveLength(1);
+  });
+
+  test('re-indexing a message that shrank removes orphan chunks', async () => {
+    const { indexer: idx, svc } = makeIndexer({ tmpDir, chunkMaxChars: 200, chunkOverlapChars: 40 });
+    // First version: 6 chunks
+    const long = 'A. '.repeat(200);   // 600 chars, multiple chunks
+    await idx.indexOne('a1', { id: 'm1', role: 'user', content: long });
+    const store1 = await svc.getStore({ agentId: 'a1', surface: 'reminisce' });
+    const before = (await store1.stats()).count;
+    expect(before).toBeGreaterThan(1);
+
+    // Re-index with much shorter content
+    await idx.indexOne('a1', { id: 'm1', role: 'user', content: 'tiny update' });
+    const after = (await store1.stats()).count;
+    expect(after).toBe(1);
+  });
+
+  test('deleteOne removes all chunks for the message', async () => {
+    const { indexer: idx, svc } = makeIndexer({ tmpDir, chunkMaxChars: 200, chunkOverlapChars: 40 });
+    await idx.indexOne('a1', { id: 'm1', role: 'user', content: 'A. '.repeat(200) });
+    await idx.indexOne('a1', { id: 'm2', role: 'assistant', content: 'kept' });
+    const store = await svc.getStore({ agentId: 'a1', surface: 'reminisce' });
+
+    const r = await idx.deleteOne('a1', 'm1');
+    expect(r.deleted).toBe(true);
+    expect(r.removed).toBeGreaterThan(0);
+
+    const after = (await store.stats()).count;
+    expect(after).toBe(1);   // only m2 remains
+  });
+
+  test('id format is `${messageId}#${chunkIndex}`', async () => {
+    const { indexer: idx, svc } = makeIndexer({ tmpDir, chunkMaxChars: 300, chunkOverlapChars: 50 });
+    await idx.indexOne('a1', { id: 'msg-x', role: 'user', content: 'A. '.repeat(300) });
+    const store = await svc.getStore({ agentId: 'a1', surface: 'reminisce' });
+    // Walk the store via internals — we just verify some id has the convention.
+    const sample = store._rows[0];
+    expect(sample.id).toMatch(/^msg-x#\d+$/);
+  });
+});
+
+describe('ReminisceIndexer — batched indexMany', () => {
+  let tmpDir, indexer, provider;
+  beforeEach(async () => {
+    tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'loxia-remidx-'));
+    ({ indexer, provider } = makeIndexer({ tmpDir }));
+  });
+  afterEach(async () => { await fsp.rm(tmpDir, { recursive: true, force: true }); });
+
+  test('indexMany batches all chunks into one embed call', async () => {
+    const msgs = [
+      { id: 'm1', role: 'user',      content: 'short one' },
+      { id: 'm2', role: 'assistant', content: 'short two' },
+      { id: 'm3', role: 'user',      content: 'short three' },
+    ];
+    const r = await indexer.indexMany('a1', msgs);
+    expect(r.indexedMessages).toBe(3);
+    expect(r.indexedChunks).toBeGreaterThanOrEqual(3);
+    expect(r.failed).toBe(0);
+    expect(provider.calls).toHaveLength(1);
+  });
+
+  test('skips messages with no id / no text', async () => {
+    const msgs = [
+      { id: 'm1', role: 'user', content: 'kept' },
+      { role: 'user', content: 'no id' },
+      { id: 'm-empty', role: 'user' },
+    ];
+    const r = await indexer.indexMany('a1', msgs);
+    expect(r.indexedMessages).toBe(1);
+    expect(r.skipped).toBe(2);
+  });
+
+  test('long messages within the batch are split into chunks', async () => {
+    const { indexer: idx, provider: p } = makeIndexer({
+      tmpDir, chunkMaxChars: 200, chunkOverlapChars: 40,
+    });
+    const msgs = [
+      { id: 'short', role: 'user', content: 'just a short one' },
+      { id: 'long', role: 'assistant', content: 'A. '.repeat(200) },   // ~600 chars → 3+ chunks
+    ];
+    const r = await idx.indexMany('a1', msgs);
+    expect(r.indexedMessages).toBe(2);
+    expect(r.indexedChunks).toBeGreaterThan(3);
+    expect(p.calls).toHaveLength(1);
+    expect(p.calls[0].texts.length).toBe(r.indexedChunks);
+  });
+
+  test('embed failure surfaces failed counts cleanly', async () => {
+    const apiKeyManager = { getEmbeddingApiKey: () => 'k' };
+    class FailingProvider extends StubProvider {
+      async _embed() { throw new Error('rate-limited'); }
+    }
+    const svc = new EmbeddingService({
+      config: { provider: 'azure' },
+      apiKeyManager, baseUrl: 'https://x', stateDir: tmpDir,
+      providerOverrides: { azure: () => new FailingProvider() },
+    });
+    const idx = new ReminisceIndexer({ embeddingService: svc });
+    const r = await idx.indexMany('a1', [
+      { id: 'm1', role: 'user', content: 'a' },
+      { id: 'm2', role: 'user', content: 'b' },
+    ]);
+    expect(r.indexedMessages).toBe(0);
+    expect(r.failed).toBe(2);
+    expect(r.error).toMatch(/rate-limited/);
+  });
+});
+
+describe('ReminisceIndexer — backfill', () => {
+  let tmpDir;
+  beforeEach(async () => { tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'loxia-remidx-')); });
+  afterEach(async () => { await fsp.rm(tmpDir, { recursive: true, force: true }); });
+
+  test('backfill drops existing index and reindexes the input set', async () => {
+    const { indexer } = makeIndexer({ tmpDir });
+    await indexer.indexOne('a1', { id: 'stale', role: 'user', content: 'old' });
+
+    const replacement = [
+      { id: 'new1', role: 'user', content: 'fresh one' },
+      { id: 'new2', role: 'assistant', content: 'fresh two' },
+    ];
+    const r = await indexer.backfill('a1', replacement);
+    expect(r.ok).toBe(true);
+    expect(r.indexedMessages).toBe(2);
+
+    const s = await indexer.search('a1', 'fresh');
+    expect(s.hits.find(h => h.messageId === 'stale')).toBeUndefined();
+    expect(s.hits.find(h => h.messageId === 'new1')).toBeDefined();
+  });
+});
+
+describe('ReminisceIndexer — per-agent isolation + hybrid', () => {
+  let tmpDir, indexer;
+  beforeEach(async () => {
+    tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'loxia-remidx-'));
+    ({ indexer } = makeIndexer({ tmpDir }));
+  });
+  afterEach(async () => { await fsp.rm(tmpDir, { recursive: true, force: true }); });
+
+  test('search on a1 does not see a2 messages', async () => {
+    await indexer.indexOne('a1', { id: 'm1', role: 'user', content: 'shared text alpha' });
+    await indexer.indexOne('a2', { id: 'm1', role: 'user', content: 'shared text beta' });
+    const r = await indexer.search('a1', 'shared');
+    expect(r.hits.every(h => (h.chunkText || '').includes('alpha'))).toBe(true);
+  });
+
+  test('hybrid search surfaces substring-only matches', async () => {
+    await indexer.indexOne('a1', {
+      id: 'm-needle',
+      role: 'assistant',
+      content: 'this message mentions UNIQUE_RARE_TOKEN explicitly somewhere in the middle',
+    });
+    const r = await indexer.search('a1', 'UNIQUE_RARE_TOKEN');
+    expect(r.hits.some(h => h.messageId === 'm-needle')).toBe(true);
+  });
+
+  test('hybrid:false skips substring fusion', async () => {
+    await indexer.indexOne('a1', { id: 'm1', role: 'user', content: 'something' });
+    const r = await indexer.search('a1', 'something', { hybrid: false });
+    expect(r.enabled).toBe(true);
+    expect(Array.isArray(r.hits)).toBe(true);
+  });
+});