@shadowforge0/aquifer-memory 1.0.3 → 1.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/README.md +29 -20
  2. package/consumers/claude-code.js +117 -0
  3. package/consumers/cli.js +17 -0
  4. package/consumers/default/daily-entries.js +196 -0
  5. package/consumers/default/index.js +282 -0
  6. package/consumers/default/prompts/summary.js +153 -0
  7. package/consumers/mcp.js +3 -23
  8. package/consumers/miranda/context-inject.js +119 -0
  9. package/consumers/miranda/daily-entries.js +224 -0
  10. package/consumers/miranda/index.js +353 -0
  11. package/consumers/miranda/instance.js +55 -0
  12. package/consumers/miranda/llm.js +99 -0
  13. package/consumers/miranda/prompts/summary.js +303 -0
  14. package/consumers/miranda/recall-format.js +74 -0
  15. package/consumers/miranda/workspace-files.js +91 -0
  16. package/consumers/openclaw-ext/index.js +38 -0
  17. package/consumers/openclaw-ext/openclaw.plugin.json +9 -0
  18. package/consumers/openclaw-ext/package.json +10 -0
  19. package/consumers/openclaw-plugin.js +66 -74
  20. package/consumers/opencode.js +21 -24
  21. package/consumers/shared/autodetect.js +64 -0
  22. package/consumers/shared/entity-parser.js +119 -0
  23. package/consumers/shared/ingest.js +148 -0
  24. package/consumers/shared/llm-autodetect.js +137 -0
  25. package/consumers/shared/normalize.js +129 -0
  26. package/consumers/shared/recall-format.js +110 -0
  27. package/core/aquifer.js +180 -71
  28. package/core/entity.js +1 -3
  29. package/core/storage.js +86 -28
  30. package/docs/postprocess-contract.md +132 -0
  31. package/index.js +9 -1
  32. package/package.json +23 -2
  33. package/pipeline/_http.js +1 -1
  34. package/pipeline/consolidation/apply.js +176 -0
  35. package/pipeline/consolidation/index.js +21 -0
  36. package/pipeline/extract-entities.js +2 -2
  37. package/pipeline/rerank.js +1 -1
  38. package/pipeline/summarize.js +4 -1
  39. package/schema/001-base.sql +61 -24
  40. package/schema/002-entities.sql +17 -3
  41. package/schema/004-facts.sql +67 -0
  42. package/scripts/diagnose-fts-zh.js +168 -134
  43. package/scripts/diagnose-vector.js +188 -0
  44. package/scripts/install-openclaw.sh +59 -0
  45. package/scripts/smoke.mjs +2 -2
package/core/aquifer.js CHANGED
@@ -9,6 +9,7 @@ const entity = require('./entity');
9
9
  const { hybridRank } = require('./hybrid-rank');
10
10
  const { summarize } = require('../pipeline/summarize');
11
11
  const { extractEntities } = require('../pipeline/extract-entities');
12
+ const { createEmbedder } = require('../pipeline/embed');
12
13
 
13
14
  // ---------------------------------------------------------------------------
14
15
  // Schema name validation
@@ -54,48 +55,104 @@ function buildRerankDocument(row, maxChars) {
54
55
  return text;
55
56
  }
56
57
 
58
+ // ---------------------------------------------------------------------------
59
+ // resolveEmbedFn — v1.2.0 embed autodetect (explicit > object > env > null)
60
+ // ---------------------------------------------------------------------------
61
+
62
+ function resolveEmbedFn(embedConfig, env) {
63
+ if (embedConfig && typeof embedConfig.fn === 'function') {
64
+ return embedConfig.fn;
65
+ }
66
+ if (embedConfig && embedConfig.provider) {
67
+ const embedder = createEmbedder(embedConfig);
68
+ return (texts) => embedder.embedBatch(texts);
69
+ }
70
+ const provider = env.EMBED_PROVIDER;
71
+ if (!provider) return null;
72
+
73
+ const opts = { provider };
74
+ if (provider === 'ollama') {
75
+ opts.ollamaUrl = env.OLLAMA_URL || env.AQUIFER_EMBED_BASE_URL || 'http://localhost:11434';
76
+ opts.model = env.AQUIFER_EMBED_MODEL || 'bge-m3';
77
+ } else if (provider === 'openai') {
78
+ opts.openaiApiKey = env.OPENAI_API_KEY;
79
+ if (!opts.openaiApiKey) {
80
+ throw new Error('EMBED_PROVIDER=openai requires OPENAI_API_KEY');
81
+ }
82
+ opts.openaiModel = env.AQUIFER_EMBED_MODEL || 'text-embedding-3-small';
83
+ if (env.AQUIFER_EMBED_DIM) opts.openaiDimensions = Number(env.AQUIFER_EMBED_DIM);
84
+ } else {
85
+ throw new Error(`EMBED_PROVIDER=${provider} not supported by autodetect (use 'ollama' or 'openai', or pass config.embed.fn explicitly)`);
86
+ }
87
+ const embedder = createEmbedder(opts);
88
+ return (texts) => embedder.embedBatch(texts);
89
+ }
90
+
57
91
  // ---------------------------------------------------------------------------
58
92
  // createAquifer
59
93
  // ---------------------------------------------------------------------------
60
94
 
61
- function createAquifer(config) {
62
- if (!config || !config.db) {
63
- throw new Error('config.db (pg.Pool or connection string) is required');
95
+ function createAquifer(config = {}) {
96
+ // v1.2.0: db falls back to DATABASE_URL / AQUIFER_DB_URL env so hosts can
97
+ // call createAquifer() with zero args for install-and-go.
98
+ const dbInput = config.db !== undefined
99
+ ? config.db
100
+ : (process.env.DATABASE_URL || process.env.AQUIFER_DB_URL || null);
101
+
102
+ if (!dbInput) {
103
+ throw new Error(
104
+ 'Aquifer requires a database: pass config.db (pg.Pool or connection string), '
105
+ + 'or set DATABASE_URL / AQUIFER_DB_URL in the environment.'
106
+ );
64
107
  }
65
108
 
66
- const schema = config.schema || 'aquifer';
109
+ const schema = config.schema || process.env.AQUIFER_SCHEMA || 'aquifer';
67
110
  validateSchema(schema);
68
111
 
69
112
  if (config.tenantId === '') throw new Error('config.tenantId must not be empty');
70
- const tenantId = config.tenantId || 'default';
113
+ const tenantId = config.tenantId || process.env.AQUIFER_TENANT_ID || 'default';
71
114
 
72
115
  // Pool management
73
116
  let pool;
74
117
  let ownsPool = false;
75
- if (typeof config.db === 'string') {
76
- pool = new Pool({ connectionString: config.db });
118
+ if (typeof dbInput === 'string') {
119
+ pool = new Pool({ connectionString: dbInput });
77
120
  ownsPool = true;
78
121
  } else {
79
- pool = config.db;
122
+ pool = dbInput;
80
123
  ownsPool = !!config.ownsPool; // allow factory to claim ownership
81
124
  }
82
125
 
83
126
  // Embed config (lazy — only required for recall/enrich)
84
- const embedFn = config.embed && typeof config.embed.fn === 'function' ? config.embed.fn : null;
85
- let embedDim = config.embed ? (config.embed.dim || null) : null;
86
-
127
+ // v1.2.0 fallback chain:
128
+ // 1. config.embed.fn (explicit function)
129
+ // 2. config.embed.provider (build via createEmbedder)
130
+ // 3. EMBED_PROVIDER env + provider-specific key (zero-arg install-and-go)
131
+ // 4. null — defer to requireEmbed() at call time
132
+ const embedFn = resolveEmbedFn(config.embed, process.env);
87
133
  function requireEmbed(op) {
88
- if (!embedFn) throw new Error(`Aquifer.${op}() requires config.embed.fn (async (texts) => number[][])`);
134
+ if (!embedFn) throw new Error(`Aquifer.${op}() requires config.embed.fn or EMBED_PROVIDER env (async (texts) => number[][])`);
89
135
  }
90
136
 
91
137
  // LLM config (optional — only needed for enrich with built-in summarize)
92
- const llmFn = config.llm && typeof config.llm.fn === 'function' ? config.llm.fn : null;
138
+ // v1.2.0: falls back to AQUIFER_LLM_PROVIDER env + provider-specific key.
139
+ const { resolveLlmFn } = require('../consumers/shared/llm-autodetect');
140
+ const llmFn = resolveLlmFn(config.llm, process.env);
93
141
 
94
142
  // Summarize config
95
143
  const summarizePromptFn = config.summarize && config.summarize.prompt ? config.summarize.prompt : null;
96
144
 
145
+ // Enrich stale-claim window: a 'processing' session older than this is
146
+ // reclaimable by a concurrent enrich() caller (covers crashed workers).
147
+ const staleEnrichMinutes = Number.isFinite(config.staleEnrichMinutes)
148
+ ? Math.max(1, Math.floor(config.staleEnrichMinutes))
149
+ : 10;
150
+
97
151
  // Entity config
98
152
  let entitiesEnabled = config.entities && config.entities.enabled === true;
153
+
154
+ // Facts config (opt-in consolidation lifecycle)
155
+ let factsEnabled = config.facts && config.facts.enabled === true;
99
156
  const mergeCall = config.entities && config.entities.mergeCall !== undefined ? config.entities.mergeCall : true;
100
157
  const entityPromptFn = config.entities && config.entities.prompt ? config.entities.prompt : null;
101
158
  const entityScope = (config.entities && config.entities.scope) || 'default';
@@ -207,9 +264,17 @@ function createAquifer(config) {
207
264
  const trustSql = loadSql('003-trust-feedback.sql', schema);
208
265
  await pool.query(trustSql);
209
266
 
267
+ // 4. Facts / consolidation (opt-in)
268
+ if (factsEnabled) {
269
+ const factsSql = loadSql('004-facts.sql', schema);
270
+ await pool.query(factsSql);
271
+ }
272
+
210
273
  migrated = true;
211
274
  } finally {
212
- await pool.query('SELECT pg_advisory_unlock($1)', [lockKey]).catch(() => {});
275
+ await pool.query('SELECT pg_advisory_unlock($1)', [lockKey]).catch((err) => {
276
+ console.warn(`[aquifer] failed to release migration advisory lock for schema "${schema}": ${err.message}`);
277
+ });
213
278
  }
214
279
  },
215
280
 
@@ -225,7 +290,7 @@ function createAquifer(config) {
225
290
  sources.set(name, {
226
291
  type: opts.type || 'custom',
227
292
  search: opts.search || null,
228
- weight: opts.weight !== null && opts.weight !== undefined ? opts.weight : 1.0,
293
+ weight: opts.weight !== undefined && opts.weight !== undefined ? opts.weight : 1.0,
229
294
  });
230
295
  },
231
296
 
@@ -238,6 +303,32 @@ function createAquifer(config) {
238
303
  }
239
304
  },
240
305
 
306
+ async enableFacts() {
307
+ factsEnabled = true;
308
+ // Run the facts DDL (idempotent — all CREATE/ALTER use IF NOT EXISTS).
309
+ // Safe to call repeatedly; also safe to call before migrate() (will no-op
310
+ // until base schema exists, which enrich/commit will materialize).
311
+ await ensureMigrated();
312
+ const factsSql = loadSql('004-facts.sql', schema);
313
+ await pool.query(factsSql);
314
+ },
315
+
316
+ async consolidate(sessionId, opts = {}) {
317
+ if (!factsEnabled) throw new Error('aquifer.consolidate() requires enableFacts() first');
318
+ await ensureMigrated();
319
+ const { applyConsolidation } = require('../pipeline/consolidation');
320
+ const agentId = opts.agentId || 'agent';
321
+ return applyConsolidation(pool, {
322
+ actions: opts.actions || [],
323
+ agentId,
324
+ sessionId,
325
+ schema,
326
+ tenantId,
327
+ normalizeSubject: opts.normalizeSubject || null,
328
+ recapOverview: opts.recapOverview || '',
329
+ });
330
+ },
331
+
241
332
  // --- write path ---
242
333
 
243
334
  async commit(sessionId, messages, opts = {}) {
@@ -302,17 +393,19 @@ function createAquifer(config) {
302
393
  const postProcess = opts.postProcess || null; // async (ctx) => void
303
394
  const optModel = 'model' in opts ? opts.model : undefined; // undefined = no override
304
395
 
305
- // 1. Optimistic lock: claim session for processing
306
- // Also reclaim stale 'processing' sessions (stuck > 10 min = likely killed process)
307
- const STALE_MINUTES = 10;
396
+ // 1. Optimistic lock: claim session for processing.
397
+ // Also reclaim stale 'processing' sessions (likely killed worker).
398
+ // Stale window is config.staleEnrichMinutes (default 10).
308
399
  const claimResult = await pool.query(
309
400
  `UPDATE ${qi(schema)}.sessions
310
401
  SET processing_status = 'processing', processing_started_at = NOW()
311
402
  WHERE session_id = $1 AND agent_id = $2 AND tenant_id = $3
312
403
  AND (processing_status IN ('pending', 'failed')
313
- OR (processing_status = 'processing' AND (processing_started_at IS NULL OR processing_started_at < NOW() - INTERVAL '${STALE_MINUTES} minutes')))
404
+ OR (processing_status = 'processing'
405
+ AND (processing_started_at IS NULL
406
+ OR processing_started_at < NOW() - make_interval(mins => $4))))
314
407
  RETURNING *`,
315
- [sessionId, agentId, tenantId]
408
+ [sessionId, agentId, tenantId, staleEnrichMinutes]
316
409
  );
317
410
  const session = claimResult.rows[0];
318
411
  if (!session) {
@@ -333,34 +426,46 @@ function createAquifer(config) {
333
426
  // 2. Extract user turns
334
427
  const turns = storage.extractUserTurns(normalized);
335
428
 
429
+ // Collected across pre-tx and tx phases; any non-empty warnings demote
430
+ // the final status from 'succeeded' to 'partial' (see step 8 below).
431
+ const warnings = [];
432
+
336
433
  // 3. Summarize (custom or built-in)
337
434
  let summaryResult = null;
338
435
  let entityRaw = null;
339
436
  let extra = null;
340
437
 
341
438
  if (!skipSummary && normalized.length > 0) {
342
- if (customSummaryFn) {
343
- // Custom pipeline: caller handles LLM call and parsing
344
- summaryResult = await customSummaryFn(normalized);
345
- if (summaryResult.entityRaw) entityRaw = summaryResult.entityRaw;
346
- if (summaryResult.extra) extra = summaryResult.extra;
347
- } else {
348
- // Built-in pipeline
349
- const doMergeEntities = entitiesEnabled && mergeCall && !skipEntities;
350
- summaryResult = await summarize(normalized, {
351
- llmFn,
352
- promptFn: summarizePromptFn,
353
- mergeEntities: doMergeEntities,
354
- });
355
- if (summaryResult.entityRaw) {
356
- entityRaw = summaryResult.entityRaw;
439
+ // Pre-transaction failures (customSummaryFn / summarize throws) would
440
+ // otherwise bubble out and leave the session stuck in 'processing'
441
+ // until stale reclaim. Capture as a warning so status ends 'partial',
442
+ // keeping parity with how embed/entity-extract failures are treated.
443
+ try {
444
+ if (customSummaryFn) {
445
+ // Custom pipeline: caller handles LLM call and parsing
446
+ summaryResult = await customSummaryFn(normalized);
447
+ if (summaryResult && summaryResult.entityRaw) entityRaw = summaryResult.entityRaw;
448
+ if (summaryResult && summaryResult.extra) extra = summaryResult.extra;
449
+ } else {
450
+ // Built-in pipeline
451
+ const doMergeEntities = entitiesEnabled && mergeCall && !skipEntities;
452
+ summaryResult = await summarize(normalized, {
453
+ llmFn,
454
+ promptFn: summarizePromptFn,
455
+ mergeEntities: doMergeEntities,
456
+ });
457
+ if (summaryResult.entityRaw) {
458
+ entityRaw = summaryResult.entityRaw;
459
+ }
357
460
  }
461
+ } catch (e) {
462
+ warnings.push(`summary step failed: ${e.message}`);
463
+ summaryResult = null;
358
464
  }
359
465
  }
360
466
 
361
467
  // 4. Pre-compute all LLM/embed results BEFORE opening transaction
362
468
  // (avoids holding pool connection during slow LLM/embed calls)
363
- const warnings = [];
364
469
  let summaryEmbedding = null;
365
470
  let turnVectors = null;
366
471
  let parsedEntities = [];
@@ -494,7 +599,11 @@ function createAquifer(config) {
494
599
  await client.query('ROLLBACK').catch(() => {});
495
600
  try {
496
601
  await storage.markStatus(pool, session.id, 'failed', err.message, { schema });
497
- } catch (_) { /* swallow */ }
602
+ } catch (markErr) {
603
+ // Secondary failure: session is stuck in 'processing' until stale reclaim.
604
+ // Surface so operators notice and don't silently rely on the timeout.
605
+ console.warn(`[aquifer] enrich failed for session ${sessionId} AND markStatus('failed') also failed: ${markErr.message}`);
606
+ }
498
607
  throw err;
499
608
  } finally {
500
609
  client.release();
@@ -692,7 +801,7 @@ function createAquifer(config) {
692
801
  entityScoreBySession.set(row.session_id, parseInt(row.entity_count) / maxCount);
693
802
  }
694
803
  }
695
- } catch (_) { /* entity search failure non-fatal */ }
804
+ } catch { /* entity search failure non-fatal */ }
696
805
  }
697
806
 
698
807
  // 3. Run search paths in parallel (conditioned on mode)
@@ -747,7 +856,7 @@ function createAquifer(config) {
747
856
  for (const r of [...filteredFts, ...filteredEmb, ...filteredTurn]) {
748
857
  const sid = r.session_id || String(r.id);
749
858
  const ss = typeof r.structured_summary === 'string'
750
- ? (() => { try { return JSON.parse(r.structured_summary); } catch (_) { return null; } })()
859
+ ? (() => { try { return JSON.parse(r.structured_summary); } catch { return null; } })()
751
860
  : r.structured_summary;
752
861
  if (ss && Array.isArray(ss.open_loops) && ss.open_loops.length > 0) {
753
862
  openLoopSet.add(sid);
@@ -760,7 +869,7 @@ function createAquifer(config) {
760
869
  const externalPromises = [];
761
870
  for (const [name, sourceConfig] of sources) {
762
871
  if (typeof sourceConfig.search === 'function') {
763
- const w = sourceConfig.weight !== null && sourceConfig.weight !== undefined ? sourceConfig.weight : 1.0;
872
+ const w = sourceConfig.weight !== undefined && sourceConfig.weight !== undefined ? sourceConfig.weight : 1.0;
764
873
  externalPromises.push(
765
874
  Promise.race([
766
875
  sourceConfig.search(query, opts),
@@ -831,7 +940,7 @@ function createAquifer(config) {
831
940
  if (sessionRowIds.length > 0) {
832
941
  try {
833
942
  await storage.recordAccess(pool, sessionRowIds, { schema });
834
- } catch (_) { /* access recording non-fatal */ }
943
+ } catch { /* access recording non-fatal */ }
835
944
  }
836
945
 
837
946
  // 8. Format results
@@ -842,7 +951,6 @@ function createAquifer(config) {
842
951
  startedAt: r.started_at,
843
952
  summaryText: r.summary_text || null,
844
953
  structuredSummary: r.structured_summary || null,
845
- summarySnippet: r.summary_snippet || null,
846
954
  matchedTurnText: r.matched_turn_text || null,
847
955
  matchedTurnIndex: r.matched_turn_index || null,
848
956
  score: r._rerankScore ?? r._score,
@@ -913,35 +1021,31 @@ function createAquifer(config) {
913
1021
  return { id: result.rows[0].id, sessionId, agentId, status: 'skipped' };
914
1022
  },
915
1023
 
916
- async getSessionFull(sessionId) {
917
- const result = await pool.query(
918
- `SELECT * FROM ${qi(schema)}.sessions
919
- WHERE session_id = $1 AND tenant_id = $2
920
- LIMIT 1`,
921
- [sessionId, tenantId]
922
- );
923
- const session = result.rows[0];
924
- if (!session) return null;
925
-
926
- const sumResult = await pool.query(
927
- `SELECT * FROM ${qi(schema)}.session_summaries
928
- WHERE session_row_id = $1
929
- LIMIT 1`,
930
- [session.id]
931
- );
932
-
933
- return {
934
- session,
935
- summary: sumResult.rows[0] || null,
936
- };
937
- },
938
-
939
1024
  // --- public config accessor ---
940
1025
 
941
1026
  getConfig() {
942
1027
  return { schema, tenantId };
943
1028
  },
944
1029
 
1030
+ // v1.2.0: expose the internal pool so host persona layers can reuse it
1031
+ // for host-owned tables (e.g. daily_entries). Read-only — callers should
1032
+ // not call pool.end() on it; use aquifer.close() for that.
1033
+ getPool() {
1034
+ return pool;
1035
+ },
1036
+
1037
+ // v1.2.0: expose resolved LLM function. May be null if no llm.fn was
1038
+ // supplied and AQUIFER_LLM_PROVIDER env is unset. Persona layers that
1039
+ // implement custom summaryFn can reuse this instead of wiring their own.
1040
+ getLlmFn() {
1041
+ return llmFn;
1042
+ },
1043
+
1044
+ // v1.2.0: expose resolved embed function (may be null same as LLM).
1045
+ getEmbedFn() {
1046
+ return embedFn;
1047
+ },
1048
+
945
1049
  // --- admin query helpers ---
946
1050
 
947
1051
  async getStats() {
@@ -974,7 +1078,7 @@ function createAquifer(config) {
974
1078
  [tenantId]
975
1079
  );
976
1080
  entityCount = entResult.rows[0]?.count || 0;
977
- } catch (_) { /* entities table may not exist */ }
1081
+ } catch { /* entities table may not exist */ }
978
1082
 
979
1083
  return {
980
1084
  sessions: Object.fromEntries(sessions.rows.map(r => [r.processing_status, r.count])),
@@ -1033,7 +1137,11 @@ function createAquifer(config) {
1033
1137
  const maxChars = opts.maxChars || 4000;
1034
1138
  const format = opts.format || 'structured';
1035
1139
 
1036
- const where = [`s.tenant_id = $1`, `s.processing_status = 'succeeded'`];
1140
+ // 'partial' sessions have a summary but recorded warnings during enrich;
1141
+ // they are user-visible content, not in-progress — bootstrap must include
1142
+ // them alongside 'succeeded'. 'pending' / 'processing' have no summary
1143
+ // yet and are correctly excluded.
1144
+ const where = [`s.tenant_id = $1`, `s.processing_status IN ('succeeded', 'partial')`];
1037
1145
  const params = [tenantId];
1038
1146
 
1039
1147
  if (agentId) {
@@ -1046,7 +1154,10 @@ function createAquifer(config) {
1046
1154
  }
1047
1155
 
1048
1156
  params.push(lookbackDays);
1049
- where.push(`s.started_at > now() - ($${params.length} || ' days')::interval`);
1157
+ // upsertSession sets ended_at on every commit but started_at / last_message_at
1158
+ // only when the caller supplies them — fall back through both so sessions
1159
+ // committed without explicit timestamps remain reachable.
1160
+ where.push(`COALESCE(s.last_message_at, s.ended_at, s.started_at) > now() - ($${params.length} || ' days')::interval`);
1050
1161
 
1051
1162
  params.push(limit);
1052
1163
 
@@ -1056,7 +1167,7 @@ function createAquifer(config) {
1056
1167
  FROM ${qi(schema)}.sessions s
1057
1168
  JOIN ${qi(schema)}.session_summaries ss ON ss.session_row_id = s.id
1058
1169
  WHERE ${where.join(' AND ')}
1059
- ORDER BY s.started_at DESC
1170
+ ORDER BY COALESCE(s.last_message_at, s.ended_at, s.started_at) DESC
1060
1171
  LIMIT $${params.length}`,
1061
1172
  params
1062
1173
  );
@@ -1135,8 +1246,6 @@ function formatBootstrapText(data, maxChars) {
1135
1246
  }
1136
1247
 
1137
1248
  let truncated = false;
1138
- const parts = [];
1139
-
1140
1249
  // Build session lines (newest first, truncate from oldest if over budget)
1141
1250
  const sessionLines = [];
1142
1251
  for (const s of data.sessions) {
package/core/entity.js CHANGED
@@ -236,7 +236,6 @@ async function upsertEntityRelations(pool, {
236
236
  if (validPairs.length === 0) return { upserted: 0 };
237
237
 
238
238
  // Batch insert: multi-row VALUES
239
- const COLS_PER_ROW = 3;
240
239
  const valueClauses = [];
241
240
  const params = [];
242
241
 
@@ -387,8 +386,7 @@ async function resolveEntities(pool, {
387
386
  if (!normQ || seen.has(normQ)) continue;
388
387
  seen.set(normQ, true);
389
388
 
390
- const escaped = _escapeIlike(normQ);
391
- const result = await pool.query(
389
+ const result = await pool.query(
392
390
  `SELECT id, name, normalized_name
393
391
  FROM ${qi(schema)}.entities
394
392
  WHERE status = 'active'
package/core/storage.js CHANGED
@@ -281,7 +281,10 @@ async function searchSessions(pool, query, {
281
281
  FROM ${qi(schema)}.sessions s
282
282
  LEFT JOIN ${qi(schema)}.session_summaries ss ON ss.session_row_id = s.id
283
283
  WHERE ${where.join(' AND ')}
284
- ORDER BY fts_rank DESC, s.last_message_at DESC NULLS LAST
284
+ ORDER BY
285
+ COALESCE(ss.search_text ILIKE '%' || $1 || '%', FALSE) DESC,
286
+ fts_rank DESC,
287
+ s.last_message_at DESC NULLS LAST
285
288
  LIMIT $${params.length}`,
286
289
  params
287
290
  );
@@ -361,7 +364,6 @@ async function upsertTurnEmbeddings(pool, sessionRowId, {
361
364
  }
362
365
 
363
366
  // Batch insert: build multi-row VALUES clause
364
- const COLS_PER_ROW = 10;
365
367
  const valueClauses = [];
366
368
  const params = [];
367
369
 
@@ -416,6 +418,16 @@ async function searchTurnEmbeddings(pool, {
416
418
  source,
417
419
  limit = 15,
418
420
  }) {
421
+ // HNSW index fires only on `ORDER BY embedding <=> $vec LIMIT N` without
422
+ // additional predicates in the same query level. So the CTE does a plain
423
+ // nearest-neighbor scan (uses idx_turn_emb_embedding_hnsw at scale), then
424
+ // the outer SELECT applies tenant/agent/date/source filters and dedups.
425
+ //
426
+ // Filter narrowness may leave fewer than `limit` rows after post-filter;
427
+ // NN_OVERFETCH trades extra vector work for filter survival headroom.
428
+ const NN_OVERFETCH = 10;
429
+ const nnLimit = Math.max(50, limit * NN_OVERFETCH);
430
+
419
431
  const where = ['s.tenant_id = $1'];
420
432
  const params = [tenantId];
421
433
 
@@ -434,40 +446,70 @@ async function searchTurnEmbeddings(pool, {
434
446
  }
435
447
  if (agentIds) {
436
448
  params.push(agentIds);
437
- where.push(`t.agent_id = ANY($${params.length})`);
449
+ where.push(`s.agent_id = ANY($${params.length})`);
438
450
  }
439
451
  if (source) {
440
452
  params.push(source);
441
- where.push(`t.source = $${params.length}`);
453
+ where.push(`s.source = $${params.length}`);
442
454
  }
443
455
 
444
456
  params.push(`[${queryVec.join(',')}]`);
445
457
  const vecPos = params.length;
446
-
447
- // m5: use subquery with LIMIT to avoid scanning all rows
448
- params.push(limit * 3); // fetch more than needed for DISTINCT ON dedup
449
- const innerLimitPos = params.length;
458
+ params.push(nnLimit);
459
+ const nnLimitPos = params.length;
450
460
 
451
461
  const result = await pool.query(
452
- `SELECT * FROM (
453
- SELECT DISTINCT ON (t.session_row_id)
462
+ `WITH nn AS (
463
+ SELECT t.session_row_id, t.content_text, t.turn_index,
464
+ (t.embedding <=> $${vecPos}::vector) AS turn_distance
465
+ FROM ${qi(schema)}.turn_embeddings t
466
+ ORDER BY t.embedding <=> $${vecPos}::vector ASC
467
+ LIMIT $${nnLimitPos}
468
+ )
469
+ SELECT * FROM (
470
+ SELECT DISTINCT ON (nn.session_row_id)
454
471
  s.session_id, s.id AS session_row_id, s.agent_id, s.source, s.started_at,
455
472
  ss.summary_text, ss.structured_summary, ss.access_count, ss.last_accessed_at,
456
473
  COALESCE(ss.trust_score, 0.5) AS trust_score,
457
- t.content_text AS matched_turn_text, t.turn_index AS matched_turn_index,
458
- (t.embedding <=> $${vecPos}::vector) AS turn_distance
459
- FROM ${qi(schema)}.turn_embeddings t
460
- JOIN ${qi(schema)}.sessions s ON s.id = t.session_row_id
474
+ nn.content_text AS matched_turn_text, nn.turn_index AS matched_turn_index,
475
+ nn.turn_distance
476
+ FROM nn
477
+ JOIN ${qi(schema)}.sessions s ON s.id = nn.session_row_id
461
478
  LEFT JOIN ${qi(schema)}.session_summaries ss ON ss.session_row_id = s.id
462
479
  WHERE ${where.join(' AND ')}
463
- ORDER BY t.session_row_id, turn_distance ASC
464
- ) sub
465
- ORDER BY turn_distance ASC
466
- LIMIT $${innerLimitPos}`,
480
+ ORDER BY nn.session_row_id, nn.turn_distance ASC
481
+ ) dedup
482
+ ORDER BY turn_distance ASC`,
467
483
  params
468
484
  );
469
485
 
470
- return { rows: result.rows.slice(0, limit) };
486
+ if (result.rows.length > 0) {
487
+ return { rows: result.rows.slice(0, limit) };
488
+ }
489
+
490
+ // Fallback: HNSW-first path filtered out to nothing. This can happen when
491
+ // tenant/agent filters are narrow enough to eliminate every NN candidate.
492
+ // Pay the cost of a filter-first scan to guarantee we don't silently return
493
+ // empty when qualifying rows exist. No HNSW on this path — slower, correct.
494
+ const fallbackParams = params.slice(0, params.length - 1); // drop nnLimit
495
+ fallbackParams.push(limit);
496
+ const fallbackLimitPos = fallbackParams.length;
497
+ const fallback = await pool.query(
498
+ `SELECT DISTINCT ON (t.session_row_id)
499
+ s.session_id, s.id AS session_row_id, s.agent_id, s.source, s.started_at,
500
+ ss.summary_text, ss.structured_summary, ss.access_count, ss.last_accessed_at,
501
+ COALESCE(ss.trust_score, 0.5) AS trust_score,
502
+ t.content_text AS matched_turn_text, t.turn_index AS matched_turn_index,
503
+ (t.embedding <=> $${vecPos}::vector) AS turn_distance
504
+ FROM ${qi(schema)}.turn_embeddings t
505
+ JOIN ${qi(schema)}.sessions s ON s.id = t.session_row_id
506
+ LEFT JOIN ${qi(schema)}.session_summaries ss ON ss.session_row_id = s.id
507
+ WHERE ${where.join(' AND ')}
508
+ ORDER BY t.session_row_id, t.embedding <=> $${vecPos}::vector ASC
509
+ LIMIT $${fallbackLimitPos}`,
510
+ fallbackParams
511
+ );
512
+ return { rows: fallback.rows };
471
513
  }
472
514
 
473
515
  // ---------------------------------------------------------------------------
@@ -504,16 +546,32 @@ async function recordFeedback(pool, {
504
546
  }
505
547
 
506
548
  const trustBefore = parseFloat(current.rows[0].trust_score);
507
- const trustAfter = verdict === 'helpful'
508
- ? Math.min(1.0, trustBefore + TRUST_UP)
509
- : Math.max(0.0, trustBefore - TRUST_DOWN);
510
549
 
511
- await client.query(
512
- `UPDATE ${qi(schema)}.session_summaries
513
- SET trust_score = $1, updated_at = now()
514
- WHERE session_row_id = $2`,
515
- [trustAfter, sessionRowId]
550
+ // Dedupe: the same (agent, verdict) applied more than once must not stack.
551
+ // Audit row is still inserted so the sequence of feedback events is
552
+ // preserved; only the trust_score delta is skipped.
553
+ const prior = await client.query(
554
+ `SELECT 1 FROM ${qi(schema)}.session_feedback
555
+ WHERE session_row_id = $1 AND agent_id = $2 AND verdict = $3
556
+ LIMIT 1`,
557
+ [sessionRowId, agentId, verdict]
516
558
  );
559
+ const isDup = prior.rows.length > 0;
560
+
561
+ const trustAfter = isDup
562
+ ? trustBefore
563
+ : (verdict === 'helpful'
564
+ ? Math.min(1.0, trustBefore + TRUST_UP)
565
+ : Math.max(0.0, trustBefore - TRUST_DOWN));
566
+
567
+ if (!isDup) {
568
+ await client.query(
569
+ `UPDATE ${qi(schema)}.session_summaries
570
+ SET trust_score = $1, updated_at = now()
571
+ WHERE session_row_id = $2`,
572
+ [trustAfter, sessionRowId]
573
+ );
574
+ }
517
575
 
518
576
  await client.query(
519
577
  `INSERT INTO ${qi(schema)}.session_feedback
@@ -523,7 +581,7 @@ async function recordFeedback(pool, {
523
581
  );
524
582
 
525
583
  await client.query('COMMIT');
526
- return { trustBefore, trustAfter, verdict };
584
+ return { trustBefore, trustAfter, verdict, duplicate: isDup };
527
585
  } catch (err) {
528
586
  await client.query('ROLLBACK').catch(() => {});
529
587
  throw err;