@shadowforge0/aquifer-memory 1.0.3 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -20
- package/consumers/claude-code.js +117 -0
- package/consumers/cli.js +17 -0
- package/consumers/default/daily-entries.js +196 -0
- package/consumers/default/index.js +282 -0
- package/consumers/default/prompts/summary.js +153 -0
- package/consumers/mcp.js +3 -23
- package/consumers/miranda/context-inject.js +119 -0
- package/consumers/miranda/daily-entries.js +224 -0
- package/consumers/miranda/index.js +353 -0
- package/consumers/miranda/instance.js +55 -0
- package/consumers/miranda/llm.js +99 -0
- package/consumers/miranda/prompts/summary.js +303 -0
- package/consumers/miranda/recall-format.js +74 -0
- package/consumers/miranda/workspace-files.js +91 -0
- package/consumers/openclaw-ext/index.js +38 -0
- package/consumers/openclaw-ext/openclaw.plugin.json +9 -0
- package/consumers/openclaw-ext/package.json +10 -0
- package/consumers/openclaw-plugin.js +66 -74
- package/consumers/opencode.js +21 -24
- package/consumers/shared/autodetect.js +64 -0
- package/consumers/shared/entity-parser.js +119 -0
- package/consumers/shared/ingest.js +148 -0
- package/consumers/shared/llm-autodetect.js +137 -0
- package/consumers/shared/normalize.js +129 -0
- package/consumers/shared/recall-format.js +110 -0
- package/core/aquifer.js +180 -71
- package/core/entity.js +1 -3
- package/core/storage.js +86 -28
- package/docs/postprocess-contract.md +132 -0
- package/index.js +9 -1
- package/package.json +23 -2
- package/pipeline/_http.js +1 -1
- package/pipeline/consolidation/apply.js +176 -0
- package/pipeline/consolidation/index.js +21 -0
- package/pipeline/extract-entities.js +2 -2
- package/pipeline/rerank.js +1 -1
- package/pipeline/summarize.js +4 -1
- package/schema/001-base.sql +61 -24
- package/schema/002-entities.sql +17 -3
- package/schema/004-facts.sql +67 -0
- package/scripts/diagnose-fts-zh.js +168 -134
- package/scripts/diagnose-vector.js +188 -0
- package/scripts/install-openclaw.sh +59 -0
- package/scripts/smoke.mjs +2 -2
package/core/aquifer.js
CHANGED
|
@@ -9,6 +9,7 @@ const entity = require('./entity');
|
|
|
9
9
|
const { hybridRank } = require('./hybrid-rank');
|
|
10
10
|
const { summarize } = require('../pipeline/summarize');
|
|
11
11
|
const { extractEntities } = require('../pipeline/extract-entities');
|
|
12
|
+
const { createEmbedder } = require('../pipeline/embed');
|
|
12
13
|
|
|
13
14
|
// ---------------------------------------------------------------------------
|
|
14
15
|
// Schema name validation
|
|
@@ -54,48 +55,104 @@ function buildRerankDocument(row, maxChars) {
|
|
|
54
55
|
return text;
|
|
55
56
|
}
|
|
56
57
|
|
|
58
|
+
// ---------------------------------------------------------------------------
|
|
59
|
+
// resolveEmbedFn — v1.2.0 embed autodetect (explicit > object > env > null)
|
|
60
|
+
// ---------------------------------------------------------------------------
|
|
61
|
+
|
|
62
|
+
function resolveEmbedFn(embedConfig, env) {
|
|
63
|
+
if (embedConfig && typeof embedConfig.fn === 'function') {
|
|
64
|
+
return embedConfig.fn;
|
|
65
|
+
}
|
|
66
|
+
if (embedConfig && embedConfig.provider) {
|
|
67
|
+
const embedder = createEmbedder(embedConfig);
|
|
68
|
+
return (texts) => embedder.embedBatch(texts);
|
|
69
|
+
}
|
|
70
|
+
const provider = env.EMBED_PROVIDER;
|
|
71
|
+
if (!provider) return null;
|
|
72
|
+
|
|
73
|
+
const opts = { provider };
|
|
74
|
+
if (provider === 'ollama') {
|
|
75
|
+
opts.ollamaUrl = env.OLLAMA_URL || env.AQUIFER_EMBED_BASE_URL || 'http://localhost:11434';
|
|
76
|
+
opts.model = env.AQUIFER_EMBED_MODEL || 'bge-m3';
|
|
77
|
+
} else if (provider === 'openai') {
|
|
78
|
+
opts.openaiApiKey = env.OPENAI_API_KEY;
|
|
79
|
+
if (!opts.openaiApiKey) {
|
|
80
|
+
throw new Error('EMBED_PROVIDER=openai requires OPENAI_API_KEY');
|
|
81
|
+
}
|
|
82
|
+
opts.openaiModel = env.AQUIFER_EMBED_MODEL || 'text-embedding-3-small';
|
|
83
|
+
if (env.AQUIFER_EMBED_DIM) opts.openaiDimensions = Number(env.AQUIFER_EMBED_DIM);
|
|
84
|
+
} else {
|
|
85
|
+
throw new Error(`EMBED_PROVIDER=${provider} not supported by autodetect (use 'ollama' or 'openai', or pass config.embed.fn explicitly)`);
|
|
86
|
+
}
|
|
87
|
+
const embedder = createEmbedder(opts);
|
|
88
|
+
return (texts) => embedder.embedBatch(texts);
|
|
89
|
+
}
|
|
90
|
+
|
|
57
91
|
// ---------------------------------------------------------------------------
|
|
58
92
|
// createAquifer
|
|
59
93
|
// ---------------------------------------------------------------------------
|
|
60
94
|
|
|
61
|
-
function createAquifer(config) {
|
|
62
|
-
|
|
63
|
-
|
|
95
|
+
function createAquifer(config = {}) {
|
|
96
|
+
// v1.2.0: db falls back to DATABASE_URL / AQUIFER_DB_URL env so hosts can
|
|
97
|
+
// call createAquifer() with zero args for install-and-go.
|
|
98
|
+
const dbInput = config.db !== undefined
|
|
99
|
+
? config.db
|
|
100
|
+
: (process.env.DATABASE_URL || process.env.AQUIFER_DB_URL || null);
|
|
101
|
+
|
|
102
|
+
if (!dbInput) {
|
|
103
|
+
throw new Error(
|
|
104
|
+
'Aquifer requires a database: pass config.db (pg.Pool or connection string), '
|
|
105
|
+
+ 'or set DATABASE_URL / AQUIFER_DB_URL in the environment.'
|
|
106
|
+
);
|
|
64
107
|
}
|
|
65
108
|
|
|
66
|
-
const schema = config.schema || 'aquifer';
|
|
109
|
+
const schema = config.schema || process.env.AQUIFER_SCHEMA || 'aquifer';
|
|
67
110
|
validateSchema(schema);
|
|
68
111
|
|
|
69
112
|
if (config.tenantId === '') throw new Error('config.tenantId must not be empty');
|
|
70
|
-
const tenantId = config.tenantId || 'default';
|
|
113
|
+
const tenantId = config.tenantId || process.env.AQUIFER_TENANT_ID || 'default';
|
|
71
114
|
|
|
72
115
|
// Pool management
|
|
73
116
|
let pool;
|
|
74
117
|
let ownsPool = false;
|
|
75
|
-
if (typeof
|
|
76
|
-
pool = new Pool({ connectionString:
|
|
118
|
+
if (typeof dbInput === 'string') {
|
|
119
|
+
pool = new Pool({ connectionString: dbInput });
|
|
77
120
|
ownsPool = true;
|
|
78
121
|
} else {
|
|
79
|
-
pool =
|
|
122
|
+
pool = dbInput;
|
|
80
123
|
ownsPool = !!config.ownsPool; // allow factory to claim ownership
|
|
81
124
|
}
|
|
82
125
|
|
|
83
126
|
// Embed config (lazy — only required for recall/enrich)
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
127
|
+
// v1.2.0 fallback chain:
|
|
128
|
+
// 1. config.embed.fn (explicit function)
|
|
129
|
+
// 2. config.embed.provider (build via createEmbedder)
|
|
130
|
+
// 3. EMBED_PROVIDER env + provider-specific key (zero-arg install-and-go)
|
|
131
|
+
// 4. null — defer to requireEmbed() at call time
|
|
132
|
+
const embedFn = resolveEmbedFn(config.embed, process.env);
|
|
87
133
|
function requireEmbed(op) {
|
|
88
|
-
if (!embedFn) throw new Error(`Aquifer.${op}() requires config.embed.fn (async (texts) => number[][])`);
|
|
134
|
+
if (!embedFn) throw new Error(`Aquifer.${op}() requires config.embed.fn or EMBED_PROVIDER env (async (texts) => number[][])`);
|
|
89
135
|
}
|
|
90
136
|
|
|
91
137
|
// LLM config (optional — only needed for enrich with built-in summarize)
|
|
92
|
-
|
|
138
|
+
// v1.2.0: falls back to AQUIFER_LLM_PROVIDER env + provider-specific key.
|
|
139
|
+
const { resolveLlmFn } = require('../consumers/shared/llm-autodetect');
|
|
140
|
+
const llmFn = resolveLlmFn(config.llm, process.env);
|
|
93
141
|
|
|
94
142
|
// Summarize config
|
|
95
143
|
const summarizePromptFn = config.summarize && config.summarize.prompt ? config.summarize.prompt : null;
|
|
96
144
|
|
|
145
|
+
// Enrich stale-claim window: a 'processing' session older than this is
|
|
146
|
+
// reclaimable by a concurrent enrich() caller (covers crashed workers).
|
|
147
|
+
const staleEnrichMinutes = Number.isFinite(config.staleEnrichMinutes)
|
|
148
|
+
? Math.max(1, Math.floor(config.staleEnrichMinutes))
|
|
149
|
+
: 10;
|
|
150
|
+
|
|
97
151
|
// Entity config
|
|
98
152
|
let entitiesEnabled = config.entities && config.entities.enabled === true;
|
|
153
|
+
|
|
154
|
+
// Facts config (opt-in consolidation lifecycle)
|
|
155
|
+
let factsEnabled = config.facts && config.facts.enabled === true;
|
|
99
156
|
const mergeCall = config.entities && config.entities.mergeCall !== undefined ? config.entities.mergeCall : true;
|
|
100
157
|
const entityPromptFn = config.entities && config.entities.prompt ? config.entities.prompt : null;
|
|
101
158
|
const entityScope = (config.entities && config.entities.scope) || 'default';
|
|
@@ -207,9 +264,17 @@ function createAquifer(config) {
|
|
|
207
264
|
const trustSql = loadSql('003-trust-feedback.sql', schema);
|
|
208
265
|
await pool.query(trustSql);
|
|
209
266
|
|
|
267
|
+
// 4. Facts / consolidation (opt-in)
|
|
268
|
+
if (factsEnabled) {
|
|
269
|
+
const factsSql = loadSql('004-facts.sql', schema);
|
|
270
|
+
await pool.query(factsSql);
|
|
271
|
+
}
|
|
272
|
+
|
|
210
273
|
migrated = true;
|
|
211
274
|
} finally {
|
|
212
|
-
await pool.query('SELECT pg_advisory_unlock($1)', [lockKey]).catch(() => {
|
|
275
|
+
await pool.query('SELECT pg_advisory_unlock($1)', [lockKey]).catch((err) => {
|
|
276
|
+
console.warn(`[aquifer] failed to release migration advisory lock for schema "${schema}": ${err.message}`);
|
|
277
|
+
});
|
|
213
278
|
}
|
|
214
279
|
},
|
|
215
280
|
|
|
@@ -225,7 +290,7 @@ function createAquifer(config) {
|
|
|
225
290
|
sources.set(name, {
|
|
226
291
|
type: opts.type || 'custom',
|
|
227
292
|
search: opts.search || null,
|
|
228
|
-
weight: opts.weight !==
|
|
293
|
+
weight: opts.weight !== undefined && opts.weight !== undefined ? opts.weight : 1.0,
|
|
229
294
|
});
|
|
230
295
|
},
|
|
231
296
|
|
|
@@ -238,6 +303,32 @@ function createAquifer(config) {
|
|
|
238
303
|
}
|
|
239
304
|
},
|
|
240
305
|
|
|
306
|
+
async enableFacts() {
|
|
307
|
+
factsEnabled = true;
|
|
308
|
+
// Run the facts DDL (idempotent — all CREATE/ALTER use IF NOT EXISTS).
|
|
309
|
+
// Safe to call repeatedly; also safe to call before migrate() (will no-op
|
|
310
|
+
// until base schema exists, which enrich/commit will materialize).
|
|
311
|
+
await ensureMigrated();
|
|
312
|
+
const factsSql = loadSql('004-facts.sql', schema);
|
|
313
|
+
await pool.query(factsSql);
|
|
314
|
+
},
|
|
315
|
+
|
|
316
|
+
async consolidate(sessionId, opts = {}) {
|
|
317
|
+
if (!factsEnabled) throw new Error('aquifer.consolidate() requires enableFacts() first');
|
|
318
|
+
await ensureMigrated();
|
|
319
|
+
const { applyConsolidation } = require('../pipeline/consolidation');
|
|
320
|
+
const agentId = opts.agentId || 'agent';
|
|
321
|
+
return applyConsolidation(pool, {
|
|
322
|
+
actions: opts.actions || [],
|
|
323
|
+
agentId,
|
|
324
|
+
sessionId,
|
|
325
|
+
schema,
|
|
326
|
+
tenantId,
|
|
327
|
+
normalizeSubject: opts.normalizeSubject || null,
|
|
328
|
+
recapOverview: opts.recapOverview || '',
|
|
329
|
+
});
|
|
330
|
+
},
|
|
331
|
+
|
|
241
332
|
// --- write path ---
|
|
242
333
|
|
|
243
334
|
async commit(sessionId, messages, opts = {}) {
|
|
@@ -302,17 +393,19 @@ function createAquifer(config) {
|
|
|
302
393
|
const postProcess = opts.postProcess || null; // async (ctx) => void
|
|
303
394
|
const optModel = 'model' in opts ? opts.model : undefined; // undefined = no override
|
|
304
395
|
|
|
305
|
-
// 1. Optimistic lock: claim session for processing
|
|
306
|
-
// Also reclaim stale 'processing' sessions (
|
|
307
|
-
|
|
396
|
+
// 1. Optimistic lock: claim session for processing.
|
|
397
|
+
// Also reclaim stale 'processing' sessions (likely killed worker).
|
|
398
|
+
// Stale window is config.staleEnrichMinutes (default 10).
|
|
308
399
|
const claimResult = await pool.query(
|
|
309
400
|
`UPDATE ${qi(schema)}.sessions
|
|
310
401
|
SET processing_status = 'processing', processing_started_at = NOW()
|
|
311
402
|
WHERE session_id = $1 AND agent_id = $2 AND tenant_id = $3
|
|
312
403
|
AND (processing_status IN ('pending', 'failed')
|
|
313
|
-
OR (processing_status = 'processing'
|
|
404
|
+
OR (processing_status = 'processing'
|
|
405
|
+
AND (processing_started_at IS NULL
|
|
406
|
+
OR processing_started_at < NOW() - make_interval(mins => $4))))
|
|
314
407
|
RETURNING *`,
|
|
315
|
-
[sessionId, agentId, tenantId]
|
|
408
|
+
[sessionId, agentId, tenantId, staleEnrichMinutes]
|
|
316
409
|
);
|
|
317
410
|
const session = claimResult.rows[0];
|
|
318
411
|
if (!session) {
|
|
@@ -333,34 +426,46 @@ function createAquifer(config) {
|
|
|
333
426
|
// 2. Extract user turns
|
|
334
427
|
const turns = storage.extractUserTurns(normalized);
|
|
335
428
|
|
|
429
|
+
// Collected across pre-tx and tx phases; any non-empty warnings demote
|
|
430
|
+
// the final status from 'succeeded' to 'partial' (see step 8 below).
|
|
431
|
+
const warnings = [];
|
|
432
|
+
|
|
336
433
|
// 3. Summarize (custom or built-in)
|
|
337
434
|
let summaryResult = null;
|
|
338
435
|
let entityRaw = null;
|
|
339
436
|
let extra = null;
|
|
340
437
|
|
|
341
438
|
if (!skipSummary && normalized.length > 0) {
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
439
|
+
// Pre-transaction failures (customSummaryFn / summarize throws) would
|
|
440
|
+
// otherwise bubble out and leave the session stuck in 'processing'
|
|
441
|
+
// until stale reclaim. Capture as a warning so status ends 'partial',
|
|
442
|
+
// keeping parity with how embed/entity-extract failures are treated.
|
|
443
|
+
try {
|
|
444
|
+
if (customSummaryFn) {
|
|
445
|
+
// Custom pipeline: caller handles LLM call and parsing
|
|
446
|
+
summaryResult = await customSummaryFn(normalized);
|
|
447
|
+
if (summaryResult && summaryResult.entityRaw) entityRaw = summaryResult.entityRaw;
|
|
448
|
+
if (summaryResult && summaryResult.extra) extra = summaryResult.extra;
|
|
449
|
+
} else {
|
|
450
|
+
// Built-in pipeline
|
|
451
|
+
const doMergeEntities = entitiesEnabled && mergeCall && !skipEntities;
|
|
452
|
+
summaryResult = await summarize(normalized, {
|
|
453
|
+
llmFn,
|
|
454
|
+
promptFn: summarizePromptFn,
|
|
455
|
+
mergeEntities: doMergeEntities,
|
|
456
|
+
});
|
|
457
|
+
if (summaryResult.entityRaw) {
|
|
458
|
+
entityRaw = summaryResult.entityRaw;
|
|
459
|
+
}
|
|
357
460
|
}
|
|
461
|
+
} catch (e) {
|
|
462
|
+
warnings.push(`summary step failed: ${e.message}`);
|
|
463
|
+
summaryResult = null;
|
|
358
464
|
}
|
|
359
465
|
}
|
|
360
466
|
|
|
361
467
|
// 4. Pre-compute all LLM/embed results BEFORE opening transaction
|
|
362
468
|
// (avoids holding pool connection during slow LLM/embed calls)
|
|
363
|
-
const warnings = [];
|
|
364
469
|
let summaryEmbedding = null;
|
|
365
470
|
let turnVectors = null;
|
|
366
471
|
let parsedEntities = [];
|
|
@@ -494,7 +599,11 @@ function createAquifer(config) {
|
|
|
494
599
|
await client.query('ROLLBACK').catch(() => {});
|
|
495
600
|
try {
|
|
496
601
|
await storage.markStatus(pool, session.id, 'failed', err.message, { schema });
|
|
497
|
-
} catch (
|
|
602
|
+
} catch (markErr) {
|
|
603
|
+
// Secondary failure: session is stuck in 'processing' until stale reclaim.
|
|
604
|
+
// Surface so operators notice and don't silently rely on the timeout.
|
|
605
|
+
console.warn(`[aquifer] enrich failed for session ${sessionId} AND markStatus('failed') also failed: ${markErr.message}`);
|
|
606
|
+
}
|
|
498
607
|
throw err;
|
|
499
608
|
} finally {
|
|
500
609
|
client.release();
|
|
@@ -692,7 +801,7 @@ function createAquifer(config) {
|
|
|
692
801
|
entityScoreBySession.set(row.session_id, parseInt(row.entity_count) / maxCount);
|
|
693
802
|
}
|
|
694
803
|
}
|
|
695
|
-
} catch
|
|
804
|
+
} catch { /* entity search failure non-fatal */ }
|
|
696
805
|
}
|
|
697
806
|
|
|
698
807
|
// 3. Run search paths in parallel (conditioned on mode)
|
|
@@ -747,7 +856,7 @@ function createAquifer(config) {
|
|
|
747
856
|
for (const r of [...filteredFts, ...filteredEmb, ...filteredTurn]) {
|
|
748
857
|
const sid = r.session_id || String(r.id);
|
|
749
858
|
const ss = typeof r.structured_summary === 'string'
|
|
750
|
-
? (() => { try { return JSON.parse(r.structured_summary); } catch
|
|
859
|
+
? (() => { try { return JSON.parse(r.structured_summary); } catch { return null; } })()
|
|
751
860
|
: r.structured_summary;
|
|
752
861
|
if (ss && Array.isArray(ss.open_loops) && ss.open_loops.length > 0) {
|
|
753
862
|
openLoopSet.add(sid);
|
|
@@ -760,7 +869,7 @@ function createAquifer(config) {
|
|
|
760
869
|
const externalPromises = [];
|
|
761
870
|
for (const [name, sourceConfig] of sources) {
|
|
762
871
|
if (typeof sourceConfig.search === 'function') {
|
|
763
|
-
const w = sourceConfig.weight !==
|
|
872
|
+
const w = sourceConfig.weight !== undefined && sourceConfig.weight !== undefined ? sourceConfig.weight : 1.0;
|
|
764
873
|
externalPromises.push(
|
|
765
874
|
Promise.race([
|
|
766
875
|
sourceConfig.search(query, opts),
|
|
@@ -831,7 +940,7 @@ function createAquifer(config) {
|
|
|
831
940
|
if (sessionRowIds.length > 0) {
|
|
832
941
|
try {
|
|
833
942
|
await storage.recordAccess(pool, sessionRowIds, { schema });
|
|
834
|
-
} catch
|
|
943
|
+
} catch { /* access recording non-fatal */ }
|
|
835
944
|
}
|
|
836
945
|
|
|
837
946
|
// 8. Format results
|
|
@@ -842,7 +951,6 @@ function createAquifer(config) {
|
|
|
842
951
|
startedAt: r.started_at,
|
|
843
952
|
summaryText: r.summary_text || null,
|
|
844
953
|
structuredSummary: r.structured_summary || null,
|
|
845
|
-
summarySnippet: r.summary_snippet || null,
|
|
846
954
|
matchedTurnText: r.matched_turn_text || null,
|
|
847
955
|
matchedTurnIndex: r.matched_turn_index || null,
|
|
848
956
|
score: r._rerankScore ?? r._score,
|
|
@@ -913,35 +1021,31 @@ function createAquifer(config) {
|
|
|
913
1021
|
return { id: result.rows[0].id, sessionId, agentId, status: 'skipped' };
|
|
914
1022
|
},
|
|
915
1023
|
|
|
916
|
-
async getSessionFull(sessionId) {
|
|
917
|
-
const result = await pool.query(
|
|
918
|
-
`SELECT * FROM ${qi(schema)}.sessions
|
|
919
|
-
WHERE session_id = $1 AND tenant_id = $2
|
|
920
|
-
LIMIT 1`,
|
|
921
|
-
[sessionId, tenantId]
|
|
922
|
-
);
|
|
923
|
-
const session = result.rows[0];
|
|
924
|
-
if (!session) return null;
|
|
925
|
-
|
|
926
|
-
const sumResult = await pool.query(
|
|
927
|
-
`SELECT * FROM ${qi(schema)}.session_summaries
|
|
928
|
-
WHERE session_row_id = $1
|
|
929
|
-
LIMIT 1`,
|
|
930
|
-
[session.id]
|
|
931
|
-
);
|
|
932
|
-
|
|
933
|
-
return {
|
|
934
|
-
session,
|
|
935
|
-
summary: sumResult.rows[0] || null,
|
|
936
|
-
};
|
|
937
|
-
},
|
|
938
|
-
|
|
939
1024
|
// --- public config accessor ---
|
|
940
1025
|
|
|
941
1026
|
getConfig() {
|
|
942
1027
|
return { schema, tenantId };
|
|
943
1028
|
},
|
|
944
1029
|
|
|
1030
|
+
// v1.2.0: expose the internal pool so host persona layers can reuse it
|
|
1031
|
+
// for host-owned tables (e.g. daily_entries). Read-only — callers should
|
|
1032
|
+
// not call pool.end() on it; use aquifer.close() for that.
|
|
1033
|
+
getPool() {
|
|
1034
|
+
return pool;
|
|
1035
|
+
},
|
|
1036
|
+
|
|
1037
|
+
// v1.2.0: expose resolved LLM function. May be null if no llm.fn was
|
|
1038
|
+
// supplied and AQUIFER_LLM_PROVIDER env is unset. Persona layers that
|
|
1039
|
+
// implement custom summaryFn can reuse this instead of wiring their own.
|
|
1040
|
+
getLlmFn() {
|
|
1041
|
+
return llmFn;
|
|
1042
|
+
},
|
|
1043
|
+
|
|
1044
|
+
// v1.2.0: expose resolved embed function (may be null same as LLM).
|
|
1045
|
+
getEmbedFn() {
|
|
1046
|
+
return embedFn;
|
|
1047
|
+
},
|
|
1048
|
+
|
|
945
1049
|
// --- admin query helpers ---
|
|
946
1050
|
|
|
947
1051
|
async getStats() {
|
|
@@ -974,7 +1078,7 @@ function createAquifer(config) {
|
|
|
974
1078
|
[tenantId]
|
|
975
1079
|
);
|
|
976
1080
|
entityCount = entResult.rows[0]?.count || 0;
|
|
977
|
-
} catch
|
|
1081
|
+
} catch { /* entities table may not exist */ }
|
|
978
1082
|
|
|
979
1083
|
return {
|
|
980
1084
|
sessions: Object.fromEntries(sessions.rows.map(r => [r.processing_status, r.count])),
|
|
@@ -1033,7 +1137,11 @@ function createAquifer(config) {
|
|
|
1033
1137
|
const maxChars = opts.maxChars || 4000;
|
|
1034
1138
|
const format = opts.format || 'structured';
|
|
1035
1139
|
|
|
1036
|
-
|
|
1140
|
+
// 'partial' sessions have a summary but recorded warnings during enrich;
|
|
1141
|
+
// they are user-visible content, not in-progress — bootstrap must include
|
|
1142
|
+
// them alongside 'succeeded'. 'pending' / 'processing' have no summary
|
|
1143
|
+
// yet and are correctly excluded.
|
|
1144
|
+
const where = [`s.tenant_id = $1`, `s.processing_status IN ('succeeded', 'partial')`];
|
|
1037
1145
|
const params = [tenantId];
|
|
1038
1146
|
|
|
1039
1147
|
if (agentId) {
|
|
@@ -1046,7 +1154,10 @@ function createAquifer(config) {
|
|
|
1046
1154
|
}
|
|
1047
1155
|
|
|
1048
1156
|
params.push(lookbackDays);
|
|
1049
|
-
|
|
1157
|
+
// upsertSession sets ended_at on every commit but started_at / last_message_at
|
|
1158
|
+
// only when the caller supplies them — fall back through both so sessions
|
|
1159
|
+
// committed without explicit timestamps remain reachable.
|
|
1160
|
+
where.push(`COALESCE(s.last_message_at, s.ended_at, s.started_at) > now() - ($${params.length} || ' days')::interval`);
|
|
1050
1161
|
|
|
1051
1162
|
params.push(limit);
|
|
1052
1163
|
|
|
@@ -1056,7 +1167,7 @@ function createAquifer(config) {
|
|
|
1056
1167
|
FROM ${qi(schema)}.sessions s
|
|
1057
1168
|
JOIN ${qi(schema)}.session_summaries ss ON ss.session_row_id = s.id
|
|
1058
1169
|
WHERE ${where.join(' AND ')}
|
|
1059
|
-
ORDER BY s.started_at DESC
|
|
1170
|
+
ORDER BY COALESCE(s.last_message_at, s.ended_at, s.started_at) DESC
|
|
1060
1171
|
LIMIT $${params.length}`,
|
|
1061
1172
|
params
|
|
1062
1173
|
);
|
|
@@ -1135,8 +1246,6 @@ function formatBootstrapText(data, maxChars) {
|
|
|
1135
1246
|
}
|
|
1136
1247
|
|
|
1137
1248
|
let truncated = false;
|
|
1138
|
-
const parts = [];
|
|
1139
|
-
|
|
1140
1249
|
// Build session lines (newest first, truncate from oldest if over budget)
|
|
1141
1250
|
const sessionLines = [];
|
|
1142
1251
|
for (const s of data.sessions) {
|
package/core/entity.js
CHANGED
|
@@ -236,7 +236,6 @@ async function upsertEntityRelations(pool, {
|
|
|
236
236
|
if (validPairs.length === 0) return { upserted: 0 };
|
|
237
237
|
|
|
238
238
|
// Batch insert: multi-row VALUES
|
|
239
|
-
const COLS_PER_ROW = 3;
|
|
240
239
|
const valueClauses = [];
|
|
241
240
|
const params = [];
|
|
242
241
|
|
|
@@ -387,8 +386,7 @@ async function resolveEntities(pool, {
|
|
|
387
386
|
if (!normQ || seen.has(normQ)) continue;
|
|
388
387
|
seen.set(normQ, true);
|
|
389
388
|
|
|
390
|
-
|
|
391
|
-
const result = await pool.query(
|
|
389
|
+
const result = await pool.query(
|
|
392
390
|
`SELECT id, name, normalized_name
|
|
393
391
|
FROM ${qi(schema)}.entities
|
|
394
392
|
WHERE status = 'active'
|
package/core/storage.js
CHANGED
|
@@ -281,7 +281,10 @@ async function searchSessions(pool, query, {
|
|
|
281
281
|
FROM ${qi(schema)}.sessions s
|
|
282
282
|
LEFT JOIN ${qi(schema)}.session_summaries ss ON ss.session_row_id = s.id
|
|
283
283
|
WHERE ${where.join(' AND ')}
|
|
284
|
-
ORDER BY
|
|
284
|
+
ORDER BY
|
|
285
|
+
COALESCE(ss.search_text ILIKE '%' || $1 || '%', FALSE) DESC,
|
|
286
|
+
fts_rank DESC,
|
|
287
|
+
s.last_message_at DESC NULLS LAST
|
|
285
288
|
LIMIT $${params.length}`,
|
|
286
289
|
params
|
|
287
290
|
);
|
|
@@ -361,7 +364,6 @@ async function upsertTurnEmbeddings(pool, sessionRowId, {
|
|
|
361
364
|
}
|
|
362
365
|
|
|
363
366
|
// Batch insert: build multi-row VALUES clause
|
|
364
|
-
const COLS_PER_ROW = 10;
|
|
365
367
|
const valueClauses = [];
|
|
366
368
|
const params = [];
|
|
367
369
|
|
|
@@ -416,6 +418,16 @@ async function searchTurnEmbeddings(pool, {
|
|
|
416
418
|
source,
|
|
417
419
|
limit = 15,
|
|
418
420
|
}) {
|
|
421
|
+
// HNSW index fires only on `ORDER BY embedding <=> $vec LIMIT N` without
|
|
422
|
+
// additional predicates in the same query level. So the CTE does a plain
|
|
423
|
+
// nearest-neighbor scan (uses idx_turn_emb_embedding_hnsw at scale), then
|
|
424
|
+
// the outer SELECT applies tenant/agent/date/source filters and dedups.
|
|
425
|
+
//
|
|
426
|
+
// Filter narrowness may leave fewer than `limit` rows after post-filter;
|
|
427
|
+
// NN_OVERFETCH trades extra vector work for filter survival headroom.
|
|
428
|
+
const NN_OVERFETCH = 10;
|
|
429
|
+
const nnLimit = Math.max(50, limit * NN_OVERFETCH);
|
|
430
|
+
|
|
419
431
|
const where = ['s.tenant_id = $1'];
|
|
420
432
|
const params = [tenantId];
|
|
421
433
|
|
|
@@ -434,40 +446,70 @@ async function searchTurnEmbeddings(pool, {
|
|
|
434
446
|
}
|
|
435
447
|
if (agentIds) {
|
|
436
448
|
params.push(agentIds);
|
|
437
|
-
where.push(`
|
|
449
|
+
where.push(`s.agent_id = ANY($${params.length})`);
|
|
438
450
|
}
|
|
439
451
|
if (source) {
|
|
440
452
|
params.push(source);
|
|
441
|
-
where.push(`
|
|
453
|
+
where.push(`s.source = $${params.length}`);
|
|
442
454
|
}
|
|
443
455
|
|
|
444
456
|
params.push(`[${queryVec.join(',')}]`);
|
|
445
457
|
const vecPos = params.length;
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
params.push(limit * 3); // fetch more than needed for DISTINCT ON dedup
|
|
449
|
-
const innerLimitPos = params.length;
|
|
458
|
+
params.push(nnLimit);
|
|
459
|
+
const nnLimitPos = params.length;
|
|
450
460
|
|
|
451
461
|
const result = await pool.query(
|
|
452
|
-
`
|
|
453
|
-
SELECT
|
|
462
|
+
`WITH nn AS (
|
|
463
|
+
SELECT t.session_row_id, t.content_text, t.turn_index,
|
|
464
|
+
(t.embedding <=> $${vecPos}::vector) AS turn_distance
|
|
465
|
+
FROM ${qi(schema)}.turn_embeddings t
|
|
466
|
+
ORDER BY t.embedding <=> $${vecPos}::vector ASC
|
|
467
|
+
LIMIT $${nnLimitPos}
|
|
468
|
+
)
|
|
469
|
+
SELECT * FROM (
|
|
470
|
+
SELECT DISTINCT ON (nn.session_row_id)
|
|
454
471
|
s.session_id, s.id AS session_row_id, s.agent_id, s.source, s.started_at,
|
|
455
472
|
ss.summary_text, ss.structured_summary, ss.access_count, ss.last_accessed_at,
|
|
456
473
|
COALESCE(ss.trust_score, 0.5) AS trust_score,
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
FROM
|
|
460
|
-
JOIN ${qi(schema)}.sessions s ON s.id =
|
|
474
|
+
nn.content_text AS matched_turn_text, nn.turn_index AS matched_turn_index,
|
|
475
|
+
nn.turn_distance
|
|
476
|
+
FROM nn
|
|
477
|
+
JOIN ${qi(schema)}.sessions s ON s.id = nn.session_row_id
|
|
461
478
|
LEFT JOIN ${qi(schema)}.session_summaries ss ON ss.session_row_id = s.id
|
|
462
479
|
WHERE ${where.join(' AND ')}
|
|
463
|
-
ORDER BY
|
|
464
|
-
)
|
|
465
|
-
ORDER BY turn_distance ASC
|
|
466
|
-
LIMIT $${innerLimitPos}`,
|
|
480
|
+
ORDER BY nn.session_row_id, nn.turn_distance ASC
|
|
481
|
+
) dedup
|
|
482
|
+
ORDER BY turn_distance ASC`,
|
|
467
483
|
params
|
|
468
484
|
);
|
|
469
485
|
|
|
470
|
-
|
|
486
|
+
if (result.rows.length > 0) {
|
|
487
|
+
return { rows: result.rows.slice(0, limit) };
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
// Fallback: HNSW-first path filtered out to nothing. This can happen when
|
|
491
|
+
// tenant/agent filters are narrow enough to eliminate every NN candidate.
|
|
492
|
+
// Pay the cost of a filter-first scan to guarantee we don't silently return
|
|
493
|
+
// empty when qualifying rows exist. No HNSW on this path — slower, correct.
|
|
494
|
+
const fallbackParams = params.slice(0, params.length - 1); // drop nnLimit
|
|
495
|
+
fallbackParams.push(limit);
|
|
496
|
+
const fallbackLimitPos = fallbackParams.length;
|
|
497
|
+
const fallback = await pool.query(
|
|
498
|
+
`SELECT DISTINCT ON (t.session_row_id)
|
|
499
|
+
s.session_id, s.id AS session_row_id, s.agent_id, s.source, s.started_at,
|
|
500
|
+
ss.summary_text, ss.structured_summary, ss.access_count, ss.last_accessed_at,
|
|
501
|
+
COALESCE(ss.trust_score, 0.5) AS trust_score,
|
|
502
|
+
t.content_text AS matched_turn_text, t.turn_index AS matched_turn_index,
|
|
503
|
+
(t.embedding <=> $${vecPos}::vector) AS turn_distance
|
|
504
|
+
FROM ${qi(schema)}.turn_embeddings t
|
|
505
|
+
JOIN ${qi(schema)}.sessions s ON s.id = t.session_row_id
|
|
506
|
+
LEFT JOIN ${qi(schema)}.session_summaries ss ON ss.session_row_id = s.id
|
|
507
|
+
WHERE ${where.join(' AND ')}
|
|
508
|
+
ORDER BY t.session_row_id, t.embedding <=> $${vecPos}::vector ASC
|
|
509
|
+
LIMIT $${fallbackLimitPos}`,
|
|
510
|
+
fallbackParams
|
|
511
|
+
);
|
|
512
|
+
return { rows: fallback.rows };
|
|
471
513
|
}
|
|
472
514
|
|
|
473
515
|
// ---------------------------------------------------------------------------
|
|
@@ -504,16 +546,32 @@ async function recordFeedback(pool, {
|
|
|
504
546
|
}
|
|
505
547
|
|
|
506
548
|
const trustBefore = parseFloat(current.rows[0].trust_score);
|
|
507
|
-
const trustAfter = verdict === 'helpful'
|
|
508
|
-
? Math.min(1.0, trustBefore + TRUST_UP)
|
|
509
|
-
: Math.max(0.0, trustBefore - TRUST_DOWN);
|
|
510
549
|
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
550
|
+
// Dedupe: the same (agent, verdict) applied more than once must not stack.
|
|
551
|
+
// Audit row is still inserted so the sequence of feedback events is
|
|
552
|
+
// preserved; only the trust_score delta is skipped.
|
|
553
|
+
const prior = await client.query(
|
|
554
|
+
`SELECT 1 FROM ${qi(schema)}.session_feedback
|
|
555
|
+
WHERE session_row_id = $1 AND agent_id = $2 AND verdict = $3
|
|
556
|
+
LIMIT 1`,
|
|
557
|
+
[sessionRowId, agentId, verdict]
|
|
516
558
|
);
|
|
559
|
+
const isDup = prior.rows.length > 0;
|
|
560
|
+
|
|
561
|
+
const trustAfter = isDup
|
|
562
|
+
? trustBefore
|
|
563
|
+
: (verdict === 'helpful'
|
|
564
|
+
? Math.min(1.0, trustBefore + TRUST_UP)
|
|
565
|
+
: Math.max(0.0, trustBefore - TRUST_DOWN));
|
|
566
|
+
|
|
567
|
+
if (!isDup) {
|
|
568
|
+
await client.query(
|
|
569
|
+
`UPDATE ${qi(schema)}.session_summaries
|
|
570
|
+
SET trust_score = $1, updated_at = now()
|
|
571
|
+
WHERE session_row_id = $2`,
|
|
572
|
+
[trustAfter, sessionRowId]
|
|
573
|
+
);
|
|
574
|
+
}
|
|
517
575
|
|
|
518
576
|
await client.query(
|
|
519
577
|
`INSERT INTO ${qi(schema)}.session_feedback
|
|
@@ -523,7 +581,7 @@ async function recordFeedback(pool, {
|
|
|
523
581
|
);
|
|
524
582
|
|
|
525
583
|
await client.query('COMMIT');
|
|
526
|
-
return { trustBefore, trustAfter, verdict };
|
|
584
|
+
return { trustBefore, trustAfter, verdict, duplicate: isDup };
|
|
527
585
|
} catch (err) {
|
|
528
586
|
await client.query('ROLLBACK').catch(() => {});
|
|
529
587
|
throw err;
|