@shadowforge0/aquifer-memory 1.7.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +8 -0
- package/README.md +66 -0
- package/aquifer.config.example.json +19 -0
- package/consumers/cli.js +192 -12
- package/consumers/codex-active-checkpoint.js +186 -0
- package/consumers/codex-current-memory.js +106 -0
- package/consumers/codex-handoff.js +442 -3
- package/consumers/codex.js +164 -107
- package/consumers/mcp.js +144 -6
- package/consumers/shared/config.js +60 -1
- package/consumers/shared/factory.js +10 -3
- package/core/aquifer.js +351 -840
- package/core/backends/capabilities.js +89 -0
- package/core/backends/local.js +430 -0
- package/core/legacy-bootstrap.js +140 -0
- package/core/mcp-manifest.js +66 -2
- package/core/memory-promotion.js +157 -26
- package/core/memory-recall.js +341 -22
- package/core/memory-records.js +128 -8
- package/core/memory-serving.js +132 -0
- package/core/postgres-migrations.js +533 -0
- package/core/public-session-filter.js +40 -0
- package/core/recall-runtime.js +115 -0
- package/core/scope-attribution.js +279 -0
- package/core/session-checkpoint-producer.js +412 -0
- package/core/session-checkpoints.js +432 -0
- package/core/session-finalization.js +82 -1
- package/core/storage-checkpoints.js +546 -0
- package/core/storage.js +121 -8
- package/docs/setup.md +22 -0
- package/package.json +8 -4
- package/schema/014-v1-checkpoint-runs.sql +349 -0
- package/schema/015-v1-evidence-items.sql +92 -0
- package/schema/016-v1-evidence-ref-multi-item.sql +19 -0
- package/schema/017-v1-memory-record-embeddings.sql +25 -0
- package/schema/018-v1-finalization-candidate-envelope.sql +39 -0
- package/scripts/codex-checkpoint-commands.js +464 -0
- package/scripts/codex-checkpoint-runtime.js +520 -0
- package/scripts/codex-recovery.js +105 -0
package/core/aquifer.js
CHANGED
|
@@ -1,171 +1,31 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
3
|
const { Pool } = require('pg');
|
|
4
|
-
const path = require('path');
|
|
5
|
-
const fs = require('fs');
|
|
6
4
|
|
|
7
5
|
const storage = require('./storage');
|
|
8
6
|
const entity = require('./entity');
|
|
9
7
|
const { hybridRank } = require('./hybrid-rank');
|
|
10
8
|
const { summarize } = require('../pipeline/summarize');
|
|
11
9
|
const { extractEntities } = require('../pipeline/extract-entities');
|
|
12
|
-
const { createEmbedder } = require('../pipeline/embed');
|
|
13
10
|
const { applyEnrichSafetyGate, sanitizeSummaryResult } = require('./memory-safety-gate');
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
const
|
|
20
|
-
|
|
21
|
-
function validateSchema(schema) {
|
|
22
|
-
if (!SCHEMA_RE.test(schema)) {
|
|
23
|
-
throw new Error(`Invalid schema name: "${schema}". Must match /^[a-zA-Z_]\\w{0,62}$/`);
|
|
24
|
-
}
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
// C1 fix: quote identifiers to handle reserved words safely
|
|
28
|
-
function qi(identifier) { return `"${identifier}"`; }
|
|
29
|
-
|
|
30
|
-
// ---------------------------------------------------------------------------
|
|
31
|
-
// SQL file loader — replaces ${schema} placeholders
|
|
32
|
-
// ---------------------------------------------------------------------------
|
|
33
|
-
|
|
34
|
-
function loadSql(filename, schema) {
|
|
35
|
-
const filePath = path.join(__dirname, '..', 'schema', filename);
|
|
36
|
-
const raw = fs.readFileSync(filePath, 'utf8');
|
|
37
|
-
// C1: use quoted identifier for safety
|
|
38
|
-
return raw.replace(/\$\{schema\}/g, qi(schema));
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
// ---------------------------------------------------------------------------
|
|
42
|
-
// buildRerankDocument — assemble text for cross-encoder reranking
|
|
43
|
-
// ---------------------------------------------------------------------------
|
|
44
|
-
|
|
45
|
-
function buildRerankDocument(row, maxChars) {
|
|
46
|
-
// Prefer structured_summary fields when available — title/overview carry
|
|
47
|
-
// more signal than summary_text for short Chinese recaps, and topics /
|
|
48
|
-
// decisions / open_loops give the cross-encoder substantive content.
|
|
49
|
-
// Fall back to summary_text / matched_turn_text when structured is absent.
|
|
50
|
-
const ss = row.structured_summary || null;
|
|
51
|
-
const parts = [];
|
|
52
|
-
if (ss) {
|
|
53
|
-
if (ss.title) parts.push(String(ss.title).trim());
|
|
54
|
-
if (ss.overview) parts.push(String(ss.overview).trim());
|
|
55
|
-
if (Array.isArray(ss.topics)) {
|
|
56
|
-
const topics = ss.topics
|
|
57
|
-
.map(t => typeof t === 'string' ? t : (t && t.name ? `${t.name}${t.summary ? ': ' + t.summary : ''}` : ''))
|
|
58
|
-
.filter(Boolean).join(' / ');
|
|
59
|
-
if (topics) parts.push(topics);
|
|
60
|
-
}
|
|
61
|
-
if (Array.isArray(ss.decisions)) {
|
|
62
|
-
const decisions = ss.decisions
|
|
63
|
-
.map(d => typeof d === 'string' ? d : (d && d.decision ? d.decision : ''))
|
|
64
|
-
.filter(Boolean).join(' / ');
|
|
65
|
-
if (decisions) parts.push(`Decisions: ${decisions}`);
|
|
66
|
-
}
|
|
67
|
-
if (Array.isArray(ss.open_loops)) {
|
|
68
|
-
const loops = ss.open_loops
|
|
69
|
-
.map(l => typeof l === 'string' ? l : (l && l.item ? l.item : ''))
|
|
70
|
-
.filter(Boolean).join(' / ');
|
|
71
|
-
if (loops) parts.push(`Open loops: ${loops}`);
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
if (!parts.length) {
|
|
75
|
-
const bare = (row.summary_text || row.summary_snippet || '').trim();
|
|
76
|
-
if (bare) parts.push(bare);
|
|
77
|
-
}
|
|
78
|
-
const turn = (row.matched_turn_text || '').replace(/\s+/g, ' ').trim();
|
|
79
|
-
if (turn) {
|
|
80
|
-
const joined = parts.join(' \n ');
|
|
81
|
-
if (!joined.includes(turn)) parts.push(`Matched turn: ${turn}`);
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
let text = parts.join('\n\n').replace(/[ \t]+/g, ' ').trim();
|
|
85
|
-
if (text.length > maxChars) text = text.slice(0, maxChars);
|
|
86
|
-
return text;
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
// ---------------------------------------------------------------------------
|
|
90
|
-
// resolveEmbedFn — v1.2.0 embed autodetect (explicit > object > env > null)
|
|
91
|
-
// ---------------------------------------------------------------------------
|
|
92
|
-
|
|
93
|
-
function resolveEmbedFn(embedConfig, env) {
|
|
94
|
-
if (embedConfig && typeof embedConfig.fn === 'function') {
|
|
95
|
-
return embedConfig.fn;
|
|
96
|
-
}
|
|
97
|
-
if (embedConfig && embedConfig.provider) {
|
|
98
|
-
const embedder = createEmbedder(embedConfig);
|
|
99
|
-
return (texts) => embedder.embedBatch(texts);
|
|
100
|
-
}
|
|
101
|
-
const provider = env.EMBED_PROVIDER;
|
|
102
|
-
if (!provider) return null;
|
|
103
|
-
|
|
104
|
-
const opts = { provider };
|
|
105
|
-
if (provider === 'ollama') {
|
|
106
|
-
opts.ollamaUrl = env.OLLAMA_URL || env.AQUIFER_EMBED_BASE_URL || 'http://localhost:11434';
|
|
107
|
-
opts.model = env.AQUIFER_EMBED_MODEL || 'bge-m3';
|
|
108
|
-
} else if (provider === 'openai') {
|
|
109
|
-
opts.openaiApiKey = env.OPENAI_API_KEY;
|
|
110
|
-
if (!opts.openaiApiKey) {
|
|
111
|
-
throw new Error('EMBED_PROVIDER=openai requires OPENAI_API_KEY');
|
|
112
|
-
}
|
|
113
|
-
opts.openaiModel = env.AQUIFER_EMBED_MODEL || 'text-embedding-3-small';
|
|
114
|
-
if (env.AQUIFER_EMBED_DIM) opts.openaiDimensions = Number(env.AQUIFER_EMBED_DIM);
|
|
115
|
-
} else {
|
|
116
|
-
throw new Error(`EMBED_PROVIDER=${provider} not supported by autodetect (use 'ollama' or 'openai', or pass config.embed.fn explicitly)`);
|
|
117
|
-
}
|
|
118
|
-
const embedder = createEmbedder(opts);
|
|
119
|
-
return (texts) => embedder.embedBatch(texts);
|
|
120
|
-
}
|
|
11
|
+
const { backendCapabilities, normalizeBackendKind } = require('./backends/capabilities');
|
|
12
|
+
const { createPostgresMigrationRuntime, qi, validateSchema } = require('./postgres-migrations');
|
|
13
|
+
const { createMemoryServingRuntime } = require('./memory-serving');
|
|
14
|
+
const { createLegacyBootstrap } = require('./legacy-bootstrap');
|
|
15
|
+
const { buildRerankDocument, resolveEmbedFn, shouldAutoRerank } = require('./recall-runtime');
|
|
16
|
+
const { filterPublicPlaceholderSessionRows } = require('./public-session-filter');
|
|
121
17
|
|
|
122
18
|
// ---------------------------------------------------------------------------
|
|
123
19
|
// createAquifer
|
|
124
20
|
// ---------------------------------------------------------------------------
|
|
125
21
|
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
if (hasEntities && autoTrigger.alwaysWhenEntities) {
|
|
132
|
-
return { apply: true, reason: 'entities_present' };
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
const len = ranked.length;
|
|
136
|
-
if (len < autoTrigger.minResults) return { apply: false, reason: 'too_few_results' };
|
|
137
|
-
if (len > autoTrigger.maxResults) return { apply: false, reason: 'too_many_results' };
|
|
138
|
-
|
|
139
|
-
const q = String(query || '').trim();
|
|
140
|
-
const tokenCount = q.split(/\s+/).filter(Boolean).length;
|
|
141
|
-
if (q.length < autoTrigger.minQueryChars && tokenCount < autoTrigger.minQueryTokens) {
|
|
142
|
-
return { apply: false, reason: 'query_too_short' };
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
// FTS-only path: rerank when results are wide enough that semantic narrowing
|
|
146
|
-
// is valuable. Cohere-style cross-encoders excel at re-ranking keyword hits.
|
|
147
|
-
if (mode === 'fts') {
|
|
148
|
-
if (len > autoTrigger.ftsMinResults) return { apply: true, reason: 'fts_wide_shortlist' };
|
|
149
|
-
return { apply: false, reason: 'fts_shortlist_too_narrow' };
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
if (!autoTrigger.modes.includes(mode)) {
|
|
153
|
-
return { apply: false, reason: 'mode_not_in_autotrigger_modes' };
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
// Hybrid: if top-1 and top-2 are close, signals are mixed enough to benefit.
|
|
157
|
-
if (len >= 2) {
|
|
158
|
-
const s0 = ranked[0]?._score ?? 0;
|
|
159
|
-
const s1 = ranked[1]?._score ?? 0;
|
|
160
|
-
if (s0 - s1 <= autoTrigger.maxTopScoreGap) {
|
|
161
|
-
return { apply: true, reason: 'top_score_gap_close' };
|
|
162
|
-
}
|
|
22
|
+
function createAquifer(config = {}) {
|
|
23
|
+
const backendKind = normalizeBackendKind(config.backend?.kind || config.storage?.backend || 'postgres');
|
|
24
|
+
if (backendKind !== 'postgres') {
|
|
25
|
+
throw new Error(`createAquifer() only constructs the PostgreSQL backend. Use createAquiferFromConfig() for backend "${backendKind}".`);
|
|
163
26
|
}
|
|
27
|
+
const backendInfo = backendCapabilities(backendKind);
|
|
164
28
|
|
|
165
|
-
return { apply: false, reason: 'top_score_gap_wide' };
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
function createAquifer(config = {}) {
|
|
169
29
|
// v1.2.0: db falls back to DATABASE_URL / AQUIFER_DB_URL env so hosts can
|
|
170
30
|
// call createAquifer() with zero args for install-and-go.
|
|
171
31
|
const dbInput = config.db !== undefined
|
|
@@ -270,114 +130,9 @@ function createAquifer(config = {}) {
|
|
|
270
130
|
// Source registry (in-memory)
|
|
271
131
|
const sources = new Map();
|
|
272
132
|
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
let migratePromise = null;
|
|
276
|
-
|
|
277
|
-
// FTS tsconfig — auto-detected during migrate(). 'zhcfg' if zhparser is
|
|
278
|
-
// installed (better Chinese segmentation), otherwise 'simple' (legacy).
|
|
279
|
-
// Override via config.ftsConfig if you need to force one or the other.
|
|
280
|
-
let ftsConfig = config.ftsConfig || null;
|
|
281
|
-
|
|
282
|
-
const memoryCfg = config.memory || {};
|
|
283
|
-
const memoryServingMode = memoryCfg.servingMode || process.env.AQUIFER_MEMORY_SERVING_MODE || 'legacy';
|
|
284
|
-
function splitScopePath(value) {
|
|
285
|
-
if (Array.isArray(value)) return value.map(v => String(v).trim()).filter(Boolean);
|
|
286
|
-
if (typeof value !== 'string') return null;
|
|
287
|
-
const parts = value.split(',').map(v => v.trim()).filter(Boolean);
|
|
288
|
-
return parts.length > 0 ? parts : null;
|
|
289
|
-
}
|
|
290
|
-
const defaultActiveScopeKey = memoryCfg.activeScopeKey || process.env.AQUIFER_MEMORY_ACTIVE_SCOPE_KEY || null;
|
|
291
|
-
const defaultActiveScopePath = splitScopePath(
|
|
292
|
-
memoryCfg.activeScopePath || process.env.AQUIFER_MEMORY_ACTIVE_SCOPE_PATH || null,
|
|
293
|
-
);
|
|
294
|
-
function resolveMemoryServingMode(opts = {}) {
|
|
295
|
-
const mode = opts.memoryMode || opts.servingMode || memoryServingMode;
|
|
296
|
-
if (mode === 'legacy' || mode === 'evidence') return 'legacy';
|
|
297
|
-
if (mode === 'curated') return 'curated';
|
|
298
|
-
throw new Error(`Invalid memory serving mode: "${mode}". Must be one of: legacy, curated`);
|
|
299
|
-
}
|
|
300
|
-
function withDefaultMemoryScope(opts = {}) {
|
|
301
|
-
const next = { ...opts };
|
|
302
|
-
if (!next.activeScopePath && defaultActiveScopePath) next.activeScopePath = defaultActiveScopePath;
|
|
303
|
-
if (!next.activeScopeKey && defaultActiveScopeKey) next.activeScopeKey = defaultActiveScopeKey;
|
|
304
|
-
return next;
|
|
305
|
-
}
|
|
306
|
-
function assertCuratedRecallOpts(opts = {}) {
|
|
307
|
-
const unsupported = [];
|
|
308
|
-
for (const key of ['agentId', 'agentIds', 'source', 'dateFrom', 'dateTo', 'entities', 'entityMode', 'mode', 'weights', 'rerank', 'allowUnsafeDebug', 'unsafeDebug']) {
|
|
309
|
-
if (opts[key] !== undefined && opts[key] !== null) unsupported.push(key);
|
|
310
|
-
}
|
|
311
|
-
if (unsupported.length > 0) {
|
|
312
|
-
throw new Error(`curated session_recall does not support legacy filters: ${unsupported.join(', ')}. Use activeScopeKey/activeScopePath or evidence_recall.`);
|
|
313
|
-
}
|
|
314
|
-
}
|
|
315
|
-
function assertCuratedBootstrapOpts(opts = {}) {
|
|
316
|
-
const unsupported = [];
|
|
317
|
-
for (const key of ['agentId', 'source', 'lookbackDays', 'dateFrom', 'dateTo']) {
|
|
318
|
-
if (opts[key] !== undefined && opts[key] !== null) unsupported.push(key);
|
|
319
|
-
}
|
|
320
|
-
if (unsupported.length > 0) {
|
|
321
|
-
throw new Error(`curated session_bootstrap does not support legacy filters: ${unsupported.join(', ')}. Use activeScopeKey/activeScopePath.`);
|
|
322
|
-
}
|
|
323
|
-
}
|
|
324
|
-
function hasEvidenceBoundary(opts = {}) {
|
|
325
|
-
return Boolean(
|
|
326
|
-
opts.agentId
|
|
327
|
-
|| (Array.isArray(opts.agentIds) && opts.agentIds.length > 0)
|
|
328
|
-
|| opts.source
|
|
329
|
-
|| opts.dateFrom
|
|
330
|
-
|| opts.dateTo
|
|
331
|
-
|| opts.host
|
|
332
|
-
|| opts.sessionId
|
|
333
|
-
|| opts.allowUnsafeDebug === true
|
|
334
|
-
|| opts.unsafeDebug === true
|
|
335
|
-
);
|
|
336
|
-
}
|
|
337
|
-
function curatedRecallTitle(row = {}) {
|
|
338
|
-
const title = row.title || row.summary || row.canonical_key || row.canonicalKey || row.memory_type || row.memoryType || 'memory';
|
|
339
|
-
return String(title).trim();
|
|
340
|
-
}
|
|
341
|
-
function curatedRecallSummary(row = {}) {
|
|
342
|
-
const summary = row.summary || row.title || row.canonical_key || row.canonicalKey || '';
|
|
343
|
-
return String(summary).trim();
|
|
344
|
-
}
|
|
345
|
-
function normalizeCuratedRecallRow(row = {}) {
|
|
346
|
-
const memoryId = row.memoryId || row.memory_id || row.id || null;
|
|
347
|
-
const canonicalKey = row.canonicalKey || row.canonical_key || null;
|
|
348
|
-
const memoryType = row.memoryType || row.memory_type || null;
|
|
349
|
-
const scopeKey = row.scopeKey || row.scope_key || null;
|
|
350
|
-
const scopeKind = row.scopeKind || row.scope_kind || null;
|
|
351
|
-
const summaryText = curatedRecallSummary(row) || null;
|
|
352
|
-
const title = curatedRecallTitle(row) || null;
|
|
353
|
-
const scoreValue = row.recall_score ?? row.score ?? row.lexical_rank ?? null;
|
|
354
|
-
const score = scoreValue === null ? null : Number(scoreValue);
|
|
355
|
-
return {
|
|
356
|
-
...row,
|
|
357
|
-
memoryId: memoryId === null ? null : String(memoryId),
|
|
358
|
-
canonicalKey,
|
|
359
|
-
memoryType,
|
|
360
|
-
scopeKey,
|
|
361
|
-
scopeKind,
|
|
362
|
-
title,
|
|
363
|
-
summaryText,
|
|
364
|
-
structuredSummary: {
|
|
365
|
-
title,
|
|
366
|
-
overview: summaryText,
|
|
367
|
-
},
|
|
368
|
-
startedAt: row.acceptedAt || row.accepted_at || row.observedAt || row.observed_at || null,
|
|
369
|
-
score: Number.isFinite(score) ? score : null,
|
|
370
|
-
feedbackTarget: {
|
|
371
|
-
kind: 'memory_feedback',
|
|
372
|
-
memoryId: memoryId === null ? null : String(memoryId),
|
|
373
|
-
canonicalKey,
|
|
374
|
-
},
|
|
375
|
-
};
|
|
376
|
-
}
|
|
133
|
+
const memoryServing = createMemoryServingRuntime(config.memory || {}, process.env);
|
|
134
|
+
const legacyBootstrap = createLegacyBootstrap({ pool, schema, tenantId, formatBootstrapText });
|
|
377
135
|
|
|
378
|
-
// State-change extraction (Q3): off by default. When enabled, enrich() runs
|
|
379
|
-
// an extra LLM call to capture temporal state transitions on whitelisted
|
|
380
|
-
// entities. See pipeline/extract-state-changes.js + core/entity-state.js.
|
|
381
136
|
const stateChangesCfg = config.stateChanges || {};
|
|
382
137
|
const stateChangesEnabled = stateChangesCfg.enabled === true;
|
|
383
138
|
const stateChangesWhitelist = new Set(
|
|
@@ -392,112 +147,17 @@ function createAquifer(config = {}) {
|
|
|
392
147
|
const stateChangesMaxOutputTokens = Number.isFinite(stateChangesCfg.maxOutputTokens)
|
|
393
148
|
? stateChangesCfg.maxOutputTokens : 600;
|
|
394
149
|
|
|
395
|
-
const
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
? Math.max(0, migrationsCfg.lockTimeoutMs) : 30000;
|
|
404
|
-
const migrationStartupTimeoutMs = Number.isFinite(migrationsCfg.startupTimeoutMs)
|
|
405
|
-
? Math.max(0, migrationsCfg.startupTimeoutMs) : 60000;
|
|
406
|
-
const migrationOnEvent = typeof migrationsCfg.onEvent === 'function' ? migrationsCfg.onEvent : null;
|
|
407
|
-
|
|
408
|
-
function emitMigrationEvent(name, payload) {
|
|
409
|
-
if (!migrationOnEvent) return;
|
|
410
|
-
try { migrationOnEvent({ name, schema, ...payload }); } catch (err) {
|
|
411
|
-
console.warn(`[aquifer] migrations.onEvent handler threw: ${err.message}`);
|
|
412
|
-
}
|
|
413
|
-
}
|
|
414
|
-
|
|
415
|
-
// Expected migration set — used for lazy plan introspection. `always: true`
|
|
416
|
-
// runs every migrate(); others are gated by feature flags. Signature tables
|
|
417
|
-
// let listPendingMigrations() probe pg_tables without executing DDL.
|
|
418
|
-
const MIGRATION_PLAN = [
|
|
419
|
-
{ id: '001-base', file: '001-base.sql', always: true, signature: 'sessions' },
|
|
420
|
-
{ id: '002-entities', file: '002-entities.sql', gate: 'entities', signature: 'entities' },
|
|
421
|
-
{ id: '003-trust-feedback', file: '003-trust-feedback.sql', always: true, signature: 'session_feedback' },
|
|
422
|
-
{ id: '004-facts', file: '004-facts.sql', gate: 'facts', signature: 'facts' },
|
|
423
|
-
{ id: '004-completion', file: '004-completion.sql', always: true, signature: 'narratives' },
|
|
424
|
-
{ id: '005-entity-state-history',file: '005-entity-state-history.sql',gate: 'entities', signature: 'entity_state_history' },
|
|
425
|
-
{ id: '006-insights', file: '006-insights.sql', always: true, signature: 'insights' },
|
|
426
|
-
{ id: '007-v1-foundation', file: '007-v1-foundation.sql', always: true, signature: 'memory_records' },
|
|
427
|
-
{ id: '008-session-finalizations',file: '008-session-finalizations.sql',always: true, signature: 'session_finalizations' },
|
|
428
|
-
{ id: '009-v1-assertion-plane', file: '009-v1-assertion-plane.sql', always: true, signature: 'fact_assertions_v1' },
|
|
429
|
-
{ id: '010-v1-finalization-review',file: '010-v1-finalization-review.sql',always: true, signature: 'finalization_candidates' },
|
|
430
|
-
{ id: '011-v1-compaction-claim', file: '011-v1-compaction-claim.sql', always: true, signature: { table: 'compaction_runs', column: 'apply_token' } },
|
|
431
|
-
{ id: '012-v1-compaction-lease', file: '012-v1-compaction-lease.sql', always: true, signature: { table: 'compaction_runs', column: 'lease_expires_at' } },
|
|
432
|
-
{ id: '013-v1-compaction-lineage', file: '013-v1-compaction-lineage.sql', always: true, signature: 'compaction_candidates' },
|
|
433
|
-
];
|
|
434
|
-
|
|
435
|
-
function requiredMigrations() {
|
|
436
|
-
return MIGRATION_PLAN
|
|
437
|
-
.filter(m => m.always
|
|
438
|
-
|| (m.gate === 'entities' && entitiesEnabled)
|
|
439
|
-
|| (m.gate === 'facts' && factsEnabled))
|
|
440
|
-
.map(m => m.id);
|
|
441
|
-
}
|
|
442
|
-
|
|
443
|
-
async function readAppliedMigrations(queryRunner) {
|
|
444
|
-
const required = MIGRATION_PLAN.filter(m => m.always
|
|
445
|
-
|| (m.gate === 'entities' && entitiesEnabled)
|
|
446
|
-
|| (m.gate === 'facts' && factsEnabled));
|
|
447
|
-
const tableSignatures = required
|
|
448
|
-
.map(m => m.signature)
|
|
449
|
-
.filter(signature => typeof signature === 'string');
|
|
450
|
-
const columnSignatures = required
|
|
451
|
-
.map(m => m.signature)
|
|
452
|
-
.filter(signature => signature && typeof signature === 'object');
|
|
453
|
-
const presentTables = new Set();
|
|
454
|
-
const presentColumns = new Set();
|
|
455
|
-
if (tableSignatures.length > 0) {
|
|
456
|
-
const r = await queryRunner.query(
|
|
457
|
-
`SELECT tablename FROM pg_tables
|
|
458
|
-
WHERE schemaname = $1 AND tablename = ANY($2::text[])`,
|
|
459
|
-
[schema, tableSignatures]
|
|
460
|
-
);
|
|
461
|
-
for (const row of r.rows) presentTables.add(row.tablename);
|
|
462
|
-
}
|
|
463
|
-
if (columnSignatures.length > 0) {
|
|
464
|
-
const tables = [...new Set(columnSignatures.map(signature => signature.table))];
|
|
465
|
-
const r = await queryRunner.query(
|
|
466
|
-
`SELECT table_name, column_name
|
|
467
|
-
FROM information_schema.columns
|
|
468
|
-
WHERE table_schema = $1 AND table_name = ANY($2::text[])`,
|
|
469
|
-
[schema, tables]
|
|
470
|
-
);
|
|
471
|
-
for (const row of r.rows) presentColumns.add(`${row.table_name}.${row.column_name}`);
|
|
472
|
-
}
|
|
473
|
-
return required
|
|
474
|
-
.filter(m => {
|
|
475
|
-
if (typeof m.signature === 'string') return presentTables.has(m.signature);
|
|
476
|
-
return presentColumns.has(`${m.signature.table}.${m.signature.column}`);
|
|
477
|
-
})
|
|
478
|
-
.map(m => m.id);
|
|
479
|
-
}
|
|
480
|
-
|
|
481
|
-
async function buildMigrationPlan(queryRunner) {
|
|
482
|
-
const required = requiredMigrations();
|
|
483
|
-
const applied = await readAppliedMigrations(queryRunner);
|
|
484
|
-
const appliedSet = new Set(applied);
|
|
485
|
-
const pending = required.filter(id => !appliedSet.has(id));
|
|
486
|
-
return { required, applied, pending };
|
|
487
|
-
}
|
|
150
|
+
const migrationRuntime = createPostgresMigrationRuntime({
|
|
151
|
+
pool,
|
|
152
|
+
schema,
|
|
153
|
+
migrations: config.migrations || {},
|
|
154
|
+
getEntitiesEnabled: () => entitiesEnabled,
|
|
155
|
+
getFactsEnabled: () => factsEnabled,
|
|
156
|
+
initialFtsConfig: config.ftsConfig || null,
|
|
157
|
+
});
|
|
488
158
|
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
if (migratePromise) return migratePromise;
|
|
492
|
-
if (migrationsMode === 'off') { migrated = true; return; }
|
|
493
|
-
if (migrationsMode === 'check') {
|
|
494
|
-
// Lazy compare only — don't execute DDL implicitly.
|
|
495
|
-
const plan = await buildMigrationPlan(pool).catch(() => null);
|
|
496
|
-
if (plan && plan.pending.length === 0) migrated = true;
|
|
497
|
-
return;
|
|
498
|
-
}
|
|
499
|
-
migratePromise = aquifer.migrate().finally(() => { migratePromise = null; });
|
|
500
|
-
return migratePromise;
|
|
159
|
+
function ensureMigrated() {
|
|
160
|
+
return migrationRuntime.ensureMigrated();
|
|
501
161
|
}
|
|
502
162
|
|
|
503
163
|
// =========================================================================
|
|
@@ -512,273 +172,11 @@ function createAquifer(config = {}) {
|
|
|
512
172
|
},
|
|
513
173
|
|
|
514
174
|
async migrate() {
|
|
515
|
-
|
|
516
|
-
// Advisory lock prevents concurrent migrations across processes.
|
|
517
|
-
// Lock key is derived from schema name to allow parallel migration
|
|
518
|
-
// of different schemas in the same database.
|
|
519
|
-
const lockKey = Buffer.from(`aquifer:${schema}`).reduce((h, b) => (h * 31 + b) & 0x7fffffff, 0);
|
|
520
|
-
|
|
521
|
-
emitMigrationEvent('init_started', { mode: migrationsMode });
|
|
522
|
-
|
|
523
|
-
// Run all migration DDL on a single checked-out client so we can
|
|
524
|
-
// capture RAISE NOTICE/WARNING emitted by the DO blocks. node-postgres
|
|
525
|
-
// swallows notices on pool.query(); attaching a 'notice' listener to a
|
|
526
|
-
// held client surfaces them. Fall back to pool.query() when the caller
|
|
527
|
-
// passed a bare mock (no connect/release) — tests using minimal pool
|
|
528
|
-
// stubs still exercise the migration shape, just without notice capture.
|
|
529
|
-
const supportsCheckout = typeof pool.connect === 'function';
|
|
530
|
-
const client = supportsCheckout ? await pool.connect() : pool;
|
|
531
|
-
const releasesClient = supportsCheckout && typeof client.release === 'function';
|
|
532
|
-
const notices = [];
|
|
533
|
-
const onNotice = (n) => {
|
|
534
|
-
notices.push({ severity: n.severity || 'NOTICE', message: n.message || String(n) });
|
|
535
|
-
};
|
|
536
|
-
const hasEvents = typeof client.on === 'function' && typeof client.off === 'function';
|
|
537
|
-
if (hasEvents) client.on('notice', onNotice);
|
|
538
|
-
|
|
539
|
-
const ddlExecuted = [];
|
|
540
|
-
let lockAcquired = false;
|
|
541
|
-
|
|
542
|
-
try {
|
|
543
|
-
// Plan probe before lock: lets consumers see pending list and lets
|
|
544
|
-
// us emit an accurate check_completed event even when the DDL is a
|
|
545
|
-
// no-op on an already-migrated schema.
|
|
546
|
-
const planBefore = await buildMigrationPlan(client).catch(() => null);
|
|
547
|
-
emitMigrationEvent('check_completed', {
|
|
548
|
-
required: planBefore ? planBefore.required : requiredMigrations(),
|
|
549
|
-
applied: planBefore ? planBefore.applied : [],
|
|
550
|
-
pending: planBefore ? planBefore.pending : requiredMigrations(),
|
|
551
|
-
});
|
|
552
|
-
|
|
553
|
-
// Try-lock with poll + timeout. Replaces the old blocking
|
|
554
|
-
// pg_advisory_lock() which could hang indefinitely if another
|
|
555
|
-
// process crashed holding the lock. Defensive against mock pools:
|
|
556
|
-
// only poll when PG explicitly returns ok=false; a missing/empty
|
|
557
|
-
// response (test mocks that don't model pg_try_advisory_lock) is
|
|
558
|
-
// treated as acquired so suite doesn't hang on the deadline.
|
|
559
|
-
const lockDeadline = Date.now() + migrationLockTimeoutMs;
|
|
560
|
-
const pollMs = 250;
|
|
561
|
-
while (true) {
|
|
562
|
-
const r = await client.query('SELECT pg_try_advisory_lock($1) AS ok', [lockKey]);
|
|
563
|
-
const row = r && r.rows ? r.rows[0] : null;
|
|
564
|
-
if (row && row.ok === false) {
|
|
565
|
-
if (Date.now() >= lockDeadline) break;
|
|
566
|
-
await new Promise(res => setTimeout(res, pollMs));
|
|
567
|
-
continue;
|
|
568
|
-
}
|
|
569
|
-
lockAcquired = true;
|
|
570
|
-
break;
|
|
571
|
-
}
|
|
572
|
-
if (!lockAcquired) {
|
|
573
|
-
const err = new Error(`aquifer: failed to acquire migration advisory lock within ${migrationLockTimeoutMs}ms for schema "${schema}"`);
|
|
574
|
-
err.code = 'AQ_MIGRATION_LOCK_TIMEOUT';
|
|
575
|
-
err.failedAt = 'acquire_lock';
|
|
576
|
-
throw err;
|
|
577
|
-
}
|
|
578
|
-
|
|
579
|
-
emitMigrationEvent('apply_started', {
|
|
580
|
-
pending: planBefore ? planBefore.pending : requiredMigrations(),
|
|
581
|
-
});
|
|
582
|
-
|
|
583
|
-
try {
|
|
584
|
-
// 1. Run base DDL
|
|
585
|
-
const baseSql = loadSql('001-base.sql', schema);
|
|
586
|
-
await client.query(baseSql); ddlExecuted.push('001-base');
|
|
587
|
-
|
|
588
|
-
// 2. If entities enabled, run entity DDL
|
|
589
|
-
if (entitiesEnabled) {
|
|
590
|
-
const entitySql = loadSql('002-entities.sql', schema);
|
|
591
|
-
await client.query(entitySql); ddlExecuted.push('002-entities');
|
|
592
|
-
}
|
|
593
|
-
|
|
594
|
-
// 3. Trust + feedback (always, not gated by entities)
|
|
595
|
-
const trustSql = loadSql('003-trust-feedback.sql', schema);
|
|
596
|
-
await client.query(trustSql); ddlExecuted.push('003-trust-feedback');
|
|
597
|
-
|
|
598
|
-
// 4. Facts / consolidation (opt-in)
|
|
599
|
-
if (factsEnabled) {
|
|
600
|
-
const factsSql = loadSql('004-facts.sql', schema);
|
|
601
|
-
await client.query(factsSql); ddlExecuted.push('004-facts');
|
|
602
|
-
}
|
|
603
|
-
|
|
604
|
-
// 5. Completion foundation (always, additive): narratives,
|
|
605
|
-
// consumer_profiles, sessions.consolidation_phases. Pure additive DDL
|
|
606
|
-
// with IF NOT EXISTS guards — safe on every migrate() call.
|
|
607
|
-
const completionSql = loadSql('004-completion.sql', schema);
|
|
608
|
-
await client.query(completionSql); ddlExecuted.push('004-completion');
|
|
609
|
-
|
|
610
|
-
// 6. Entity state history (always, gated by entitiesEnabled because
|
|
611
|
-
// it FK-references entities). Drop-clean — see scripts/drop-entity-state-history.sql.
|
|
612
|
-
if (entitiesEnabled) {
|
|
613
|
-
const stateHistorySql = loadSql('005-entity-state-history.sql', schema);
|
|
614
|
-
await client.query(stateHistorySql); ddlExecuted.push('005-entity-state-history');
|
|
615
|
-
}
|
|
616
|
-
|
|
617
|
-
// 7. Insights (always, additive). No FK from anywhere into this table —
|
|
618
|
-
// safe to DROP CASCADE. See scripts/drop-insights.sql.
|
|
619
|
-
const insightsSql = loadSql('006-insights.sql', schema);
|
|
620
|
-
await client.query(insightsSql); ddlExecuted.push('006-insights');
|
|
621
|
-
|
|
622
|
-
// 8. v1 curated-memory foundation (always, additive). This creates
|
|
623
|
-
// a sidecar curated plane without changing the legacy session recall
|
|
624
|
-
// or bootstrap serving path.
|
|
625
|
-
const memoryV1Sql = loadSql('007-v1-foundation.sql', schema);
|
|
626
|
-
await client.query(memoryV1Sql); ddlExecuted.push('007-v1-foundation');
|
|
627
|
-
|
|
628
|
-
// 9. v1 session finalization ledger (always, additive). Finalization
|
|
629
|
-
// is the source-of-truth lifecycle for handoff/session-end/recovery
|
|
630
|
-
// triggers; local consumer markers are cache hints only.
|
|
631
|
-
const finalizationSql = loadSql('008-session-finalizations.sql', schema);
|
|
632
|
-
await client.query(finalizationSql); ddlExecuted.push('008-session-finalizations');
|
|
633
|
-
|
|
634
|
-
// 10. v1 structured assertion plane (always, additive). This keeps
|
|
635
|
-
// legacy 004-facts untouched and adds the minimal DB contract for
|
|
636
|
-
// backing structured assertions, tenant-safe scope ancestry guards,
|
|
637
|
-
// and compaction coverage fields.
|
|
638
|
-
const assertionPlaneSql = loadSql('009-v1-assertion-plane.sql', schema);
|
|
639
|
-
await client.query(assertionPlaneSql); ddlExecuted.push('009-v1-assertion-plane');
|
|
640
|
-
|
|
641
|
-
// 11. v1 finalization review and lineage (always, additive). This
|
|
642
|
-
// stores human review text, minimal SessionStart text, finalization
|
|
643
|
-
// candidate rows, and row-level created_by_finalization_id lineage.
|
|
644
|
-
const finalizationReviewSql = loadSql('010-v1-finalization-review.sql', schema);
|
|
645
|
-
await client.query(finalizationReviewSql); ddlExecuted.push('010-v1-finalization-review');
|
|
646
|
-
|
|
647
|
-
// 12. v1 compaction claim/apply guard (always, additive). This adds
|
|
648
|
-
// a minimal claim token and one-live-apply guard for compaction_runs;
|
|
649
|
-
// it does not create or promote aggregate memory.
|
|
650
|
-
const compactionClaimSql = loadSql('011-v1-compaction-claim.sql', schema);
|
|
651
|
-
await client.query(compactionClaimSql); ddlExecuted.push('011-v1-compaction-claim');
|
|
652
|
-
|
|
653
|
-
// 13. v1 compaction claim lease (always, additive). This persists
|
|
654
|
-
// row-level lease expiry so stale applying claims can be reclaimed
|
|
655
|
-
// without relying on caller clocks or changing runtime defaults.
|
|
656
|
-
const compactionLeaseSql = loadSql('012-v1-compaction-lease.sql', schema);
|
|
657
|
-
await client.query(compactionLeaseSql); ddlExecuted.push('012-v1-compaction-lease');
|
|
658
|
-
|
|
659
|
-
// 14. v1 compaction lineage and candidate ledger (always,
|
|
660
|
-
// additive). This records aggregate candidate outcomes and links
|
|
661
|
-
// promoted rows back to the compaction run that created them.
|
|
662
|
-
const compactionLineageSql = loadSql('013-v1-compaction-lineage.sql', schema);
|
|
663
|
-
await client.query(compactionLineageSql); ddlExecuted.push('013-v1-compaction-lineage');
|
|
664
|
-
|
|
665
|
-
migrated = true;
|
|
666
|
-
} finally {
|
|
667
|
-
await client.query('SELECT pg_advisory_unlock($1)', [lockKey]).catch((err) => {
|
|
668
|
-
console.warn(`[aquifer] failed to release migration advisory lock for schema "${schema}": ${err.message}`);
|
|
669
|
-
});
|
|
670
|
-
}
|
|
671
|
-
} catch (err) {
|
|
672
|
-
err.notices = Array.isArray(err.notices) ? err.notices : notices.slice();
|
|
673
|
-
err.failedAt = err.failedAt || 'apply_ddl';
|
|
674
|
-
emitMigrationEvent('apply_failed', {
|
|
675
|
-
error: { code: err.code || null, message: err.message },
|
|
676
|
-
failedAt: err.failedAt,
|
|
677
|
-
notices: err.notices,
|
|
678
|
-
durationMs: Date.now() - t0,
|
|
679
|
-
});
|
|
680
|
-
throw err;
|
|
681
|
-
} finally {
|
|
682
|
-
if (hasEvents) client.off('notice', onNotice);
|
|
683
|
-
if (releasesClient) client.release();
|
|
684
|
-
}
|
|
685
|
-
|
|
686
|
-
// Surface captured migration notices that operators need to see:
|
|
687
|
-
// - any WARNING/ERROR (zhcfg rebuild warnings, HNSW OOM, etc.)
|
|
688
|
-
// - aquifer-authored NOTICE messages ('[aquifer] ...' prefix in the
|
|
689
|
-
// migration DO blocks; these announce extension-install fallback,
|
|
690
|
-
// HNSW deferral, and other operational decisions)
|
|
691
|
-
// Filtered out: PG's own "relation already exists, skipping" and
|
|
692
|
-
// similar idempotent-DDL chatter that floods a re-run.
|
|
693
|
-
for (const n of notices) {
|
|
694
|
-
const sev = (n.severity || 'NOTICE').toUpperCase();
|
|
695
|
-
const msg = n.message || '';
|
|
696
|
-
const line = `[aquifer] migration ${sev.toLowerCase()}: ${msg}`;
|
|
697
|
-
if (sev === 'WARNING' || sev === 'ERROR') {
|
|
698
|
-
console.warn(line);
|
|
699
|
-
} else if (sev === 'NOTICE' && msg.startsWith('[aquifer]')) {
|
|
700
|
-
process.stderr.write(line + '\n');
|
|
701
|
-
}
|
|
702
|
-
}
|
|
703
|
-
|
|
704
|
-
// Auto-detect FTS tsconfig if not forced by config. Restrict to the
|
|
705
|
-
// public namespace — same restriction the trigger function uses — so a
|
|
706
|
-
// same-named config in another schema doesn't fool the detection.
|
|
707
|
-
if (!ftsConfig) {
|
|
708
|
-
try {
|
|
709
|
-
const r = await pool.query(
|
|
710
|
-
`SELECT 1 FROM pg_ts_config
|
|
711
|
-
WHERE cfgname = 'zhcfg' AND cfgnamespace = 'public'::regnamespace
|
|
712
|
-
LIMIT 1`);
|
|
713
|
-
ftsConfig = r.rowCount > 0 ? 'zhcfg' : 'simple';
|
|
714
|
-
} catch {
|
|
715
|
-
ftsConfig = 'simple';
|
|
716
|
-
}
|
|
717
|
-
}
|
|
718
|
-
|
|
719
|
-
// Post-flight: surface which Chinese FTS backend the migration actually
|
|
720
|
-
// landed on, and warm the backend's tokenizer so the first live query
|
|
721
|
-
// doesn't pay cold-start cost unpredictably. RAISE NOTICE/WARNING from
|
|
722
|
-
// the migration DO blocks are swallowed by node-postgres unless a
|
|
723
|
-
// notice handler is attached, so without this operators can't tell if
|
|
724
|
-
// pg_jieba silently failed to install and FTS is degraded to 'simple'.
|
|
725
|
-
//
|
|
726
|
-
// pg_jieba first-backend load is ~60MB RAM + 0.5-1s to mmap the dict.
|
|
727
|
-
// Warming once inside migrate() amortizes that on the backend that runs
|
|
728
|
-
// migration; other pool backends still pay it on first use, but the
|
|
729
|
-
// timing surfaces the cost so operators who see unexpected latency
|
|
730
|
-
// know where to look.
|
|
731
|
-
try {
|
|
732
|
-
const f = await pool.query(`
|
|
733
|
-
SELECT
|
|
734
|
-
EXISTS(SELECT 1 FROM pg_extension WHERE extname='pg_jieba') AS have_jieba,
|
|
735
|
-
EXISTS(SELECT 1 FROM pg_extension WHERE extname='zhparser') AS have_zhparser,
|
|
736
|
-
(SELECT p.prsname FROM pg_ts_config c
|
|
737
|
-
JOIN pg_ts_parser p ON c.cfgparser = p.oid
|
|
738
|
-
WHERE c.cfgname='zhcfg' AND c.cfgnamespace='public'::regnamespace
|
|
739
|
-
LIMIT 1) AS zhcfg_parser
|
|
740
|
-
`);
|
|
741
|
-
const row = f.rows[0] || {};
|
|
742
|
-
const backend = row.zhcfg_parser
|
|
743
|
-
? `zhcfg(parser=${row.zhcfg_parser})`
|
|
744
|
-
: `simple (no zhcfg in public namespace)`;
|
|
745
|
-
|
|
746
|
-
let warmupMs = null;
|
|
747
|
-
if (row.zhcfg_parser) {
|
|
748
|
-
const t0 = Date.now();
|
|
749
|
-
await pool.query(`SELECT to_tsvector('zhcfg', $1)`, ['warmup 記憶系統 aquifer'])
|
|
750
|
-
.catch(() => {});
|
|
751
|
-
warmupMs = Date.now() - t0;
|
|
752
|
-
}
|
|
753
|
-
|
|
754
|
-
const warmupNote = warmupMs !== null ? ` warmup=${warmupMs}ms` : '';
|
|
755
|
-
process.stderr.write(
|
|
756
|
-
`[aquifer] FTS post-flight: backend=${backend} ` +
|
|
757
|
-
`jieba=${row.have_jieba} zhparser=${row.have_zhparser} ` +
|
|
758
|
-
`selected=${ftsConfig}${warmupNote}\n`
|
|
759
|
-
);
|
|
760
|
-
if (warmupMs !== null && warmupMs > 500) {
|
|
761
|
-
process.stderr.write(
|
|
762
|
-
`[aquifer] Note: first FTS call paid ~${warmupMs}ms for tokenizer init ` +
|
|
763
|
-
`(dictionary mmap). Subsequent calls on the same backend are cached.\n`
|
|
764
|
-
);
|
|
765
|
-
}
|
|
766
|
-
} catch (err) {
|
|
767
|
-
console.warn(`[aquifer] FTS post-flight check failed: ${err.message}`);
|
|
768
|
-
}
|
|
769
|
-
|
|
770
|
-
const durationMs = Date.now() - t0;
|
|
771
|
-
emitMigrationEvent('apply_succeeded', {
|
|
772
|
-
ddlExecuted,
|
|
773
|
-
durationMs,
|
|
774
|
-
notices: notices.slice(),
|
|
775
|
-
});
|
|
776
|
-
return { ok: true, durationMs, notices: notices.slice(), ddlExecuted };
|
|
175
|
+
return migrationRuntime.migrate();
|
|
777
176
|
},
|
|
778
177
|
|
|
779
178
|
async listPendingMigrations() {
|
|
780
|
-
|
|
781
|
-
return { ...plan, lastRunAt: null };
|
|
179
|
+
return migrationRuntime.listPendingMigrations();
|
|
782
180
|
},
|
|
783
181
|
|
|
784
182
|
async getMigrationStatus() {
|
|
@@ -786,94 +184,7 @@ function createAquifer(config = {}) {
|
|
|
786
184
|
},
|
|
787
185
|
|
|
788
186
|
async init() {
|
|
789
|
-
|
|
790
|
-
const mode = migrationsMode;
|
|
791
|
-
|
|
792
|
-
let deadlineTimer = null;
|
|
793
|
-
const startupDeadline = migrationStartupTimeoutMs > 0
|
|
794
|
-
? new Promise((_, reject) => {
|
|
795
|
-
deadlineTimer = setTimeout(() => {
|
|
796
|
-
const err = new Error(`aquifer: init() exceeded startupTimeoutMs=${migrationStartupTimeoutMs}ms`);
|
|
797
|
-
err.code = 'AQ_MIGRATION_STARTUP_TIMEOUT';
|
|
798
|
-
reject(err);
|
|
799
|
-
}, migrationStartupTimeoutMs);
|
|
800
|
-
if (typeof deadlineTimer.unref === 'function') deadlineTimer.unref();
|
|
801
|
-
})
|
|
802
|
-
: null;
|
|
803
|
-
const withDeadline = (p) => startupDeadline ? Promise.race([p, startupDeadline]) : p;
|
|
804
|
-
const clearDeadline = () => { if (deadlineTimer) { clearTimeout(deadlineTimer); deadlineTimer = null; } };
|
|
805
|
-
|
|
806
|
-
try {
|
|
807
|
-
let plan;
|
|
808
|
-
try {
|
|
809
|
-
plan = await withDeadline(buildMigrationPlan(pool));
|
|
810
|
-
} catch (err) {
|
|
811
|
-
const durationMs = Date.now() - t0;
|
|
812
|
-
emitMigrationEvent('apply_failed', {
|
|
813
|
-
error: { code: err.code || null, message: err.message },
|
|
814
|
-
failedAt: 'plan_probe',
|
|
815
|
-
notices: [],
|
|
816
|
-
durationMs,
|
|
817
|
-
});
|
|
818
|
-
return {
|
|
819
|
-
ready: false,
|
|
820
|
-
memoryMode: 'off',
|
|
821
|
-
migrationMode: mode,
|
|
822
|
-
pendingMigrations: [],
|
|
823
|
-
appliedMigrations: [],
|
|
824
|
-
error: { code: err.code || 'AQ_MIGRATION_PROBE_FAILED', message: err.message },
|
|
825
|
-
durationMs,
|
|
826
|
-
};
|
|
827
|
-
}
|
|
828
|
-
|
|
829
|
-
if (mode === 'off') {
|
|
830
|
-
return {
|
|
831
|
-
ready: true, memoryMode: 'rw', migrationMode: mode,
|
|
832
|
-
pendingMigrations: plan.pending, appliedMigrations: plan.applied,
|
|
833
|
-
error: null, durationMs: Date.now() - t0,
|
|
834
|
-
};
|
|
835
|
-
}
|
|
836
|
-
|
|
837
|
-
if (mode === 'check') {
|
|
838
|
-
const ready = plan.pending.length === 0;
|
|
839
|
-
if (ready) migrated = true;
|
|
840
|
-
return {
|
|
841
|
-
ready, memoryMode: ready ? 'rw' : 'ro', migrationMode: mode,
|
|
842
|
-
pendingMigrations: plan.pending, appliedMigrations: plan.applied,
|
|
843
|
-
error: null, durationMs: Date.now() - t0,
|
|
844
|
-
};
|
|
845
|
-
}
|
|
846
|
-
|
|
847
|
-
// mode === 'apply'
|
|
848
|
-
if (plan.pending.length === 0) {
|
|
849
|
-
migrated = true;
|
|
850
|
-
return {
|
|
851
|
-
ready: true, memoryMode: 'rw', migrationMode: mode,
|
|
852
|
-
pendingMigrations: [], appliedMigrations: plan.applied,
|
|
853
|
-
error: null, durationMs: Date.now() - t0,
|
|
854
|
-
};
|
|
855
|
-
}
|
|
856
|
-
|
|
857
|
-
try {
|
|
858
|
-
const result = await withDeadline(this.migrate());
|
|
859
|
-
const planAfter = await buildMigrationPlan(pool).catch(() => null);
|
|
860
|
-
return {
|
|
861
|
-
ready: true, memoryMode: 'rw', migrationMode: mode,
|
|
862
|
-
pendingMigrations: planAfter ? planAfter.pending : [],
|
|
863
|
-
appliedMigrations: planAfter ? planAfter.applied : plan.required,
|
|
864
|
-
error: null, durationMs: result.durationMs || (Date.now() - t0),
|
|
865
|
-
};
|
|
866
|
-
} catch (err) {
|
|
867
|
-
return {
|
|
868
|
-
ready: false, memoryMode: 'ro', migrationMode: mode,
|
|
869
|
-
pendingMigrations: plan.pending, appliedMigrations: plan.applied,
|
|
870
|
-
error: { code: err.code || 'AQ_MIGRATION_FAILED', message: err.message },
|
|
871
|
-
durationMs: Date.now() - t0,
|
|
872
|
-
};
|
|
873
|
-
}
|
|
874
|
-
} finally {
|
|
875
|
-
clearDeadline();
|
|
876
|
-
}
|
|
187
|
+
return migrationRuntime.init();
|
|
877
188
|
},
|
|
878
189
|
|
|
879
190
|
async close() {
|
|
@@ -895,8 +206,8 @@ function createAquifer(config = {}) {
|
|
|
895
206
|
async enableEntities() {
|
|
896
207
|
entitiesEnabled = true;
|
|
897
208
|
// M4: if already migrated, run entity DDL now
|
|
898
|
-
if (
|
|
899
|
-
const entitySql = loadSql('002-entities.sql'
|
|
209
|
+
if (migrationRuntime.isMigrated()) {
|
|
210
|
+
const entitySql = migrationRuntime.loadSql('002-entities.sql');
|
|
900
211
|
await pool.query(entitySql);
|
|
901
212
|
}
|
|
902
213
|
},
|
|
@@ -907,7 +218,7 @@ function createAquifer(config = {}) {
|
|
|
907
218
|
// Safe to call repeatedly; also safe to call before migrate() (will no-op
|
|
908
219
|
// until base schema exists, which enrich/commit will materialize).
|
|
909
220
|
await ensureMigrated();
|
|
910
|
-
const factsSql = loadSql('004-facts.sql'
|
|
221
|
+
const factsSql = migrationRuntime.loadSql('004-facts.sql');
|
|
911
222
|
await pool.query(factsSql);
|
|
912
223
|
},
|
|
913
224
|
|
|
@@ -1345,16 +656,55 @@ function createAquifer(config = {}) {
|
|
|
1345
656
|
|
|
1346
657
|
// --- read path ---
|
|
1347
658
|
|
|
1348
|
-
async
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
659
|
+
async memoryRecall(query, opts = {}) {
|
|
660
|
+
memoryServing.assertCuratedRecallOpts(opts);
|
|
661
|
+
await ensureMigrated();
|
|
662
|
+
if (typeof query !== 'string' || query.trim().length === 0) {
|
|
663
|
+
throw new Error('memory.recall(query): query must be a non-empty string');
|
|
664
|
+
}
|
|
665
|
+
const validModes = new Set(['fts', 'hybrid', 'vector']);
|
|
666
|
+
const mode = opts.mode || 'hybrid';
|
|
667
|
+
if (!validModes.has(mode)) {
|
|
668
|
+
throw new Error(`Invalid curated recall mode: "${mode}". Must be one of: fts, hybrid, vector`);
|
|
1354
669
|
}
|
|
670
|
+
let queryVec = null;
|
|
671
|
+
if (mode === 'hybrid' || mode === 'vector') {
|
|
672
|
+
if (!embedFn) {
|
|
673
|
+
if (mode === 'vector') {
|
|
674
|
+
throw new Error('curated memory_recall mode=vector requires config.embed.fn or EMBED_PROVIDER env');
|
|
675
|
+
}
|
|
676
|
+
} else {
|
|
677
|
+
const embedded = await embedFn([query]);
|
|
678
|
+
queryVec = Array.isArray(embedded) && Array.isArray(embedded[0]) ? embedded[0] : null;
|
|
679
|
+
if (!queryVec && mode === 'vector') throw new Error('embedFn returned empty vector for curated memory_recall');
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
const scopedOpts = memoryServing.withDefaultScope(opts);
|
|
683
|
+
const limit = Math.max(1, Math.min(50, scopedOpts.limit || 10));
|
|
684
|
+
const runLexical = mode === 'fts' || mode === 'hybrid';
|
|
685
|
+
const runVector = (mode === 'vector' || mode === 'hybrid') && queryVec;
|
|
686
|
+
const [lexicalRows, embeddingRows] = await Promise.all([
|
|
687
|
+
runLexical ? aquifer.memory.recall(query, {
|
|
688
|
+
...scopedOpts,
|
|
689
|
+
ftsConfig: migrationRuntime.getFtsConfig(),
|
|
690
|
+
}) : Promise.resolve([]),
|
|
691
|
+
runVector ? aquifer.memory.recallViaMemoryEmbeddings(queryVec, scopedOpts) : Promise.resolve([]),
|
|
692
|
+
]);
|
|
693
|
+
const rows = aquifer.memory.rankHybridMemoryRows(lexicalRows, embeddingRows, { limit });
|
|
694
|
+
return rows.map(memoryServing.normalizeCuratedRecallRow);
|
|
695
|
+
},
|
|
696
|
+
|
|
697
|
+
async historicalRecall(query, opts = {}) {
|
|
1355
698
|
return aquifer.evidenceRecall(query, { ...opts, allowBroadEvidence: true });
|
|
1356
699
|
},
|
|
1357
700
|
|
|
701
|
+
async recall(query, opts = {}) {
|
|
702
|
+
if (memoryServing.resolveMode(opts) === 'curated') {
|
|
703
|
+
return aquifer.memoryRecall(query, opts);
|
|
704
|
+
}
|
|
705
|
+
return aquifer.historicalRecall(query, opts);
|
|
706
|
+
},
|
|
707
|
+
|
|
1358
708
|
async evidenceRecall(query, opts = {}) {
|
|
1359
709
|
// Contract (aligned across core / manifest / consumer tools): query must
|
|
1360
710
|
// be a non-empty string. Empty strings previously short-circuited to []
|
|
@@ -1363,7 +713,7 @@ function createAquifer(config = {}) {
|
|
|
1363
713
|
if (typeof query !== 'string' || query.trim().length === 0) {
|
|
1364
714
|
throw new Error('aquifer.recall(query): query must be a non-empty string');
|
|
1365
715
|
}
|
|
1366
|
-
if (opts.allowBroadEvidence !== true && !hasEvidenceBoundary(opts)) {
|
|
716
|
+
if (opts.allowBroadEvidence !== true && !memoryServing.hasEvidenceBoundary(opts)) {
|
|
1367
717
|
throw new Error('evidence_recall requires an audit boundary filter (agentId, source, dateFrom/dateTo, host, sessionId) or allowUnsafeDebug=true');
|
|
1368
718
|
}
|
|
1369
719
|
|
|
@@ -1533,7 +883,7 @@ function createAquifer(config = {}) {
|
|
|
1533
883
|
runFts
|
|
1534
884
|
? storage.searchSessions(pool, query, {
|
|
1535
885
|
schema, tenantId, agentIds: resolvedAgentIds, source, dateFrom, dateTo, limit: fetchLimit,
|
|
1536
|
-
ftsConfig,
|
|
886
|
+
ftsConfig: migrationRuntime.getFtsConfig(),
|
|
1537
887
|
}).catch((err) => {
|
|
1538
888
|
recordSearchError('fts', err);
|
|
1539
889
|
return [];
|
|
@@ -1566,9 +916,9 @@ function createAquifer(config = {}) {
|
|
|
1566
916
|
? (rows) => rows.filter(r => candidateSessionIds.has(r.session_id || String(r.id)))
|
|
1567
917
|
: (rows) => rows;
|
|
1568
918
|
|
|
1569
|
-
const filteredFts = filterFn(ftsRows);
|
|
1570
|
-
const filteredEmb = filterFn(embRows);
|
|
1571
|
-
const filteredTurn = filterFn(turnRows);
|
|
919
|
+
const filteredFts = filterPublicPlaceholderSessionRows(filterFn(ftsRows));
|
|
920
|
+
const filteredEmb = filterPublicPlaceholderSessionRows(filterFn(embRows));
|
|
921
|
+
const filteredTurn = filterPublicPlaceholderSessionRows(filterFn(turnRows));
|
|
1572
922
|
|
|
1573
923
|
if (filteredFts.length === 0 && filteredEmb.length === 0 && filteredTurn.length === 0) {
|
|
1574
924
|
maybeThrowSearchErrors();
|
|
@@ -1613,10 +963,11 @@ function createAquifer(config = {}) {
|
|
|
1613
963
|
if (externalPromises.length > 0) await Promise.all(externalPromises);
|
|
1614
964
|
|
|
1615
965
|
// 6. Hybrid rank
|
|
966
|
+
const filteredExternalRows = filterPublicPlaceholderSessionRows(filterFn(externalRows));
|
|
1616
967
|
const mergedWeights = { ...rankWeights, ...overrideWeights };
|
|
1617
968
|
const ranked = hybridRank(
|
|
1618
969
|
filteredFts,
|
|
1619
|
-
[...filteredEmb, ...
|
|
970
|
+
[...filteredEmb, ...filteredExternalRows],
|
|
1620
971
|
filteredTurn,
|
|
1621
972
|
{
|
|
1622
973
|
limit: rerankTopK,
|
|
@@ -1673,11 +1024,11 @@ function createAquifer(config = {}) {
|
|
|
1673
1024
|
if (aR !== bR) return bR - aR;
|
|
1674
1025
|
return (b._hybridScore || 0) - (a._hybridScore || 0);
|
|
1675
1026
|
});
|
|
1676
|
-
finalRanked = finalRanked.slice(0, limit);
|
|
1027
|
+
finalRanked = filterPublicPlaceholderSessionRows(finalRanked).slice(0, limit);
|
|
1677
1028
|
} catch (rerankErr) {
|
|
1678
1029
|
// Fallback: use original hybrid-rank order, flag in debug
|
|
1679
1030
|
if (process.env.AQUIFER_DEBUG) console.error('[aquifer] rerank error:', rerankErr.message);
|
|
1680
|
-
finalRanked = ranked.slice(0, limit).map(r => ({
|
|
1031
|
+
finalRanked = filterPublicPlaceholderSessionRows(ranked).slice(0, limit).map(r => ({
|
|
1681
1032
|
...r,
|
|
1682
1033
|
_rerankFallback: true,
|
|
1683
1034
|
_rerankReason: rerankDecision.reason,
|
|
@@ -1685,7 +1036,7 @@ function createAquifer(config = {}) {
|
|
|
1685
1036
|
}));
|
|
1686
1037
|
}
|
|
1687
1038
|
} else {
|
|
1688
|
-
finalRanked = ranked.slice(0, limit).map(r => ({ ...r, _rerankReason: rerankDecision.reason }));
|
|
1039
|
+
finalRanked = filterPublicPlaceholderSessionRows(ranked).slice(0, limit).map(r => ({ ...r, _rerankReason: rerankDecision.reason }));
|
|
1689
1040
|
}
|
|
1690
1041
|
|
|
1691
1042
|
// 7. Record access
|
|
@@ -1838,7 +1189,20 @@ function createAquifer(config = {}) {
|
|
|
1838
1189
|
// --- public config accessor ---
|
|
1839
1190
|
|
|
1840
1191
|
getConfig() {
|
|
1841
|
-
return {
|
|
1192
|
+
return {
|
|
1193
|
+
schema,
|
|
1194
|
+
tenantId,
|
|
1195
|
+
memoryServingMode: memoryServing.servingMode,
|
|
1196
|
+
memoryActiveScopeKey: memoryServing.defaultActiveScopeKey,
|
|
1197
|
+
memoryActiveScopePath: memoryServing.defaultActiveScopePath,
|
|
1198
|
+
backendKind,
|
|
1199
|
+
backendProfile: backendInfo.profile,
|
|
1200
|
+
capabilities: backendInfo.capabilities,
|
|
1201
|
+
};
|
|
1202
|
+
},
|
|
1203
|
+
|
|
1204
|
+
getCapabilities() {
|
|
1205
|
+
return backendCapabilities(backendKind);
|
|
1842
1206
|
},
|
|
1843
1207
|
|
|
1844
1208
|
// v1.2.0: expose the internal pool so host persona layers can reuse it
|
|
@@ -1894,12 +1258,92 @@ function createAquifer(config = {}) {
|
|
|
1894
1258
|
entityCount = entResult.rows[0]?.count || 0;
|
|
1895
1259
|
} catch { /* entities table may not exist */ }
|
|
1896
1260
|
|
|
1261
|
+
let memoryRecords = {
|
|
1262
|
+
available: false,
|
|
1263
|
+
total: 0,
|
|
1264
|
+
active: 0,
|
|
1265
|
+
visibleInBootstrap: 0,
|
|
1266
|
+
visibleInRecall: 0,
|
|
1267
|
+
earliest: null,
|
|
1268
|
+
latest: null,
|
|
1269
|
+
};
|
|
1270
|
+
try {
|
|
1271
|
+
const memoryResult = await pool.query(
|
|
1272
|
+
`SELECT
|
|
1273
|
+
COUNT(*)::int AS total,
|
|
1274
|
+
COUNT(*) FILTER (WHERE status = 'active')::int AS active,
|
|
1275
|
+
COUNT(*) FILTER (WHERE status = 'active' AND visible_in_bootstrap = true)::int AS visible_in_bootstrap,
|
|
1276
|
+
COUNT(*) FILTER (WHERE status = 'active' AND visible_in_recall = true)::int AS visible_in_recall,
|
|
1277
|
+
MIN(accepted_at) AS earliest,
|
|
1278
|
+
MAX(accepted_at) AS latest
|
|
1279
|
+
FROM ${qi(schema)}.memory_records
|
|
1280
|
+
WHERE tenant_id = $1`,
|
|
1281
|
+
[tenantId]
|
|
1282
|
+
);
|
|
1283
|
+
const row = memoryResult.rows[0] || {};
|
|
1284
|
+
memoryRecords = {
|
|
1285
|
+
available: true,
|
|
1286
|
+
total: row.total || 0,
|
|
1287
|
+
active: row.active || 0,
|
|
1288
|
+
visibleInBootstrap: row.visible_in_bootstrap || 0,
|
|
1289
|
+
visibleInRecall: row.visible_in_recall || 0,
|
|
1290
|
+
earliest: row.earliest || null,
|
|
1291
|
+
latest: row.latest || null,
|
|
1292
|
+
};
|
|
1293
|
+
} catch { /* memory_records table may not exist on older installs */ }
|
|
1294
|
+
|
|
1295
|
+
let sessionFinalizations = {
|
|
1296
|
+
available: false,
|
|
1297
|
+
total: 0,
|
|
1298
|
+
statuses: {},
|
|
1299
|
+
latestFinalizedAt: null,
|
|
1300
|
+
latestUpdatedAt: null,
|
|
1301
|
+
};
|
|
1302
|
+
try {
|
|
1303
|
+
const finalizationResult = await pool.query(
|
|
1304
|
+
`SELECT
|
|
1305
|
+
status,
|
|
1306
|
+
COUNT(*)::int AS count,
|
|
1307
|
+
MAX(finalized_at) AS latest_finalized_at,
|
|
1308
|
+
MAX(updated_at) AS latest_updated_at
|
|
1309
|
+
FROM ${qi(schema)}.session_finalizations
|
|
1310
|
+
WHERE tenant_id = $1
|
|
1311
|
+
GROUP BY status`,
|
|
1312
|
+
[tenantId]
|
|
1313
|
+
);
|
|
1314
|
+
const statuses = Object.fromEntries(finalizationResult.rows.map(row => [row.status, row.count]));
|
|
1315
|
+
sessionFinalizations = {
|
|
1316
|
+
available: true,
|
|
1317
|
+
total: finalizationResult.rows.reduce((sum, row) => sum + row.count, 0),
|
|
1318
|
+
statuses,
|
|
1319
|
+
latestFinalizedAt: finalizationResult.rows
|
|
1320
|
+
.map(row => row.latest_finalized_at)
|
|
1321
|
+
.filter(Boolean)
|
|
1322
|
+
.sort()
|
|
1323
|
+
.pop() || null,
|
|
1324
|
+
latestUpdatedAt: finalizationResult.rows
|
|
1325
|
+
.map(row => row.latest_updated_at)
|
|
1326
|
+
.filter(Boolean)
|
|
1327
|
+
.sort()
|
|
1328
|
+
.pop() || null,
|
|
1329
|
+
};
|
|
1330
|
+
} catch { /* session_finalizations table may not exist on older installs */ }
|
|
1331
|
+
|
|
1897
1332
|
return {
|
|
1333
|
+
backendKind,
|
|
1334
|
+
backendProfile: backendInfo.profile,
|
|
1335
|
+
serving: {
|
|
1336
|
+
mode: memoryServing.servingMode,
|
|
1337
|
+
activeScopeKey: memoryServing.defaultActiveScopeKey,
|
|
1338
|
+
activeScopePath: memoryServing.defaultActiveScopePath,
|
|
1339
|
+
},
|
|
1898
1340
|
sessions: Object.fromEntries(sessions.rows.map(r => [r.processing_status, r.count])),
|
|
1899
1341
|
sessionTotal: sessions.rows.reduce((s, r) => s + r.count, 0),
|
|
1900
1342
|
summaries: summaries.rows[0]?.count || 0,
|
|
1901
1343
|
turnEmbeddings: turns.rows[0]?.count || 0,
|
|
1902
1344
|
entities: entityCount,
|
|
1345
|
+
memoryRecords,
|
|
1346
|
+
sessionFinalizations,
|
|
1903
1347
|
earliest: timeRange.rows[0]?.earliest || null,
|
|
1904
1348
|
latest: timeRange.rows[0]?.latest || null,
|
|
1905
1349
|
};
|
|
@@ -1941,113 +1385,23 @@ function createAquifer(config = {}) {
|
|
|
1941
1385
|
return result.rows;
|
|
1942
1386
|
},
|
|
1943
1387
|
|
|
1944
|
-
async
|
|
1388
|
+
async memoryBootstrap(opts = {}) {
|
|
1945
1389
|
await ensureMigrated();
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
}
|
|
1950
|
-
|
|
1951
|
-
const agentId = opts.agentId || null;
|
|
1952
|
-
const source = opts.source || null;
|
|
1953
|
-
const limit = Math.max(1, Math.min(20, opts.limit || 5));
|
|
1954
|
-
const lookbackDays = opts.lookbackDays || 14;
|
|
1955
|
-
const maxChars = opts.maxChars || 4000;
|
|
1956
|
-
const format = opts.format || 'structured';
|
|
1957
|
-
|
|
1958
|
-
// 'partial' sessions have a summary but recorded warnings during enrich;
|
|
1959
|
-
// they are user-visible content, not in-progress — bootstrap must include
|
|
1960
|
-
// them alongside 'succeeded'. 'pending' / 'processing' have no summary
|
|
1961
|
-
// yet and are correctly excluded.
|
|
1962
|
-
const where = [`s.tenant_id = $1`, `s.processing_status IN ('succeeded', 'partial')`];
|
|
1963
|
-
const params = [tenantId];
|
|
1964
|
-
|
|
1965
|
-
if (agentId) {
|
|
1966
|
-
params.push(agentId);
|
|
1967
|
-
where.push(`s.agent_id = $${params.length}`);
|
|
1968
|
-
}
|
|
1969
|
-
if (source) {
|
|
1970
|
-
params.push(source);
|
|
1971
|
-
where.push(`s.source = $${params.length}`);
|
|
1972
|
-
}
|
|
1973
|
-
|
|
1974
|
-
params.push(lookbackDays);
|
|
1975
|
-
// upsertSession sets ended_at on every commit but started_at / last_message_at
|
|
1976
|
-
// only when the caller supplies them — fall back through both so sessions
|
|
1977
|
-
// committed without explicit timestamps remain reachable.
|
|
1978
|
-
where.push(`COALESCE(s.last_message_at, s.ended_at, s.started_at) > now() - ($${params.length} || ' days')::interval`);
|
|
1979
|
-
|
|
1980
|
-
params.push(limit);
|
|
1981
|
-
|
|
1982
|
-
const result = await pool.query(
|
|
1983
|
-
`SELECT s.session_id, s.agent_id, s.source, s.started_at, s.msg_count,
|
|
1984
|
-
ss.summary_text, ss.structured_summary
|
|
1985
|
-
FROM ${qi(schema)}.sessions s
|
|
1986
|
-
JOIN ${qi(schema)}.session_summaries ss ON ss.session_row_id = s.id
|
|
1987
|
-
WHERE ${where.join(' AND ')}
|
|
1988
|
-
ORDER BY COALESCE(s.last_message_at, s.ended_at, s.started_at) DESC
|
|
1989
|
-
LIMIT $${params.length}`,
|
|
1990
|
-
params
|
|
1991
|
-
);
|
|
1992
|
-
|
|
1993
|
-
const sessions = result.rows.map(r => {
|
|
1994
|
-
const ss = r.structured_summary || {};
|
|
1995
|
-
const hasSS = ss.title || ss.overview;
|
|
1996
|
-
return {
|
|
1997
|
-
sessionId: r.session_id,
|
|
1998
|
-
agentId: r.agent_id,
|
|
1999
|
-
source: r.source,
|
|
2000
|
-
startedAt: r.started_at,
|
|
2001
|
-
title: ss.title || (hasSS ? null : (r.summary_text || '').slice(0, 60).trim() || null),
|
|
2002
|
-
overview: ss.overview || (hasSS ? null : (r.summary_text || '').slice(0, 200).trim() || null),
|
|
2003
|
-
topics: Array.isArray(ss.topics) ? ss.topics : [],
|
|
2004
|
-
decisions: Array.isArray(ss.decisions) ? ss.decisions : [],
|
|
2005
|
-
openLoops: Array.isArray(ss.open_loops) ? ss.open_loops : [],
|
|
2006
|
-
importantFacts: Array.isArray(ss.important_facts) ? ss.important_facts : [],
|
|
2007
|
-
};
|
|
2008
|
-
});
|
|
2009
|
-
|
|
2010
|
-
// Cross-session open loops merge + dedup + sentinel filter
|
|
2011
|
-
const SENTINELS = new Set(['無', 'none', 'n/a', 'na', 'done', '']);
|
|
2012
|
-
const seenLoops = new Set();
|
|
2013
|
-
const openLoops = [];
|
|
2014
|
-
for (const s of sessions) {
|
|
2015
|
-
for (const loop of s.openLoops) {
|
|
2016
|
-
const raw = typeof loop === 'string' ? loop : (loop.item || '');
|
|
2017
|
-
const normalized = raw.trim().replace(/\s+/g, ' ').toLowerCase();
|
|
2018
|
-
if (SENTINELS.has(normalized) || !normalized || seenLoops.has(normalized)) continue;
|
|
2019
|
-
seenLoops.add(normalized);
|
|
2020
|
-
openLoops.push({ item: raw.trim(), fromSession: s.sessionId, latestStartedAt: s.startedAt });
|
|
2021
|
-
}
|
|
2022
|
-
}
|
|
2023
|
-
|
|
2024
|
-
// Cross-session recent decisions dedup
|
|
2025
|
-
const seenDecisions = new Set();
|
|
2026
|
-
const recentDecisions = [];
|
|
2027
|
-
for (const s of sessions) {
|
|
2028
|
-
for (const d of s.decisions) {
|
|
2029
|
-
const key = typeof d === 'string' ? d : (d.decision || '');
|
|
2030
|
-
const normalized = key.trim().replace(/\s+/g, ' ').toLowerCase();
|
|
2031
|
-
if (!normalized || seenDecisions.has(normalized)) continue;
|
|
2032
|
-
seenDecisions.add(normalized);
|
|
2033
|
-
recentDecisions.push({ decision: key.trim(), reason: d.reason || null, fromSession: s.sessionId });
|
|
2034
|
-
}
|
|
2035
|
-
}
|
|
1390
|
+
memoryServing.assertCuratedBootstrapOpts(opts);
|
|
1391
|
+
return aquifer.memory.bootstrap(memoryServing.withDefaultScope(opts));
|
|
1392
|
+
},
|
|
2036
1393
|
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
meta: { lookbackDays, count: sessions.length, maxChars, truncated: false },
|
|
2042
|
-
};
|
|
1394
|
+
async historicalBootstrap(opts = {}) {
|
|
1395
|
+
await ensureMigrated();
|
|
1396
|
+
return legacyBootstrap(opts);
|
|
1397
|
+
},
|
|
2043
1398
|
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
structured.meta.truncated = textResult.truncated;
|
|
1399
|
+
async bootstrap(opts = {}) {
|
|
1400
|
+
if (memoryServing.resolveMode(opts) === 'curated') {
|
|
1401
|
+
return aquifer.memoryBootstrap(opts);
|
|
2048
1402
|
}
|
|
2049
1403
|
|
|
2050
|
-
return
|
|
1404
|
+
return aquifer.historicalBootstrap(opts);
|
|
2051
1405
|
},
|
|
2052
1406
|
};
|
|
2053
1407
|
|
|
@@ -2066,11 +1420,12 @@ function createAquifer(config = {}) {
|
|
|
2066
1420
|
const { createEntityState } = require('./entity-state');
|
|
2067
1421
|
const { createInsights } = require('./insights');
|
|
2068
1422
|
const { createMemoryRecords } = require('./memory-records');
|
|
2069
|
-
const { createMemoryPromotion } = require('./memory-promotion');
|
|
1423
|
+
const { createMemoryPromotion, buildMemoryEmbeddingText } = require('./memory-promotion');
|
|
2070
1424
|
const { createMemoryBootstrap } = require('./memory-bootstrap');
|
|
2071
1425
|
const { createMemoryRecall } = require('./memory-recall');
|
|
2072
1426
|
const { createMemoryConsolidation } = require('./memory-consolidation');
|
|
2073
1427
|
const { createSessionFinalization } = require('./session-finalization');
|
|
1428
|
+
const { createSessionCheckpoints } = require('./session-checkpoints');
|
|
2074
1429
|
const qSchema = qi(schema);
|
|
2075
1430
|
aquifer.narratives = createNarratives({ pool, schema: qSchema, defaultTenantId: tenantId });
|
|
2076
1431
|
aquifer.timeline = createTimeline({ pool, schema: qSchema, defaultTenantId: tenantId });
|
|
@@ -2100,7 +1455,7 @@ function createAquifer(config = {}) {
|
|
|
2100
1455
|
});
|
|
2101
1456
|
|
|
2102
1457
|
const memoryRecords = createMemoryRecords({ pool, schema: qSchema, defaultTenantId: tenantId });
|
|
2103
|
-
const memoryPromotion = createMemoryPromotion({ records: memoryRecords });
|
|
1458
|
+
const memoryPromotion = createMemoryPromotion({ records: memoryRecords, embedFn });
|
|
2104
1459
|
const memoryBootstrap = createMemoryBootstrap({ records: memoryRecords });
|
|
2105
1460
|
const memoryRecall = createMemoryRecall({ pool, schema: qSchema, defaultTenantId: tenantId });
|
|
2106
1461
|
const memoryConsolidation = createMemoryConsolidation({
|
|
@@ -2114,8 +1469,22 @@ function createAquifer(config = {}) {
|
|
|
2114
1469
|
schema,
|
|
2115
1470
|
recordsSchema: qSchema,
|
|
2116
1471
|
defaultTenantId: tenantId,
|
|
1472
|
+
embedFn,
|
|
1473
|
+
});
|
|
1474
|
+
const sessionCheckpoints = createSessionCheckpoints({
|
|
1475
|
+
pool,
|
|
1476
|
+
schema,
|
|
1477
|
+
defaultTenantId: tenantId,
|
|
2117
1478
|
});
|
|
2118
1479
|
|
|
1480
|
+
function currentMemoryScopeKeys(opts = {}) {
|
|
1481
|
+
if (Array.isArray(opts.activeScopePath) && opts.activeScopePath.length > 0) {
|
|
1482
|
+
return opts.activeScopePath.map(value => String(value)).filter(Boolean);
|
|
1483
|
+
}
|
|
1484
|
+
if (opts.activeScopeKey) return [String(opts.activeScopeKey)];
|
|
1485
|
+
return null;
|
|
1486
|
+
}
|
|
1487
|
+
|
|
2119
1488
|
// v1 curated-memory sidecar. Top-level recall/bootstrap can opt into this
|
|
2120
1489
|
// plane through memory.servingMode while legacy/evidence mode remains
|
|
2121
1490
|
// available for compatibility and debugging.
|
|
@@ -2132,6 +1501,10 @@ function createAquifer(config = {}) {
|
|
|
2132
1501
|
await ensureMigrated();
|
|
2133
1502
|
return memoryRecords.upsertMemory(input);
|
|
2134
1503
|
},
|
|
1504
|
+
upsertEvidenceItem: async (input = {}) => {
|
|
1505
|
+
await ensureMigrated();
|
|
1506
|
+
return memoryRecords.upsertEvidenceItem(input);
|
|
1507
|
+
},
|
|
2135
1508
|
linkEvidence: async (input = {}) => {
|
|
2136
1509
|
await ensureMigrated();
|
|
2137
1510
|
return memoryRecords.linkEvidence(input);
|
|
@@ -2152,16 +1525,110 @@ function createAquifer(config = {}) {
|
|
|
2152
1525
|
},
|
|
2153
1526
|
current: async (opts = {}) => {
|
|
2154
1527
|
await ensureMigrated();
|
|
2155
|
-
return memoryRecords.currentProjection(
|
|
1528
|
+
return memoryRecords.currentProjection(memoryServing.withDefaultScope(opts));
|
|
2156
1529
|
},
|
|
2157
1530
|
listCurrentMemory: async (opts = {}) => {
|
|
2158
1531
|
await ensureMigrated();
|
|
2159
|
-
return memoryRecords.currentProjection(
|
|
1532
|
+
return memoryRecords.currentProjection(memoryServing.withDefaultScope(opts));
|
|
1533
|
+
},
|
|
1534
|
+
backfillEmbeddings: async (opts = {}) => {
|
|
1535
|
+
await ensureMigrated();
|
|
1536
|
+
requireEmbed('memory.backfillEmbeddings');
|
|
1537
|
+
const scopedOpts = memoryServing.withDefaultScope(opts);
|
|
1538
|
+
const listInput = {
|
|
1539
|
+
tenantId: scopedOpts.tenantId || tenantId,
|
|
1540
|
+
asOf: scopedOpts.asOf,
|
|
1541
|
+
scopeId: scopedOpts.scopeId,
|
|
1542
|
+
scopeKeys: currentMemoryScopeKeys(scopedOpts),
|
|
1543
|
+
withoutEmbedding: true,
|
|
1544
|
+
limit: Math.max(1, Math.min(200, scopedOpts.limit || 50)),
|
|
1545
|
+
};
|
|
1546
|
+
if (scopedOpts.visibleInRecall !== undefined) {
|
|
1547
|
+
listInput.visibleInRecall = scopedOpts.visibleInRecall;
|
|
1548
|
+
} else if (scopedOpts.visibleInBootstrap === undefined) {
|
|
1549
|
+
listInput.visibleInRecall = true;
|
|
1550
|
+
}
|
|
1551
|
+
if (scopedOpts.visibleInBootstrap !== undefined) {
|
|
1552
|
+
listInput.visibleInBootstrap = scopedOpts.visibleInBootstrap;
|
|
1553
|
+
}
|
|
1554
|
+
const sourceRows = await memoryRecords.listActive(listInput);
|
|
1555
|
+
const rowsToEmbed = [];
|
|
1556
|
+
const texts = [];
|
|
1557
|
+
for (const row of sourceRows) {
|
|
1558
|
+
const text = buildMemoryEmbeddingText(row);
|
|
1559
|
+
if (!text) continue;
|
|
1560
|
+
rowsToEmbed.push(row);
|
|
1561
|
+
texts.push(text);
|
|
1562
|
+
}
|
|
1563
|
+
if (rowsToEmbed.length === 0) {
|
|
1564
|
+
return {
|
|
1565
|
+
scanned: sourceRows.length,
|
|
1566
|
+
embedded: 0,
|
|
1567
|
+
skipped: sourceRows.length,
|
|
1568
|
+
memories: [],
|
|
1569
|
+
};
|
|
1570
|
+
}
|
|
1571
|
+
const vectors = await embedFn(texts);
|
|
1572
|
+
if (!Array.isArray(vectors) || vectors.length !== texts.length) {
|
|
1573
|
+
throw new Error(`memory.backfillEmbeddings embedFn returned ${Array.isArray(vectors) ? vectors.length : 'invalid'} vectors for ${texts.length} memory rows`);
|
|
1574
|
+
}
|
|
1575
|
+
const updatedRows = [];
|
|
1576
|
+
let skipped = sourceRows.length - rowsToEmbed.length;
|
|
1577
|
+
const skippedMemories = [];
|
|
1578
|
+
for (let i = 0; i < rowsToEmbed.length; i++) {
|
|
1579
|
+
const vector = vectors[i];
|
|
1580
|
+
if (!Array.isArray(vector) || vector.length === 0) {
|
|
1581
|
+
skipped += 1;
|
|
1582
|
+
skippedMemories.push({
|
|
1583
|
+
memoryId: String(rowsToEmbed[i].id),
|
|
1584
|
+
canonicalKey: rowsToEmbed[i].canonical_key || rowsToEmbed[i].canonicalKey || null,
|
|
1585
|
+
reason: 'empty_vector',
|
|
1586
|
+
});
|
|
1587
|
+
continue;
|
|
1588
|
+
}
|
|
1589
|
+
const updateResult = await memoryRecords.updateMemoryEmbedding({
|
|
1590
|
+
tenantId: scopedOpts.tenantId || tenantId,
|
|
1591
|
+
memoryId: rowsToEmbed[i].id,
|
|
1592
|
+
embedding: vector,
|
|
1593
|
+
});
|
|
1594
|
+
if (updateResult && updateResult.updated && updateResult.memory) {
|
|
1595
|
+
updatedRows.push(updateResult.memory);
|
|
1596
|
+
continue;
|
|
1597
|
+
}
|
|
1598
|
+
skipped += 1;
|
|
1599
|
+
skippedMemories.push({
|
|
1600
|
+
memoryId: String(rowsToEmbed[i].id),
|
|
1601
|
+
canonicalKey: rowsToEmbed[i].canonical_key || rowsToEmbed[i].canonicalKey || null,
|
|
1602
|
+
reason: updateResult && updateResult.status ? updateResult.status : 'not_updated',
|
|
1603
|
+
});
|
|
1604
|
+
}
|
|
1605
|
+
return {
|
|
1606
|
+
scanned: sourceRows.length,
|
|
1607
|
+
embedded: updatedRows.length,
|
|
1608
|
+
skipped,
|
|
1609
|
+
memories: updatedRows.map(memoryRecords.normalizeCurrentMemoryRow),
|
|
1610
|
+
skippedMemories,
|
|
1611
|
+
};
|
|
2160
1612
|
},
|
|
2161
1613
|
recall: async (query, opts = {}) => {
|
|
2162
1614
|
await ensureMigrated();
|
|
2163
1615
|
return memoryRecall.recall(query, opts);
|
|
2164
1616
|
},
|
|
1617
|
+
recallViaEvidenceItems: async (query, opts = {}) => {
|
|
1618
|
+
await ensureMigrated();
|
|
1619
|
+
return memoryRecall.recallViaEvidenceItems(query, opts);
|
|
1620
|
+
},
|
|
1621
|
+
recallViaMemoryEmbeddings: async (queryVec, opts = {}) => {
|
|
1622
|
+
await ensureMigrated();
|
|
1623
|
+
return memoryRecall.recallViaMemoryEmbeddings(queryVec, opts);
|
|
1624
|
+
},
|
|
1625
|
+
recallViaLinkedSummaryEmbeddings: async (queryVec, opts = {}) => {
|
|
1626
|
+
await ensureMigrated();
|
|
1627
|
+
return memoryRecall.recallViaLinkedSummaryEmbeddings(queryVec, opts);
|
|
1628
|
+
},
|
|
1629
|
+
rankHybridMemoryRows: (lexicalRows, embeddingRows, opts = {}) => {
|
|
1630
|
+
return memoryRecall.rankHybridMemoryRows(lexicalRows, embeddingRows, opts);
|
|
1631
|
+
},
|
|
2165
1632
|
consolidation: {
|
|
2166
1633
|
plan: memoryConsolidation.plan,
|
|
2167
1634
|
distillArchiveSnapshot: memoryConsolidation.distillArchiveSnapshot,
|
|
@@ -2211,6 +1678,50 @@ function createAquifer(config = {}) {
|
|
|
2211
1678
|
},
|
|
2212
1679
|
};
|
|
2213
1680
|
|
|
1681
|
+
aquifer.checkpoints = {
|
|
1682
|
+
upsertRun: async (input = {}) => {
|
|
1683
|
+
await ensureMigrated();
|
|
1684
|
+
return sessionCheckpoints.upsertRun(input);
|
|
1685
|
+
},
|
|
1686
|
+
updateRunStatus: async (input = {}) => {
|
|
1687
|
+
await ensureMigrated();
|
|
1688
|
+
return sessionCheckpoints.updateRunStatus(input);
|
|
1689
|
+
},
|
|
1690
|
+
listRuns: async (input = {}) => {
|
|
1691
|
+
await ensureMigrated();
|
|
1692
|
+
return sessionCheckpoints.listRuns(input);
|
|
1693
|
+
},
|
|
1694
|
+
upsertSources: async (rows = [], input = {}) => {
|
|
1695
|
+
await ensureMigrated();
|
|
1696
|
+
return sessionCheckpoints.upsertSources(rows, input);
|
|
1697
|
+
},
|
|
1698
|
+
listSources: async (input = {}) => {
|
|
1699
|
+
await ensureMigrated();
|
|
1700
|
+
return sessionCheckpoints.listSources(input);
|
|
1701
|
+
},
|
|
1702
|
+
buildSynthesisInput: (input = {}) => sessionCheckpoints.buildSynthesisInput(input),
|
|
1703
|
+
buildSynthesisPrompt: (input = {}, opts = {}) => sessionCheckpoints.buildSynthesisPrompt(input, opts),
|
|
1704
|
+
buildRunInputFromSynthesis: (input = {}, summary = {}, opts = {}) => (
|
|
1705
|
+
sessionCheckpoints.buildRunInputFromSynthesis(input, summary, opts)
|
|
1706
|
+
),
|
|
1707
|
+
planFromFinalizations: async (input = {}) => {
|
|
1708
|
+
await ensureMigrated();
|
|
1709
|
+
return sessionCheckpoints.planFromFinalizations(input);
|
|
1710
|
+
},
|
|
1711
|
+
runProducer: async (input = {}) => {
|
|
1712
|
+
await ensureMigrated();
|
|
1713
|
+
return sessionCheckpoints.runProducer(input);
|
|
1714
|
+
},
|
|
1715
|
+
listForHandoff: async (input = {}) => {
|
|
1716
|
+
await ensureMigrated();
|
|
1717
|
+
return sessionCheckpoints.listForHandoff(input);
|
|
1718
|
+
},
|
|
1719
|
+
listAcceptedForHandoff: async (input = {}) => {
|
|
1720
|
+
await ensureMigrated();
|
|
1721
|
+
return sessionCheckpoints.listAcceptedForHandoff(input);
|
|
1722
|
+
},
|
|
1723
|
+
};
|
|
1724
|
+
|
|
2214
1725
|
aquifer.finalizeSession = aquifer.finalization.finalizeSession;
|
|
2215
1726
|
|
|
2216
1727
|
return aquifer;
|