@shadowforge0/aquifer-memory 1.7.0 → 1.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/.env.example +8 -0
  2. package/README.md +66 -0
  3. package/aquifer.config.example.json +19 -0
  4. package/consumers/cli.js +192 -12
  5. package/consumers/codex-active-checkpoint.js +186 -0
  6. package/consumers/codex-current-memory.js +106 -0
  7. package/consumers/codex-handoff.js +442 -3
  8. package/consumers/codex.js +164 -107
  9. package/consumers/mcp.js +144 -6
  10. package/consumers/shared/config.js +60 -1
  11. package/consumers/shared/factory.js +10 -3
  12. package/core/aquifer.js +351 -840
  13. package/core/backends/capabilities.js +89 -0
  14. package/core/backends/local.js +430 -0
  15. package/core/legacy-bootstrap.js +140 -0
  16. package/core/mcp-manifest.js +66 -2
  17. package/core/memory-promotion.js +157 -26
  18. package/core/memory-recall.js +341 -22
  19. package/core/memory-records.js +128 -8
  20. package/core/memory-serving.js +132 -0
  21. package/core/postgres-migrations.js +533 -0
  22. package/core/public-session-filter.js +40 -0
  23. package/core/recall-runtime.js +115 -0
  24. package/core/scope-attribution.js +279 -0
  25. package/core/session-checkpoint-producer.js +412 -0
  26. package/core/session-checkpoints.js +432 -0
  27. package/core/session-finalization.js +82 -1
  28. package/core/storage-checkpoints.js +546 -0
  29. package/core/storage.js +121 -8
  30. package/docs/setup.md +22 -0
  31. package/package.json +8 -4
  32. package/schema/014-v1-checkpoint-runs.sql +349 -0
  33. package/schema/015-v1-evidence-items.sql +92 -0
  34. package/schema/016-v1-evidence-ref-multi-item.sql +19 -0
  35. package/schema/017-v1-memory-record-embeddings.sql +25 -0
  36. package/schema/018-v1-finalization-candidate-envelope.sql +39 -0
  37. package/scripts/codex-checkpoint-commands.js +464 -0
  38. package/scripts/codex-checkpoint-runtime.js +520 -0
  39. package/scripts/codex-recovery.js +105 -0
package/core/aquifer.js CHANGED
@@ -1,171 +1,31 @@
1
1
  'use strict';
2
2
 
3
3
  const { Pool } = require('pg');
4
- const path = require('path');
5
- const fs = require('fs');
6
4
 
7
5
  const storage = require('./storage');
8
6
  const entity = require('./entity');
9
7
  const { hybridRank } = require('./hybrid-rank');
10
8
  const { summarize } = require('../pipeline/summarize');
11
9
  const { extractEntities } = require('../pipeline/extract-entities');
12
- const { createEmbedder } = require('../pipeline/embed');
13
10
  const { applyEnrichSafetyGate, sanitizeSummaryResult } = require('./memory-safety-gate');
14
-
15
- // ---------------------------------------------------------------------------
16
- // Schema name validation
17
- // ---------------------------------------------------------------------------
18
-
19
- const SCHEMA_RE = /^[a-zA-Z_]\w{0,62}$/;
20
-
21
- function validateSchema(schema) {
22
- if (!SCHEMA_RE.test(schema)) {
23
- throw new Error(`Invalid schema name: "${schema}". Must match /^[a-zA-Z_]\\w{0,62}$/`);
24
- }
25
- }
26
-
27
- // C1 fix: quote identifiers to handle reserved words safely
28
- function qi(identifier) { return `"${identifier}"`; }
29
-
30
- // ---------------------------------------------------------------------------
31
- // SQL file loader — replaces ${schema} placeholders
32
- // ---------------------------------------------------------------------------
33
-
34
- function loadSql(filename, schema) {
35
- const filePath = path.join(__dirname, '..', 'schema', filename);
36
- const raw = fs.readFileSync(filePath, 'utf8');
37
- // C1: use quoted identifier for safety
38
- return raw.replace(/\$\{schema\}/g, qi(schema));
39
- }
40
-
41
- // ---------------------------------------------------------------------------
42
- // buildRerankDocument — assemble text for cross-encoder reranking
43
- // ---------------------------------------------------------------------------
44
-
45
- function buildRerankDocument(row, maxChars) {
46
- // Prefer structured_summary fields when available — title/overview carry
47
- // more signal than summary_text for short Chinese recaps, and topics /
48
- // decisions / open_loops give the cross-encoder substantive content.
49
- // Fall back to summary_text / matched_turn_text when structured is absent.
50
- const ss = row.structured_summary || null;
51
- const parts = [];
52
- if (ss) {
53
- if (ss.title) parts.push(String(ss.title).trim());
54
- if (ss.overview) parts.push(String(ss.overview).trim());
55
- if (Array.isArray(ss.topics)) {
56
- const topics = ss.topics
57
- .map(t => typeof t === 'string' ? t : (t && t.name ? `${t.name}${t.summary ? ': ' + t.summary : ''}` : ''))
58
- .filter(Boolean).join(' / ');
59
- if (topics) parts.push(topics);
60
- }
61
- if (Array.isArray(ss.decisions)) {
62
- const decisions = ss.decisions
63
- .map(d => typeof d === 'string' ? d : (d && d.decision ? d.decision : ''))
64
- .filter(Boolean).join(' / ');
65
- if (decisions) parts.push(`Decisions: ${decisions}`);
66
- }
67
- if (Array.isArray(ss.open_loops)) {
68
- const loops = ss.open_loops
69
- .map(l => typeof l === 'string' ? l : (l && l.item ? l.item : ''))
70
- .filter(Boolean).join(' / ');
71
- if (loops) parts.push(`Open loops: ${loops}`);
72
- }
73
- }
74
- if (!parts.length) {
75
- const bare = (row.summary_text || row.summary_snippet || '').trim();
76
- if (bare) parts.push(bare);
77
- }
78
- const turn = (row.matched_turn_text || '').replace(/\s+/g, ' ').trim();
79
- if (turn) {
80
- const joined = parts.join(' \n ');
81
- if (!joined.includes(turn)) parts.push(`Matched turn: ${turn}`);
82
- }
83
-
84
- let text = parts.join('\n\n').replace(/[ \t]+/g, ' ').trim();
85
- if (text.length > maxChars) text = text.slice(0, maxChars);
86
- return text;
87
- }
88
-
89
- // ---------------------------------------------------------------------------
90
- // resolveEmbedFn — v1.2.0 embed autodetect (explicit > object > env > null)
91
- // ---------------------------------------------------------------------------
92
-
93
- function resolveEmbedFn(embedConfig, env) {
94
- if (embedConfig && typeof embedConfig.fn === 'function') {
95
- return embedConfig.fn;
96
- }
97
- if (embedConfig && embedConfig.provider) {
98
- const embedder = createEmbedder(embedConfig);
99
- return (texts) => embedder.embedBatch(texts);
100
- }
101
- const provider = env.EMBED_PROVIDER;
102
- if (!provider) return null;
103
-
104
- const opts = { provider };
105
- if (provider === 'ollama') {
106
- opts.ollamaUrl = env.OLLAMA_URL || env.AQUIFER_EMBED_BASE_URL || 'http://localhost:11434';
107
- opts.model = env.AQUIFER_EMBED_MODEL || 'bge-m3';
108
- } else if (provider === 'openai') {
109
- opts.openaiApiKey = env.OPENAI_API_KEY;
110
- if (!opts.openaiApiKey) {
111
- throw new Error('EMBED_PROVIDER=openai requires OPENAI_API_KEY');
112
- }
113
- opts.openaiModel = env.AQUIFER_EMBED_MODEL || 'text-embedding-3-small';
114
- if (env.AQUIFER_EMBED_DIM) opts.openaiDimensions = Number(env.AQUIFER_EMBED_DIM);
115
- } else {
116
- throw new Error(`EMBED_PROVIDER=${provider} not supported by autodetect (use 'ollama' or 'openai', or pass config.embed.fn explicitly)`);
117
- }
118
- const embedder = createEmbedder(opts);
119
- return (texts) => embedder.embedBatch(texts);
120
- }
11
+ const { backendCapabilities, normalizeBackendKind } = require('./backends/capabilities');
12
+ const { createPostgresMigrationRuntime, qi, validateSchema } = require('./postgres-migrations');
13
+ const { createMemoryServingRuntime } = require('./memory-serving');
14
+ const { createLegacyBootstrap } = require('./legacy-bootstrap');
15
+ const { buildRerankDocument, resolveEmbedFn, shouldAutoRerank } = require('./recall-runtime');
16
+ const { filterPublicPlaceholderSessionRows } = require('./public-session-filter');
121
17
 
122
18
  // ---------------------------------------------------------------------------
123
19
  // createAquifer
124
20
  // ---------------------------------------------------------------------------
125
21
 
126
- // Decide whether to invoke the optional reranker on this recall call.
127
- // Returns `{ apply: boolean, reason: string }`. Pure function — no side effects.
128
- function shouldAutoRerank({ query, mode, ranked, hasEntities, autoTrigger }) {
129
- if (!autoTrigger.enabled) return { apply: false, reason: 'auto_disabled' };
130
-
131
- if (hasEntities && autoTrigger.alwaysWhenEntities) {
132
- return { apply: true, reason: 'entities_present' };
133
- }
134
-
135
- const len = ranked.length;
136
- if (len < autoTrigger.minResults) return { apply: false, reason: 'too_few_results' };
137
- if (len > autoTrigger.maxResults) return { apply: false, reason: 'too_many_results' };
138
-
139
- const q = String(query || '').trim();
140
- const tokenCount = q.split(/\s+/).filter(Boolean).length;
141
- if (q.length < autoTrigger.minQueryChars && tokenCount < autoTrigger.minQueryTokens) {
142
- return { apply: false, reason: 'query_too_short' };
143
- }
144
-
145
- // FTS-only path: rerank when results are wide enough that semantic narrowing
146
- // is valuable. Cohere-style cross-encoders excel at re-ranking keyword hits.
147
- if (mode === 'fts') {
148
- if (len > autoTrigger.ftsMinResults) return { apply: true, reason: 'fts_wide_shortlist' };
149
- return { apply: false, reason: 'fts_shortlist_too_narrow' };
150
- }
151
-
152
- if (!autoTrigger.modes.includes(mode)) {
153
- return { apply: false, reason: 'mode_not_in_autotrigger_modes' };
154
- }
155
-
156
- // Hybrid: if top-1 and top-2 are close, signals are mixed enough to benefit.
157
- if (len >= 2) {
158
- const s0 = ranked[0]?._score ?? 0;
159
- const s1 = ranked[1]?._score ?? 0;
160
- if (s0 - s1 <= autoTrigger.maxTopScoreGap) {
161
- return { apply: true, reason: 'top_score_gap_close' };
162
- }
22
+ function createAquifer(config = {}) {
23
+ const backendKind = normalizeBackendKind(config.backend?.kind || config.storage?.backend || 'postgres');
24
+ if (backendKind !== 'postgres') {
25
+ throw new Error(`createAquifer() only constructs the PostgreSQL backend. Use createAquiferFromConfig() for backend "${backendKind}".`);
163
26
  }
27
+ const backendInfo = backendCapabilities(backendKind);
164
28
 
165
- return { apply: false, reason: 'top_score_gap_wide' };
166
- }
167
-
168
- function createAquifer(config = {}) {
169
29
  // v1.2.0: db falls back to DATABASE_URL / AQUIFER_DB_URL env so hosts can
170
30
  // call createAquifer() with zero args for install-and-go.
171
31
  const dbInput = config.db !== undefined
@@ -270,114 +130,9 @@ function createAquifer(config = {}) {
270
130
  // Source registry (in-memory)
271
131
  const sources = new Map();
272
132
 
273
- // Track if migrate was called
274
- let migrated = false;
275
- let migratePromise = null;
276
-
277
- // FTS tsconfig — auto-detected during migrate(). 'zhcfg' if zhparser is
278
- // installed (better Chinese segmentation), otherwise 'simple' (legacy).
279
- // Override via config.ftsConfig if you need to force one or the other.
280
- let ftsConfig = config.ftsConfig || null;
281
-
282
- const memoryCfg = config.memory || {};
283
- const memoryServingMode = memoryCfg.servingMode || process.env.AQUIFER_MEMORY_SERVING_MODE || 'legacy';
284
- function splitScopePath(value) {
285
- if (Array.isArray(value)) return value.map(v => String(v).trim()).filter(Boolean);
286
- if (typeof value !== 'string') return null;
287
- const parts = value.split(',').map(v => v.trim()).filter(Boolean);
288
- return parts.length > 0 ? parts : null;
289
- }
290
- const defaultActiveScopeKey = memoryCfg.activeScopeKey || process.env.AQUIFER_MEMORY_ACTIVE_SCOPE_KEY || null;
291
- const defaultActiveScopePath = splitScopePath(
292
- memoryCfg.activeScopePath || process.env.AQUIFER_MEMORY_ACTIVE_SCOPE_PATH || null,
293
- );
294
- function resolveMemoryServingMode(opts = {}) {
295
- const mode = opts.memoryMode || opts.servingMode || memoryServingMode;
296
- if (mode === 'legacy' || mode === 'evidence') return 'legacy';
297
- if (mode === 'curated') return 'curated';
298
- throw new Error(`Invalid memory serving mode: "${mode}". Must be one of: legacy, curated`);
299
- }
300
- function withDefaultMemoryScope(opts = {}) {
301
- const next = { ...opts };
302
- if (!next.activeScopePath && defaultActiveScopePath) next.activeScopePath = defaultActiveScopePath;
303
- if (!next.activeScopeKey && defaultActiveScopeKey) next.activeScopeKey = defaultActiveScopeKey;
304
- return next;
305
- }
306
- function assertCuratedRecallOpts(opts = {}) {
307
- const unsupported = [];
308
- for (const key of ['agentId', 'agentIds', 'source', 'dateFrom', 'dateTo', 'entities', 'entityMode', 'mode', 'weights', 'rerank', 'allowUnsafeDebug', 'unsafeDebug']) {
309
- if (opts[key] !== undefined && opts[key] !== null) unsupported.push(key);
310
- }
311
- if (unsupported.length > 0) {
312
- throw new Error(`curated session_recall does not support legacy filters: ${unsupported.join(', ')}. Use activeScopeKey/activeScopePath or evidence_recall.`);
313
- }
314
- }
315
- function assertCuratedBootstrapOpts(opts = {}) {
316
- const unsupported = [];
317
- for (const key of ['agentId', 'source', 'lookbackDays', 'dateFrom', 'dateTo']) {
318
- if (opts[key] !== undefined && opts[key] !== null) unsupported.push(key);
319
- }
320
- if (unsupported.length > 0) {
321
- throw new Error(`curated session_bootstrap does not support legacy filters: ${unsupported.join(', ')}. Use activeScopeKey/activeScopePath.`);
322
- }
323
- }
324
- function hasEvidenceBoundary(opts = {}) {
325
- return Boolean(
326
- opts.agentId
327
- || (Array.isArray(opts.agentIds) && opts.agentIds.length > 0)
328
- || opts.source
329
- || opts.dateFrom
330
- || opts.dateTo
331
- || opts.host
332
- || opts.sessionId
333
- || opts.allowUnsafeDebug === true
334
- || opts.unsafeDebug === true
335
- );
336
- }
337
- function curatedRecallTitle(row = {}) {
338
- const title = row.title || row.summary || row.canonical_key || row.canonicalKey || row.memory_type || row.memoryType || 'memory';
339
- return String(title).trim();
340
- }
341
- function curatedRecallSummary(row = {}) {
342
- const summary = row.summary || row.title || row.canonical_key || row.canonicalKey || '';
343
- return String(summary).trim();
344
- }
345
- function normalizeCuratedRecallRow(row = {}) {
346
- const memoryId = row.memoryId || row.memory_id || row.id || null;
347
- const canonicalKey = row.canonicalKey || row.canonical_key || null;
348
- const memoryType = row.memoryType || row.memory_type || null;
349
- const scopeKey = row.scopeKey || row.scope_key || null;
350
- const scopeKind = row.scopeKind || row.scope_kind || null;
351
- const summaryText = curatedRecallSummary(row) || null;
352
- const title = curatedRecallTitle(row) || null;
353
- const scoreValue = row.recall_score ?? row.score ?? row.lexical_rank ?? null;
354
- const score = scoreValue === null ? null : Number(scoreValue);
355
- return {
356
- ...row,
357
- memoryId: memoryId === null ? null : String(memoryId),
358
- canonicalKey,
359
- memoryType,
360
- scopeKey,
361
- scopeKind,
362
- title,
363
- summaryText,
364
- structuredSummary: {
365
- title,
366
- overview: summaryText,
367
- },
368
- startedAt: row.acceptedAt || row.accepted_at || row.observedAt || row.observed_at || null,
369
- score: Number.isFinite(score) ? score : null,
370
- feedbackTarget: {
371
- kind: 'memory_feedback',
372
- memoryId: memoryId === null ? null : String(memoryId),
373
- canonicalKey,
374
- },
375
- };
376
- }
133
+ const memoryServing = createMemoryServingRuntime(config.memory || {}, process.env);
134
+ const legacyBootstrap = createLegacyBootstrap({ pool, schema, tenantId, formatBootstrapText });
377
135
 
378
- // State-change extraction (Q3): off by default. When enabled, enrich() runs
379
- // an extra LLM call to capture temporal state transitions on whitelisted
380
- // entities. See pipeline/extract-state-changes.js + core/entity-state.js.
381
136
  const stateChangesCfg = config.stateChanges || {};
382
137
  const stateChangesEnabled = stateChangesCfg.enabled === true;
383
138
  const stateChangesWhitelist = new Set(
@@ -392,112 +147,17 @@ function createAquifer(config = {}) {
392
147
  const stateChangesMaxOutputTokens = Number.isFinite(stateChangesCfg.maxOutputTokens)
393
148
  ? stateChangesCfg.maxOutputTokens : 600;
394
149
 
395
- const migrationsCfg = config.migrations || {};
396
- const migrationsMode = (() => {
397
- const raw = migrationsCfg.mode;
398
- if (raw === 'apply' || raw === 'check' || raw === 'off') return raw;
399
- if (raw === undefined || raw === null) return 'apply';
400
- throw new Error(`config.migrations.mode must be 'apply' | 'check' | 'off' (got ${JSON.stringify(raw)})`);
401
- })();
402
- const migrationLockTimeoutMs = Number.isFinite(migrationsCfg.lockTimeoutMs)
403
- ? Math.max(0, migrationsCfg.lockTimeoutMs) : 30000;
404
- const migrationStartupTimeoutMs = Number.isFinite(migrationsCfg.startupTimeoutMs)
405
- ? Math.max(0, migrationsCfg.startupTimeoutMs) : 60000;
406
- const migrationOnEvent = typeof migrationsCfg.onEvent === 'function' ? migrationsCfg.onEvent : null;
407
-
408
- function emitMigrationEvent(name, payload) {
409
- if (!migrationOnEvent) return;
410
- try { migrationOnEvent({ name, schema, ...payload }); } catch (err) {
411
- console.warn(`[aquifer] migrations.onEvent handler threw: ${err.message}`);
412
- }
413
- }
414
-
415
- // Expected migration set — used for lazy plan introspection. `always: true`
416
- // runs every migrate(); others are gated by feature flags. Signature tables
417
- // let listPendingMigrations() probe pg_tables without executing DDL.
418
- const MIGRATION_PLAN = [
419
- { id: '001-base', file: '001-base.sql', always: true, signature: 'sessions' },
420
- { id: '002-entities', file: '002-entities.sql', gate: 'entities', signature: 'entities' },
421
- { id: '003-trust-feedback', file: '003-trust-feedback.sql', always: true, signature: 'session_feedback' },
422
- { id: '004-facts', file: '004-facts.sql', gate: 'facts', signature: 'facts' },
423
- { id: '004-completion', file: '004-completion.sql', always: true, signature: 'narratives' },
424
- { id: '005-entity-state-history',file: '005-entity-state-history.sql',gate: 'entities', signature: 'entity_state_history' },
425
- { id: '006-insights', file: '006-insights.sql', always: true, signature: 'insights' },
426
- { id: '007-v1-foundation', file: '007-v1-foundation.sql', always: true, signature: 'memory_records' },
427
- { id: '008-session-finalizations',file: '008-session-finalizations.sql',always: true, signature: 'session_finalizations' },
428
- { id: '009-v1-assertion-plane', file: '009-v1-assertion-plane.sql', always: true, signature: 'fact_assertions_v1' },
429
- { id: '010-v1-finalization-review',file: '010-v1-finalization-review.sql',always: true, signature: 'finalization_candidates' },
430
- { id: '011-v1-compaction-claim', file: '011-v1-compaction-claim.sql', always: true, signature: { table: 'compaction_runs', column: 'apply_token' } },
431
- { id: '012-v1-compaction-lease', file: '012-v1-compaction-lease.sql', always: true, signature: { table: 'compaction_runs', column: 'lease_expires_at' } },
432
- { id: '013-v1-compaction-lineage', file: '013-v1-compaction-lineage.sql', always: true, signature: 'compaction_candidates' },
433
- ];
434
-
435
- function requiredMigrations() {
436
- return MIGRATION_PLAN
437
- .filter(m => m.always
438
- || (m.gate === 'entities' && entitiesEnabled)
439
- || (m.gate === 'facts' && factsEnabled))
440
- .map(m => m.id);
441
- }
442
-
443
- async function readAppliedMigrations(queryRunner) {
444
- const required = MIGRATION_PLAN.filter(m => m.always
445
- || (m.gate === 'entities' && entitiesEnabled)
446
- || (m.gate === 'facts' && factsEnabled));
447
- const tableSignatures = required
448
- .map(m => m.signature)
449
- .filter(signature => typeof signature === 'string');
450
- const columnSignatures = required
451
- .map(m => m.signature)
452
- .filter(signature => signature && typeof signature === 'object');
453
- const presentTables = new Set();
454
- const presentColumns = new Set();
455
- if (tableSignatures.length > 0) {
456
- const r = await queryRunner.query(
457
- `SELECT tablename FROM pg_tables
458
- WHERE schemaname = $1 AND tablename = ANY($2::text[])`,
459
- [schema, tableSignatures]
460
- );
461
- for (const row of r.rows) presentTables.add(row.tablename);
462
- }
463
- if (columnSignatures.length > 0) {
464
- const tables = [...new Set(columnSignatures.map(signature => signature.table))];
465
- const r = await queryRunner.query(
466
- `SELECT table_name, column_name
467
- FROM information_schema.columns
468
- WHERE table_schema = $1 AND table_name = ANY($2::text[])`,
469
- [schema, tables]
470
- );
471
- for (const row of r.rows) presentColumns.add(`${row.table_name}.${row.column_name}`);
472
- }
473
- return required
474
- .filter(m => {
475
- if (typeof m.signature === 'string') return presentTables.has(m.signature);
476
- return presentColumns.has(`${m.signature.table}.${m.signature.column}`);
477
- })
478
- .map(m => m.id);
479
- }
480
-
481
- async function buildMigrationPlan(queryRunner) {
482
- const required = requiredMigrations();
483
- const applied = await readAppliedMigrations(queryRunner);
484
- const appliedSet = new Set(applied);
485
- const pending = required.filter(id => !appliedSet.has(id));
486
- return { required, applied, pending };
487
- }
150
+ const migrationRuntime = createPostgresMigrationRuntime({
151
+ pool,
152
+ schema,
153
+ migrations: config.migrations || {},
154
+ getEntitiesEnabled: () => entitiesEnabled,
155
+ getFactsEnabled: () => factsEnabled,
156
+ initialFtsConfig: config.ftsConfig || null,
157
+ });
488
158
 
489
- async function ensureMigrated() {
490
- if (migrated) return;
491
- if (migratePromise) return migratePromise;
492
- if (migrationsMode === 'off') { migrated = true; return; }
493
- if (migrationsMode === 'check') {
494
- // Lazy compare only — don't execute DDL implicitly.
495
- const plan = await buildMigrationPlan(pool).catch(() => null);
496
- if (plan && plan.pending.length === 0) migrated = true;
497
- return;
498
- }
499
- migratePromise = aquifer.migrate().finally(() => { migratePromise = null; });
500
- return migratePromise;
159
+ function ensureMigrated() {
160
+ return migrationRuntime.ensureMigrated();
501
161
  }
502
162
 
503
163
  // =========================================================================
@@ -512,273 +172,11 @@ function createAquifer(config = {}) {
512
172
  },
513
173
 
514
174
  async migrate() {
515
- const t0 = Date.now();
516
- // Advisory lock prevents concurrent migrations across processes.
517
- // Lock key is derived from schema name to allow parallel migration
518
- // of different schemas in the same database.
519
- const lockKey = Buffer.from(`aquifer:${schema}`).reduce((h, b) => (h * 31 + b) & 0x7fffffff, 0);
520
-
521
- emitMigrationEvent('init_started', { mode: migrationsMode });
522
-
523
- // Run all migration DDL on a single checked-out client so we can
524
- // capture RAISE NOTICE/WARNING emitted by the DO blocks. node-postgres
525
- // swallows notices on pool.query(); attaching a 'notice' listener to a
526
- // held client surfaces them. Fall back to pool.query() when the caller
527
- // passed a bare mock (no connect/release) — tests using minimal pool
528
- // stubs still exercise the migration shape, just without notice capture.
529
- const supportsCheckout = typeof pool.connect === 'function';
530
- const client = supportsCheckout ? await pool.connect() : pool;
531
- const releasesClient = supportsCheckout && typeof client.release === 'function';
532
- const notices = [];
533
- const onNotice = (n) => {
534
- notices.push({ severity: n.severity || 'NOTICE', message: n.message || String(n) });
535
- };
536
- const hasEvents = typeof client.on === 'function' && typeof client.off === 'function';
537
- if (hasEvents) client.on('notice', onNotice);
538
-
539
- const ddlExecuted = [];
540
- let lockAcquired = false;
541
-
542
- try {
543
- // Plan probe before lock: lets consumers see pending list and lets
544
- // us emit an accurate check_completed event even when the DDL is a
545
- // no-op on an already-migrated schema.
546
- const planBefore = await buildMigrationPlan(client).catch(() => null);
547
- emitMigrationEvent('check_completed', {
548
- required: planBefore ? planBefore.required : requiredMigrations(),
549
- applied: planBefore ? planBefore.applied : [],
550
- pending: planBefore ? planBefore.pending : requiredMigrations(),
551
- });
552
-
553
- // Try-lock with poll + timeout. Replaces the old blocking
554
- // pg_advisory_lock() which could hang indefinitely if another
555
- // process crashed holding the lock. Defensive against mock pools:
556
- // only poll when PG explicitly returns ok=false; a missing/empty
557
- // response (test mocks that don't model pg_try_advisory_lock) is
558
- // treated as acquired so suite doesn't hang on the deadline.
559
- const lockDeadline = Date.now() + migrationLockTimeoutMs;
560
- const pollMs = 250;
561
- while (true) {
562
- const r = await client.query('SELECT pg_try_advisory_lock($1) AS ok', [lockKey]);
563
- const row = r && r.rows ? r.rows[0] : null;
564
- if (row && row.ok === false) {
565
- if (Date.now() >= lockDeadline) break;
566
- await new Promise(res => setTimeout(res, pollMs));
567
- continue;
568
- }
569
- lockAcquired = true;
570
- break;
571
- }
572
- if (!lockAcquired) {
573
- const err = new Error(`aquifer: failed to acquire migration advisory lock within ${migrationLockTimeoutMs}ms for schema "${schema}"`);
574
- err.code = 'AQ_MIGRATION_LOCK_TIMEOUT';
575
- err.failedAt = 'acquire_lock';
576
- throw err;
577
- }
578
-
579
- emitMigrationEvent('apply_started', {
580
- pending: planBefore ? planBefore.pending : requiredMigrations(),
581
- });
582
-
583
- try {
584
- // 1. Run base DDL
585
- const baseSql = loadSql('001-base.sql', schema);
586
- await client.query(baseSql); ddlExecuted.push('001-base');
587
-
588
- // 2. If entities enabled, run entity DDL
589
- if (entitiesEnabled) {
590
- const entitySql = loadSql('002-entities.sql', schema);
591
- await client.query(entitySql); ddlExecuted.push('002-entities');
592
- }
593
-
594
- // 3. Trust + feedback (always, not gated by entities)
595
- const trustSql = loadSql('003-trust-feedback.sql', schema);
596
- await client.query(trustSql); ddlExecuted.push('003-trust-feedback');
597
-
598
- // 4. Facts / consolidation (opt-in)
599
- if (factsEnabled) {
600
- const factsSql = loadSql('004-facts.sql', schema);
601
- await client.query(factsSql); ddlExecuted.push('004-facts');
602
- }
603
-
604
- // 5. Completion foundation (always, additive): narratives,
605
- // consumer_profiles, sessions.consolidation_phases. Pure additive DDL
606
- // with IF NOT EXISTS guards — safe on every migrate() call.
607
- const completionSql = loadSql('004-completion.sql', schema);
608
- await client.query(completionSql); ddlExecuted.push('004-completion');
609
-
610
- // 6. Entity state history (always, gated by entitiesEnabled because
611
- // it FK-references entities). Drop-clean — see scripts/drop-entity-state-history.sql.
612
- if (entitiesEnabled) {
613
- const stateHistorySql = loadSql('005-entity-state-history.sql', schema);
614
- await client.query(stateHistorySql); ddlExecuted.push('005-entity-state-history');
615
- }
616
-
617
- // 7. Insights (always, additive). No FK from anywhere into this table —
618
- // safe to DROP CASCADE. See scripts/drop-insights.sql.
619
- const insightsSql = loadSql('006-insights.sql', schema);
620
- await client.query(insightsSql); ddlExecuted.push('006-insights');
621
-
622
- // 8. v1 curated-memory foundation (always, additive). This creates
623
- // a sidecar curated plane without changing the legacy session recall
624
- // or bootstrap serving path.
625
- const memoryV1Sql = loadSql('007-v1-foundation.sql', schema);
626
- await client.query(memoryV1Sql); ddlExecuted.push('007-v1-foundation');
627
-
628
- // 9. v1 session finalization ledger (always, additive). Finalization
629
- // is the source-of-truth lifecycle for handoff/session-end/recovery
630
- // triggers; local consumer markers are cache hints only.
631
- const finalizationSql = loadSql('008-session-finalizations.sql', schema);
632
- await client.query(finalizationSql); ddlExecuted.push('008-session-finalizations');
633
-
634
- // 10. v1 structured assertion plane (always, additive). This keeps
635
- // legacy 004-facts untouched and adds the minimal DB contract for
636
- // backing structured assertions, tenant-safe scope ancestry guards,
637
- // and compaction coverage fields.
638
- const assertionPlaneSql = loadSql('009-v1-assertion-plane.sql', schema);
639
- await client.query(assertionPlaneSql); ddlExecuted.push('009-v1-assertion-plane');
640
-
641
- // 11. v1 finalization review and lineage (always, additive). This
642
- // stores human review text, minimal SessionStart text, finalization
643
- // candidate rows, and row-level created_by_finalization_id lineage.
644
- const finalizationReviewSql = loadSql('010-v1-finalization-review.sql', schema);
645
- await client.query(finalizationReviewSql); ddlExecuted.push('010-v1-finalization-review');
646
-
647
- // 12. v1 compaction claim/apply guard (always, additive). This adds
648
- // a minimal claim token and one-live-apply guard for compaction_runs;
649
- // it does not create or promote aggregate memory.
650
- const compactionClaimSql = loadSql('011-v1-compaction-claim.sql', schema);
651
- await client.query(compactionClaimSql); ddlExecuted.push('011-v1-compaction-claim');
652
-
653
- // 13. v1 compaction claim lease (always, additive). This persists
654
- // row-level lease expiry so stale applying claims can be reclaimed
655
- // without relying on caller clocks or changing runtime defaults.
656
- const compactionLeaseSql = loadSql('012-v1-compaction-lease.sql', schema);
657
- await client.query(compactionLeaseSql); ddlExecuted.push('012-v1-compaction-lease');
658
-
659
- // 14. v1 compaction lineage and candidate ledger (always,
660
- // additive). This records aggregate candidate outcomes and links
661
- // promoted rows back to the compaction run that created them.
662
- const compactionLineageSql = loadSql('013-v1-compaction-lineage.sql', schema);
663
- await client.query(compactionLineageSql); ddlExecuted.push('013-v1-compaction-lineage');
664
-
665
- migrated = true;
666
- } finally {
667
- await client.query('SELECT pg_advisory_unlock($1)', [lockKey]).catch((err) => {
668
- console.warn(`[aquifer] failed to release migration advisory lock for schema "${schema}": ${err.message}`);
669
- });
670
- }
671
- } catch (err) {
672
- err.notices = Array.isArray(err.notices) ? err.notices : notices.slice();
673
- err.failedAt = err.failedAt || 'apply_ddl';
674
- emitMigrationEvent('apply_failed', {
675
- error: { code: err.code || null, message: err.message },
676
- failedAt: err.failedAt,
677
- notices: err.notices,
678
- durationMs: Date.now() - t0,
679
- });
680
- throw err;
681
- } finally {
682
- if (hasEvents) client.off('notice', onNotice);
683
- if (releasesClient) client.release();
684
- }
685
-
686
- // Surface captured migration notices that operators need to see:
687
- // - any WARNING/ERROR (zhcfg rebuild warnings, HNSW OOM, etc.)
688
- // - aquifer-authored NOTICE messages ('[aquifer] ...' prefix in the
689
- // migration DO blocks; these announce extension-install fallback,
690
- // HNSW deferral, and other operational decisions)
691
- // Filtered out: PG's own "relation already exists, skipping" and
692
- // similar idempotent-DDL chatter that floods a re-run.
693
- for (const n of notices) {
694
- const sev = (n.severity || 'NOTICE').toUpperCase();
695
- const msg = n.message || '';
696
- const line = `[aquifer] migration ${sev.toLowerCase()}: ${msg}`;
697
- if (sev === 'WARNING' || sev === 'ERROR') {
698
- console.warn(line);
699
- } else if (sev === 'NOTICE' && msg.startsWith('[aquifer]')) {
700
- process.stderr.write(line + '\n');
701
- }
702
- }
703
-
704
- // Auto-detect FTS tsconfig if not forced by config. Restrict to the
705
- // public namespace — same restriction the trigger function uses — so a
706
- // same-named config in another schema doesn't fool the detection.
707
- if (!ftsConfig) {
708
- try {
709
- const r = await pool.query(
710
- `SELECT 1 FROM pg_ts_config
711
- WHERE cfgname = 'zhcfg' AND cfgnamespace = 'public'::regnamespace
712
- LIMIT 1`);
713
- ftsConfig = r.rowCount > 0 ? 'zhcfg' : 'simple';
714
- } catch {
715
- ftsConfig = 'simple';
716
- }
717
- }
718
-
719
- // Post-flight: surface which Chinese FTS backend the migration actually
720
- // landed on, and warm the backend's tokenizer so the first live query
721
- // doesn't pay cold-start cost unpredictably. RAISE NOTICE/WARNING from
722
- // the migration DO blocks are swallowed by node-postgres unless a
723
- // notice handler is attached, so without this operators can't tell if
724
- // pg_jieba silently failed to install and FTS is degraded to 'simple'.
725
- //
726
- // pg_jieba first-backend load is ~60MB RAM + 0.5-1s to mmap the dict.
727
- // Warming once inside migrate() amortizes that on the backend that runs
728
- // migration; other pool backends still pay it on first use, but the
729
- // timing surfaces the cost so operators who see unexpected latency
730
- // know where to look.
731
- try {
732
- const f = await pool.query(`
733
- SELECT
734
- EXISTS(SELECT 1 FROM pg_extension WHERE extname='pg_jieba') AS have_jieba,
735
- EXISTS(SELECT 1 FROM pg_extension WHERE extname='zhparser') AS have_zhparser,
736
- (SELECT p.prsname FROM pg_ts_config c
737
- JOIN pg_ts_parser p ON c.cfgparser = p.oid
738
- WHERE c.cfgname='zhcfg' AND c.cfgnamespace='public'::regnamespace
739
- LIMIT 1) AS zhcfg_parser
740
- `);
741
- const row = f.rows[0] || {};
742
- const backend = row.zhcfg_parser
743
- ? `zhcfg(parser=${row.zhcfg_parser})`
744
- : `simple (no zhcfg in public namespace)`;
745
-
746
- let warmupMs = null;
747
- if (row.zhcfg_parser) {
748
- const t0 = Date.now();
749
- await pool.query(`SELECT to_tsvector('zhcfg', $1)`, ['warmup 記憶系統 aquifer'])
750
- .catch(() => {});
751
- warmupMs = Date.now() - t0;
752
- }
753
-
754
- const warmupNote = warmupMs !== null ? ` warmup=${warmupMs}ms` : '';
755
- process.stderr.write(
756
- `[aquifer] FTS post-flight: backend=${backend} ` +
757
- `jieba=${row.have_jieba} zhparser=${row.have_zhparser} ` +
758
- `selected=${ftsConfig}${warmupNote}\n`
759
- );
760
- if (warmupMs !== null && warmupMs > 500) {
761
- process.stderr.write(
762
- `[aquifer] Note: first FTS call paid ~${warmupMs}ms for tokenizer init ` +
763
- `(dictionary mmap). Subsequent calls on the same backend are cached.\n`
764
- );
765
- }
766
- } catch (err) {
767
- console.warn(`[aquifer] FTS post-flight check failed: ${err.message}`);
768
- }
769
-
770
- const durationMs = Date.now() - t0;
771
- emitMigrationEvent('apply_succeeded', {
772
- ddlExecuted,
773
- durationMs,
774
- notices: notices.slice(),
775
- });
776
- return { ok: true, durationMs, notices: notices.slice(), ddlExecuted };
175
+ return migrationRuntime.migrate();
777
176
  },
778
177
 
779
178
  async listPendingMigrations() {
780
- const plan = await buildMigrationPlan(pool);
781
- return { ...plan, lastRunAt: null };
179
+ return migrationRuntime.listPendingMigrations();
782
180
  },
783
181
 
784
182
  async getMigrationStatus() {
@@ -786,94 +184,7 @@ function createAquifer(config = {}) {
786
184
  },
787
185
 
788
186
  async init() {
789
- const t0 = Date.now();
790
- const mode = migrationsMode;
791
-
792
- let deadlineTimer = null;
793
- const startupDeadline = migrationStartupTimeoutMs > 0
794
- ? new Promise((_, reject) => {
795
- deadlineTimer = setTimeout(() => {
796
- const err = new Error(`aquifer: init() exceeded startupTimeoutMs=${migrationStartupTimeoutMs}ms`);
797
- err.code = 'AQ_MIGRATION_STARTUP_TIMEOUT';
798
- reject(err);
799
- }, migrationStartupTimeoutMs);
800
- if (typeof deadlineTimer.unref === 'function') deadlineTimer.unref();
801
- })
802
- : null;
803
- const withDeadline = (p) => startupDeadline ? Promise.race([p, startupDeadline]) : p;
804
- const clearDeadline = () => { if (deadlineTimer) { clearTimeout(deadlineTimer); deadlineTimer = null; } };
805
-
806
- try {
807
- let plan;
808
- try {
809
- plan = await withDeadline(buildMigrationPlan(pool));
810
- } catch (err) {
811
- const durationMs = Date.now() - t0;
812
- emitMigrationEvent('apply_failed', {
813
- error: { code: err.code || null, message: err.message },
814
- failedAt: 'plan_probe',
815
- notices: [],
816
- durationMs,
817
- });
818
- return {
819
- ready: false,
820
- memoryMode: 'off',
821
- migrationMode: mode,
822
- pendingMigrations: [],
823
- appliedMigrations: [],
824
- error: { code: err.code || 'AQ_MIGRATION_PROBE_FAILED', message: err.message },
825
- durationMs,
826
- };
827
- }
828
-
829
- if (mode === 'off') {
830
- return {
831
- ready: true, memoryMode: 'rw', migrationMode: mode,
832
- pendingMigrations: plan.pending, appliedMigrations: plan.applied,
833
- error: null, durationMs: Date.now() - t0,
834
- };
835
- }
836
-
837
- if (mode === 'check') {
838
- const ready = plan.pending.length === 0;
839
- if (ready) migrated = true;
840
- return {
841
- ready, memoryMode: ready ? 'rw' : 'ro', migrationMode: mode,
842
- pendingMigrations: plan.pending, appliedMigrations: plan.applied,
843
- error: null, durationMs: Date.now() - t0,
844
- };
845
- }
846
-
847
- // mode === 'apply'
848
- if (plan.pending.length === 0) {
849
- migrated = true;
850
- return {
851
- ready: true, memoryMode: 'rw', migrationMode: mode,
852
- pendingMigrations: [], appliedMigrations: plan.applied,
853
- error: null, durationMs: Date.now() - t0,
854
- };
855
- }
856
-
857
- try {
858
- const result = await withDeadline(this.migrate());
859
- const planAfter = await buildMigrationPlan(pool).catch(() => null);
860
- return {
861
- ready: true, memoryMode: 'rw', migrationMode: mode,
862
- pendingMigrations: planAfter ? planAfter.pending : [],
863
- appliedMigrations: planAfter ? planAfter.applied : plan.required,
864
- error: null, durationMs: result.durationMs || (Date.now() - t0),
865
- };
866
- } catch (err) {
867
- return {
868
- ready: false, memoryMode: 'ro', migrationMode: mode,
869
- pendingMigrations: plan.pending, appliedMigrations: plan.applied,
870
- error: { code: err.code || 'AQ_MIGRATION_FAILED', message: err.message },
871
- durationMs: Date.now() - t0,
872
- };
873
- }
874
- } finally {
875
- clearDeadline();
876
- }
187
+ return migrationRuntime.init();
877
188
  },
878
189
 
879
190
  async close() {
@@ -895,8 +206,8 @@ function createAquifer(config = {}) {
895
206
  async enableEntities() {
896
207
  entitiesEnabled = true;
897
208
  // M4: if already migrated, run entity DDL now
898
- if (migrated) {
899
- const entitySql = loadSql('002-entities.sql', schema);
209
+ if (migrationRuntime.isMigrated()) {
210
+ const entitySql = migrationRuntime.loadSql('002-entities.sql');
900
211
  await pool.query(entitySql);
901
212
  }
902
213
  },
@@ -907,7 +218,7 @@ function createAquifer(config = {}) {
907
218
  // Safe to call repeatedly; also safe to call before migrate() (will no-op
908
219
  // until base schema exists, which enrich/commit will materialize).
909
220
  await ensureMigrated();
910
- const factsSql = loadSql('004-facts.sql', schema);
221
+ const factsSql = migrationRuntime.loadSql('004-facts.sql');
911
222
  await pool.query(factsSql);
912
223
  },
913
224
 
@@ -1345,16 +656,55 @@ function createAquifer(config = {}) {
1345
656
 
1346
657
  // --- read path ---
1347
658
 
1348
- async recall(query, opts = {}) {
1349
- if (resolveMemoryServingMode(opts) === 'curated') {
1350
- assertCuratedRecallOpts(opts);
1351
- await ensureMigrated();
1352
- const rows = await aquifer.memory.recall(query, withDefaultMemoryScope(opts));
1353
- return rows.map(normalizeCuratedRecallRow);
659
+ async memoryRecall(query, opts = {}) {
660
+ memoryServing.assertCuratedRecallOpts(opts);
661
+ await ensureMigrated();
662
+ if (typeof query !== 'string' || query.trim().length === 0) {
663
+ throw new Error('memory.recall(query): query must be a non-empty string');
664
+ }
665
+ const validModes = new Set(['fts', 'hybrid', 'vector']);
666
+ const mode = opts.mode || 'hybrid';
667
+ if (!validModes.has(mode)) {
668
+ throw new Error(`Invalid curated recall mode: "${mode}". Must be one of: fts, hybrid, vector`);
1354
669
  }
670
+ let queryVec = null;
671
+ if (mode === 'hybrid' || mode === 'vector') {
672
+ if (!embedFn) {
673
+ if (mode === 'vector') {
674
+ throw new Error('curated memory_recall mode=vector requires config.embed.fn or EMBED_PROVIDER env');
675
+ }
676
+ } else {
677
+ const embedded = await embedFn([query]);
678
+ queryVec = Array.isArray(embedded) && Array.isArray(embedded[0]) ? embedded[0] : null;
679
+ if (!queryVec && mode === 'vector') throw new Error('embedFn returned empty vector for curated memory_recall');
680
+ }
681
+ }
682
+ const scopedOpts = memoryServing.withDefaultScope(opts);
683
+ const limit = Math.max(1, Math.min(50, scopedOpts.limit || 10));
684
+ const runLexical = mode === 'fts' || mode === 'hybrid';
685
+ const runVector = (mode === 'vector' || mode === 'hybrid') && queryVec;
686
+ const [lexicalRows, embeddingRows] = await Promise.all([
687
+ runLexical ? aquifer.memory.recall(query, {
688
+ ...scopedOpts,
689
+ ftsConfig: migrationRuntime.getFtsConfig(),
690
+ }) : Promise.resolve([]),
691
+ runVector ? aquifer.memory.recallViaMemoryEmbeddings(queryVec, scopedOpts) : Promise.resolve([]),
692
+ ]);
693
+ const rows = aquifer.memory.rankHybridMemoryRows(lexicalRows, embeddingRows, { limit });
694
+ return rows.map(memoryServing.normalizeCuratedRecallRow);
695
+ },
696
+
697
+ async historicalRecall(query, opts = {}) {
1355
698
  return aquifer.evidenceRecall(query, { ...opts, allowBroadEvidence: true });
1356
699
  },
1357
700
 
701
+ async recall(query, opts = {}) {
702
+ if (memoryServing.resolveMode(opts) === 'curated') {
703
+ return aquifer.memoryRecall(query, opts);
704
+ }
705
+ return aquifer.historicalRecall(query, opts);
706
+ },
707
+
1358
708
  async evidenceRecall(query, opts = {}) {
1359
709
  // Contract (aligned across core / manifest / consumer tools): query must
1360
710
  // be a non-empty string. Empty strings previously short-circuited to []
@@ -1363,7 +713,7 @@ function createAquifer(config = {}) {
1363
713
  if (typeof query !== 'string' || query.trim().length === 0) {
1364
714
  throw new Error('aquifer.recall(query): query must be a non-empty string');
1365
715
  }
1366
- if (opts.allowBroadEvidence !== true && !hasEvidenceBoundary(opts)) {
716
+ if (opts.allowBroadEvidence !== true && !memoryServing.hasEvidenceBoundary(opts)) {
1367
717
  throw new Error('evidence_recall requires an audit boundary filter (agentId, source, dateFrom/dateTo, host, sessionId) or allowUnsafeDebug=true');
1368
718
  }
1369
719
 
@@ -1533,7 +883,7 @@ function createAquifer(config = {}) {
1533
883
  runFts
1534
884
  ? storage.searchSessions(pool, query, {
1535
885
  schema, tenantId, agentIds: resolvedAgentIds, source, dateFrom, dateTo, limit: fetchLimit,
1536
- ftsConfig,
886
+ ftsConfig: migrationRuntime.getFtsConfig(),
1537
887
  }).catch((err) => {
1538
888
  recordSearchError('fts', err);
1539
889
  return [];
@@ -1566,9 +916,9 @@ function createAquifer(config = {}) {
1566
916
  ? (rows) => rows.filter(r => candidateSessionIds.has(r.session_id || String(r.id)))
1567
917
  : (rows) => rows;
1568
918
 
1569
- const filteredFts = filterFn(ftsRows);
1570
- const filteredEmb = filterFn(embRows);
1571
- const filteredTurn = filterFn(turnRows);
919
+ const filteredFts = filterPublicPlaceholderSessionRows(filterFn(ftsRows));
920
+ const filteredEmb = filterPublicPlaceholderSessionRows(filterFn(embRows));
921
+ const filteredTurn = filterPublicPlaceholderSessionRows(filterFn(turnRows));
1572
922
 
1573
923
  if (filteredFts.length === 0 && filteredEmb.length === 0 && filteredTurn.length === 0) {
1574
924
  maybeThrowSearchErrors();
@@ -1613,10 +963,11 @@ function createAquifer(config = {}) {
1613
963
  if (externalPromises.length > 0) await Promise.all(externalPromises);
1614
964
 
1615
965
  // 6. Hybrid rank
966
+ const filteredExternalRows = filterPublicPlaceholderSessionRows(filterFn(externalRows));
1616
967
  const mergedWeights = { ...rankWeights, ...overrideWeights };
1617
968
  const ranked = hybridRank(
1618
969
  filteredFts,
1619
- [...filteredEmb, ...filterFn(externalRows)],
970
+ [...filteredEmb, ...filteredExternalRows],
1620
971
  filteredTurn,
1621
972
  {
1622
973
  limit: rerankTopK,
@@ -1673,11 +1024,11 @@ function createAquifer(config = {}) {
1673
1024
  if (aR !== bR) return bR - aR;
1674
1025
  return (b._hybridScore || 0) - (a._hybridScore || 0);
1675
1026
  });
1676
- finalRanked = finalRanked.slice(0, limit);
1027
+ finalRanked = filterPublicPlaceholderSessionRows(finalRanked).slice(0, limit);
1677
1028
  } catch (rerankErr) {
1678
1029
  // Fallback: use original hybrid-rank order, flag in debug
1679
1030
  if (process.env.AQUIFER_DEBUG) console.error('[aquifer] rerank error:', rerankErr.message);
1680
- finalRanked = ranked.slice(0, limit).map(r => ({
1031
+ finalRanked = filterPublicPlaceholderSessionRows(ranked).slice(0, limit).map(r => ({
1681
1032
  ...r,
1682
1033
  _rerankFallback: true,
1683
1034
  _rerankReason: rerankDecision.reason,
@@ -1685,7 +1036,7 @@ function createAquifer(config = {}) {
1685
1036
  }));
1686
1037
  }
1687
1038
  } else {
1688
- finalRanked = ranked.slice(0, limit).map(r => ({ ...r, _rerankReason: rerankDecision.reason }));
1039
+ finalRanked = filterPublicPlaceholderSessionRows(ranked).slice(0, limit).map(r => ({ ...r, _rerankReason: rerankDecision.reason }));
1689
1040
  }
1690
1041
 
1691
1042
  // 7. Record access
@@ -1838,7 +1189,20 @@ function createAquifer(config = {}) {
1838
1189
  // --- public config accessor ---
1839
1190
 
1840
1191
  getConfig() {
1841
- return { schema, tenantId, memoryServingMode };
1192
+ return {
1193
+ schema,
1194
+ tenantId,
1195
+ memoryServingMode: memoryServing.servingMode,
1196
+ memoryActiveScopeKey: memoryServing.defaultActiveScopeKey,
1197
+ memoryActiveScopePath: memoryServing.defaultActiveScopePath,
1198
+ backendKind,
1199
+ backendProfile: backendInfo.profile,
1200
+ capabilities: backendInfo.capabilities,
1201
+ };
1202
+ },
1203
+
1204
+ getCapabilities() {
1205
+ return backendCapabilities(backendKind);
1842
1206
  },
1843
1207
 
1844
1208
  // v1.2.0: expose the internal pool so host persona layers can reuse it
@@ -1894,12 +1258,92 @@ function createAquifer(config = {}) {
1894
1258
  entityCount = entResult.rows[0]?.count || 0;
1895
1259
  } catch { /* entities table may not exist */ }
1896
1260
 
1261
+ let memoryRecords = {
1262
+ available: false,
1263
+ total: 0,
1264
+ active: 0,
1265
+ visibleInBootstrap: 0,
1266
+ visibleInRecall: 0,
1267
+ earliest: null,
1268
+ latest: null,
1269
+ };
1270
+ try {
1271
+ const memoryResult = await pool.query(
1272
+ `SELECT
1273
+ COUNT(*)::int AS total,
1274
+ COUNT(*) FILTER (WHERE status = 'active')::int AS active,
1275
+ COUNT(*) FILTER (WHERE status = 'active' AND visible_in_bootstrap = true)::int AS visible_in_bootstrap,
1276
+ COUNT(*) FILTER (WHERE status = 'active' AND visible_in_recall = true)::int AS visible_in_recall,
1277
+ MIN(accepted_at) AS earliest,
1278
+ MAX(accepted_at) AS latest
1279
+ FROM ${qi(schema)}.memory_records
1280
+ WHERE tenant_id = $1`,
1281
+ [tenantId]
1282
+ );
1283
+ const row = memoryResult.rows[0] || {};
1284
+ memoryRecords = {
1285
+ available: true,
1286
+ total: row.total || 0,
1287
+ active: row.active || 0,
1288
+ visibleInBootstrap: row.visible_in_bootstrap || 0,
1289
+ visibleInRecall: row.visible_in_recall || 0,
1290
+ earliest: row.earliest || null,
1291
+ latest: row.latest || null,
1292
+ };
1293
+ } catch { /* memory_records table may not exist on older installs */ }
1294
+
1295
+ let sessionFinalizations = {
1296
+ available: false,
1297
+ total: 0,
1298
+ statuses: {},
1299
+ latestFinalizedAt: null,
1300
+ latestUpdatedAt: null,
1301
+ };
1302
+ try {
1303
+ const finalizationResult = await pool.query(
1304
+ `SELECT
1305
+ status,
1306
+ COUNT(*)::int AS count,
1307
+ MAX(finalized_at) AS latest_finalized_at,
1308
+ MAX(updated_at) AS latest_updated_at
1309
+ FROM ${qi(schema)}.session_finalizations
1310
+ WHERE tenant_id = $1
1311
+ GROUP BY status`,
1312
+ [tenantId]
1313
+ );
1314
+ const statuses = Object.fromEntries(finalizationResult.rows.map(row => [row.status, row.count]));
1315
+ sessionFinalizations = {
1316
+ available: true,
1317
+ total: finalizationResult.rows.reduce((sum, row) => sum + row.count, 0),
1318
+ statuses,
1319
+ latestFinalizedAt: finalizationResult.rows
1320
+ .map(row => row.latest_finalized_at)
1321
+ .filter(Boolean)
1322
+ .sort()
1323
+ .pop() || null,
1324
+ latestUpdatedAt: finalizationResult.rows
1325
+ .map(row => row.latest_updated_at)
1326
+ .filter(Boolean)
1327
+ .sort()
1328
+ .pop() || null,
1329
+ };
1330
+ } catch { /* session_finalizations table may not exist on older installs */ }
1331
+
1897
1332
  return {
1333
+ backendKind,
1334
+ backendProfile: backendInfo.profile,
1335
+ serving: {
1336
+ mode: memoryServing.servingMode,
1337
+ activeScopeKey: memoryServing.defaultActiveScopeKey,
1338
+ activeScopePath: memoryServing.defaultActiveScopePath,
1339
+ },
1898
1340
  sessions: Object.fromEntries(sessions.rows.map(r => [r.processing_status, r.count])),
1899
1341
  sessionTotal: sessions.rows.reduce((s, r) => s + r.count, 0),
1900
1342
  summaries: summaries.rows[0]?.count || 0,
1901
1343
  turnEmbeddings: turns.rows[0]?.count || 0,
1902
1344
  entities: entityCount,
1345
+ memoryRecords,
1346
+ sessionFinalizations,
1903
1347
  earliest: timeRange.rows[0]?.earliest || null,
1904
1348
  latest: timeRange.rows[0]?.latest || null,
1905
1349
  };
@@ -1941,113 +1385,23 @@ function createAquifer(config = {}) {
1941
1385
  return result.rows;
1942
1386
  },
1943
1387
 
1944
- async bootstrap(opts = {}) {
1388
+ async memoryBootstrap(opts = {}) {
1945
1389
  await ensureMigrated();
1946
- if (resolveMemoryServingMode(opts) === 'curated') {
1947
- assertCuratedBootstrapOpts(opts);
1948
- return aquifer.memory.bootstrap(withDefaultMemoryScope(opts));
1949
- }
1950
-
1951
- const agentId = opts.agentId || null;
1952
- const source = opts.source || null;
1953
- const limit = Math.max(1, Math.min(20, opts.limit || 5));
1954
- const lookbackDays = opts.lookbackDays || 14;
1955
- const maxChars = opts.maxChars || 4000;
1956
- const format = opts.format || 'structured';
1957
-
1958
- // 'partial' sessions have a summary but recorded warnings during enrich;
1959
- // they are user-visible content, not in-progress — bootstrap must include
1960
- // them alongside 'succeeded'. 'pending' / 'processing' have no summary
1961
- // yet and are correctly excluded.
1962
- const where = [`s.tenant_id = $1`, `s.processing_status IN ('succeeded', 'partial')`];
1963
- const params = [tenantId];
1964
-
1965
- if (agentId) {
1966
- params.push(agentId);
1967
- where.push(`s.agent_id = $${params.length}`);
1968
- }
1969
- if (source) {
1970
- params.push(source);
1971
- where.push(`s.source = $${params.length}`);
1972
- }
1973
-
1974
- params.push(lookbackDays);
1975
- // upsertSession sets ended_at on every commit but started_at / last_message_at
1976
- // only when the caller supplies them — fall back through both so sessions
1977
- // committed without explicit timestamps remain reachable.
1978
- where.push(`COALESCE(s.last_message_at, s.ended_at, s.started_at) > now() - ($${params.length} || ' days')::interval`);
1979
-
1980
- params.push(limit);
1981
-
1982
- const result = await pool.query(
1983
- `SELECT s.session_id, s.agent_id, s.source, s.started_at, s.msg_count,
1984
- ss.summary_text, ss.structured_summary
1985
- FROM ${qi(schema)}.sessions s
1986
- JOIN ${qi(schema)}.session_summaries ss ON ss.session_row_id = s.id
1987
- WHERE ${where.join(' AND ')}
1988
- ORDER BY COALESCE(s.last_message_at, s.ended_at, s.started_at) DESC
1989
- LIMIT $${params.length}`,
1990
- params
1991
- );
1992
-
1993
- const sessions = result.rows.map(r => {
1994
- const ss = r.structured_summary || {};
1995
- const hasSS = ss.title || ss.overview;
1996
- return {
1997
- sessionId: r.session_id,
1998
- agentId: r.agent_id,
1999
- source: r.source,
2000
- startedAt: r.started_at,
2001
- title: ss.title || (hasSS ? null : (r.summary_text || '').slice(0, 60).trim() || null),
2002
- overview: ss.overview || (hasSS ? null : (r.summary_text || '').slice(0, 200).trim() || null),
2003
- topics: Array.isArray(ss.topics) ? ss.topics : [],
2004
- decisions: Array.isArray(ss.decisions) ? ss.decisions : [],
2005
- openLoops: Array.isArray(ss.open_loops) ? ss.open_loops : [],
2006
- importantFacts: Array.isArray(ss.important_facts) ? ss.important_facts : [],
2007
- };
2008
- });
2009
-
2010
- // Cross-session open loops merge + dedup + sentinel filter
2011
- const SENTINELS = new Set(['無', 'none', 'n/a', 'na', 'done', '']);
2012
- const seenLoops = new Set();
2013
- const openLoops = [];
2014
- for (const s of sessions) {
2015
- for (const loop of s.openLoops) {
2016
- const raw = typeof loop === 'string' ? loop : (loop.item || '');
2017
- const normalized = raw.trim().replace(/\s+/g, ' ').toLowerCase();
2018
- if (SENTINELS.has(normalized) || !normalized || seenLoops.has(normalized)) continue;
2019
- seenLoops.add(normalized);
2020
- openLoops.push({ item: raw.trim(), fromSession: s.sessionId, latestStartedAt: s.startedAt });
2021
- }
2022
- }
2023
-
2024
- // Cross-session recent decisions dedup
2025
- const seenDecisions = new Set();
2026
- const recentDecisions = [];
2027
- for (const s of sessions) {
2028
- for (const d of s.decisions) {
2029
- const key = typeof d === 'string' ? d : (d.decision || '');
2030
- const normalized = key.trim().replace(/\s+/g, ' ').toLowerCase();
2031
- if (!normalized || seenDecisions.has(normalized)) continue;
2032
- seenDecisions.add(normalized);
2033
- recentDecisions.push({ decision: key.trim(), reason: d.reason || null, fromSession: s.sessionId });
2034
- }
2035
- }
1390
+ memoryServing.assertCuratedBootstrapOpts(opts);
1391
+ return aquifer.memory.bootstrap(memoryServing.withDefaultScope(opts));
1392
+ },
2036
1393
 
2037
- const structured = {
2038
- sessions,
2039
- openLoops,
2040
- recentDecisions,
2041
- meta: { lookbackDays, count: sessions.length, maxChars, truncated: false },
2042
- };
1394
+ async historicalBootstrap(opts = {}) {
1395
+ await ensureMigrated();
1396
+ return legacyBootstrap(opts);
1397
+ },
2043
1398
 
2044
- if (format === 'text' || format === 'both') {
2045
- const textResult = formatBootstrapText(structured, maxChars);
2046
- structured.text = textResult.text;
2047
- structured.meta.truncated = textResult.truncated;
1399
+ async bootstrap(opts = {}) {
1400
+ if (memoryServing.resolveMode(opts) === 'curated') {
1401
+ return aquifer.memoryBootstrap(opts);
2048
1402
  }
2049
1403
 
2050
- return structured;
1404
+ return aquifer.historicalBootstrap(opts);
2051
1405
  },
2052
1406
  };
2053
1407
 
@@ -2066,11 +1420,12 @@ function createAquifer(config = {}) {
2066
1420
  const { createEntityState } = require('./entity-state');
2067
1421
  const { createInsights } = require('./insights');
2068
1422
  const { createMemoryRecords } = require('./memory-records');
2069
- const { createMemoryPromotion } = require('./memory-promotion');
1423
+ const { createMemoryPromotion, buildMemoryEmbeddingText } = require('./memory-promotion');
2070
1424
  const { createMemoryBootstrap } = require('./memory-bootstrap');
2071
1425
  const { createMemoryRecall } = require('./memory-recall');
2072
1426
  const { createMemoryConsolidation } = require('./memory-consolidation');
2073
1427
  const { createSessionFinalization } = require('./session-finalization');
1428
+ const { createSessionCheckpoints } = require('./session-checkpoints');
2074
1429
  const qSchema = qi(schema);
2075
1430
  aquifer.narratives = createNarratives({ pool, schema: qSchema, defaultTenantId: tenantId });
2076
1431
  aquifer.timeline = createTimeline({ pool, schema: qSchema, defaultTenantId: tenantId });
@@ -2100,7 +1455,7 @@ function createAquifer(config = {}) {
2100
1455
  });
2101
1456
 
2102
1457
  const memoryRecords = createMemoryRecords({ pool, schema: qSchema, defaultTenantId: tenantId });
2103
- const memoryPromotion = createMemoryPromotion({ records: memoryRecords });
1458
+ const memoryPromotion = createMemoryPromotion({ records: memoryRecords, embedFn });
2104
1459
  const memoryBootstrap = createMemoryBootstrap({ records: memoryRecords });
2105
1460
  const memoryRecall = createMemoryRecall({ pool, schema: qSchema, defaultTenantId: tenantId });
2106
1461
  const memoryConsolidation = createMemoryConsolidation({
@@ -2114,8 +1469,22 @@ function createAquifer(config = {}) {
2114
1469
  schema,
2115
1470
  recordsSchema: qSchema,
2116
1471
  defaultTenantId: tenantId,
1472
+ embedFn,
1473
+ });
1474
+ const sessionCheckpoints = createSessionCheckpoints({
1475
+ pool,
1476
+ schema,
1477
+ defaultTenantId: tenantId,
2117
1478
  });
2118
1479
 
1480
+ function currentMemoryScopeKeys(opts = {}) {
1481
+ if (Array.isArray(opts.activeScopePath) && opts.activeScopePath.length > 0) {
1482
+ return opts.activeScopePath.map(value => String(value)).filter(Boolean);
1483
+ }
1484
+ if (opts.activeScopeKey) return [String(opts.activeScopeKey)];
1485
+ return null;
1486
+ }
1487
+
2119
1488
  // v1 curated-memory sidecar. Top-level recall/bootstrap can opt into this
2120
1489
  // plane through memory.servingMode while legacy/evidence mode remains
2121
1490
  // available for compatibility and debugging.
@@ -2132,6 +1501,10 @@ function createAquifer(config = {}) {
2132
1501
  await ensureMigrated();
2133
1502
  return memoryRecords.upsertMemory(input);
2134
1503
  },
1504
+ upsertEvidenceItem: async (input = {}) => {
1505
+ await ensureMigrated();
1506
+ return memoryRecords.upsertEvidenceItem(input);
1507
+ },
2135
1508
  linkEvidence: async (input = {}) => {
2136
1509
  await ensureMigrated();
2137
1510
  return memoryRecords.linkEvidence(input);
@@ -2152,16 +1525,110 @@ function createAquifer(config = {}) {
2152
1525
  },
2153
1526
  current: async (opts = {}) => {
2154
1527
  await ensureMigrated();
2155
- return memoryRecords.currentProjection(withDefaultMemoryScope(opts));
1528
+ return memoryRecords.currentProjection(memoryServing.withDefaultScope(opts));
2156
1529
  },
2157
1530
  listCurrentMemory: async (opts = {}) => {
2158
1531
  await ensureMigrated();
2159
- return memoryRecords.currentProjection(withDefaultMemoryScope(opts));
1532
+ return memoryRecords.currentProjection(memoryServing.withDefaultScope(opts));
1533
+ },
1534
+ backfillEmbeddings: async (opts = {}) => {
1535
+ await ensureMigrated();
1536
+ requireEmbed('memory.backfillEmbeddings');
1537
+ const scopedOpts = memoryServing.withDefaultScope(opts);
1538
+ const listInput = {
1539
+ tenantId: scopedOpts.tenantId || tenantId,
1540
+ asOf: scopedOpts.asOf,
1541
+ scopeId: scopedOpts.scopeId,
1542
+ scopeKeys: currentMemoryScopeKeys(scopedOpts),
1543
+ withoutEmbedding: true,
1544
+ limit: Math.max(1, Math.min(200, scopedOpts.limit || 50)),
1545
+ };
1546
+ if (scopedOpts.visibleInRecall !== undefined) {
1547
+ listInput.visibleInRecall = scopedOpts.visibleInRecall;
1548
+ } else if (scopedOpts.visibleInBootstrap === undefined) {
1549
+ listInput.visibleInRecall = true;
1550
+ }
1551
+ if (scopedOpts.visibleInBootstrap !== undefined) {
1552
+ listInput.visibleInBootstrap = scopedOpts.visibleInBootstrap;
1553
+ }
1554
+ const sourceRows = await memoryRecords.listActive(listInput);
1555
+ const rowsToEmbed = [];
1556
+ const texts = [];
1557
+ for (const row of sourceRows) {
1558
+ const text = buildMemoryEmbeddingText(row);
1559
+ if (!text) continue;
1560
+ rowsToEmbed.push(row);
1561
+ texts.push(text);
1562
+ }
1563
+ if (rowsToEmbed.length === 0) {
1564
+ return {
1565
+ scanned: sourceRows.length,
1566
+ embedded: 0,
1567
+ skipped: sourceRows.length,
1568
+ memories: [],
1569
+ };
1570
+ }
1571
+ const vectors = await embedFn(texts);
1572
+ if (!Array.isArray(vectors) || vectors.length !== texts.length) {
1573
+ throw new Error(`memory.backfillEmbeddings embedFn returned ${Array.isArray(vectors) ? vectors.length : 'invalid'} vectors for ${texts.length} memory rows`);
1574
+ }
1575
+ const updatedRows = [];
1576
+ let skipped = sourceRows.length - rowsToEmbed.length;
1577
+ const skippedMemories = [];
1578
+ for (let i = 0; i < rowsToEmbed.length; i++) {
1579
+ const vector = vectors[i];
1580
+ if (!Array.isArray(vector) || vector.length === 0) {
1581
+ skipped += 1;
1582
+ skippedMemories.push({
1583
+ memoryId: String(rowsToEmbed[i].id),
1584
+ canonicalKey: rowsToEmbed[i].canonical_key || rowsToEmbed[i].canonicalKey || null,
1585
+ reason: 'empty_vector',
1586
+ });
1587
+ continue;
1588
+ }
1589
+ const updateResult = await memoryRecords.updateMemoryEmbedding({
1590
+ tenantId: scopedOpts.tenantId || tenantId,
1591
+ memoryId: rowsToEmbed[i].id,
1592
+ embedding: vector,
1593
+ });
1594
+ if (updateResult && updateResult.updated && updateResult.memory) {
1595
+ updatedRows.push(updateResult.memory);
1596
+ continue;
1597
+ }
1598
+ skipped += 1;
1599
+ skippedMemories.push({
1600
+ memoryId: String(rowsToEmbed[i].id),
1601
+ canonicalKey: rowsToEmbed[i].canonical_key || rowsToEmbed[i].canonicalKey || null,
1602
+ reason: updateResult && updateResult.status ? updateResult.status : 'not_updated',
1603
+ });
1604
+ }
1605
+ return {
1606
+ scanned: sourceRows.length,
1607
+ embedded: updatedRows.length,
1608
+ skipped,
1609
+ memories: updatedRows.map(memoryRecords.normalizeCurrentMemoryRow),
1610
+ skippedMemories,
1611
+ };
2160
1612
  },
2161
1613
  recall: async (query, opts = {}) => {
2162
1614
  await ensureMigrated();
2163
1615
  return memoryRecall.recall(query, opts);
2164
1616
  },
1617
+ recallViaEvidenceItems: async (query, opts = {}) => {
1618
+ await ensureMigrated();
1619
+ return memoryRecall.recallViaEvidenceItems(query, opts);
1620
+ },
1621
+ recallViaMemoryEmbeddings: async (queryVec, opts = {}) => {
1622
+ await ensureMigrated();
1623
+ return memoryRecall.recallViaMemoryEmbeddings(queryVec, opts);
1624
+ },
1625
+ recallViaLinkedSummaryEmbeddings: async (queryVec, opts = {}) => {
1626
+ await ensureMigrated();
1627
+ return memoryRecall.recallViaLinkedSummaryEmbeddings(queryVec, opts);
1628
+ },
1629
+ rankHybridMemoryRows: (lexicalRows, embeddingRows, opts = {}) => {
1630
+ return memoryRecall.rankHybridMemoryRows(lexicalRows, embeddingRows, opts);
1631
+ },
2165
1632
  consolidation: {
2166
1633
  plan: memoryConsolidation.plan,
2167
1634
  distillArchiveSnapshot: memoryConsolidation.distillArchiveSnapshot,
@@ -2211,6 +1678,50 @@ function createAquifer(config = {}) {
2211
1678
  },
2212
1679
  };
2213
1680
 
1681
+ aquifer.checkpoints = {
1682
+ upsertRun: async (input = {}) => {
1683
+ await ensureMigrated();
1684
+ return sessionCheckpoints.upsertRun(input);
1685
+ },
1686
+ updateRunStatus: async (input = {}) => {
1687
+ await ensureMigrated();
1688
+ return sessionCheckpoints.updateRunStatus(input);
1689
+ },
1690
+ listRuns: async (input = {}) => {
1691
+ await ensureMigrated();
1692
+ return sessionCheckpoints.listRuns(input);
1693
+ },
1694
+ upsertSources: async (rows = [], input = {}) => {
1695
+ await ensureMigrated();
1696
+ return sessionCheckpoints.upsertSources(rows, input);
1697
+ },
1698
+ listSources: async (input = {}) => {
1699
+ await ensureMigrated();
1700
+ return sessionCheckpoints.listSources(input);
1701
+ },
1702
+ buildSynthesisInput: (input = {}) => sessionCheckpoints.buildSynthesisInput(input),
1703
+ buildSynthesisPrompt: (input = {}, opts = {}) => sessionCheckpoints.buildSynthesisPrompt(input, opts),
1704
+ buildRunInputFromSynthesis: (input = {}, summary = {}, opts = {}) => (
1705
+ sessionCheckpoints.buildRunInputFromSynthesis(input, summary, opts)
1706
+ ),
1707
+ planFromFinalizations: async (input = {}) => {
1708
+ await ensureMigrated();
1709
+ return sessionCheckpoints.planFromFinalizations(input);
1710
+ },
1711
+ runProducer: async (input = {}) => {
1712
+ await ensureMigrated();
1713
+ return sessionCheckpoints.runProducer(input);
1714
+ },
1715
+ listForHandoff: async (input = {}) => {
1716
+ await ensureMigrated();
1717
+ return sessionCheckpoints.listForHandoff(input);
1718
+ },
1719
+ listAcceptedForHandoff: async (input = {}) => {
1720
+ await ensureMigrated();
1721
+ return sessionCheckpoints.listAcceptedForHandoff(input);
1722
+ },
1723
+ };
1724
+
2214
1725
  aquifer.finalizeSession = aquifer.finalization.finalizeSession;
2215
1726
 
2216
1727
  return aquifer;