@jungjaehoon/mama-core 1.7.0 → 1.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/db/migrations/038-create-vnext-operator-contracts.sql +57 -0
- package/db/migrations/039-add-connector-event-operator-ingest-seq.sql +96 -0
- package/db/migrations/040-create-operator-memory-commit-intents.sql +19 -0
- package/db/migrations/041-enforce-operator-memory-commit-claim-invariant.sql +74 -0
- package/db/migrations/042-embedding-prefix-scheme.sql +46 -0
- package/dist/db-adapter/node-sqlite-adapter.d.ts +14 -1
- package/dist/db-adapter/node-sqlite-adapter.js +323 -6
- package/dist/db-manager.d.ts +9 -10
- package/dist/db-manager.js +59 -4
- package/dist/embedding-client.d.ts +3 -1
- package/dist/embedding-client.js +3 -2
- package/dist/embedding-server/index.d.ts +2 -0
- package/dist/embedding-server/index.js +10 -2
- package/dist/embeddings.d.ts +5 -3
- package/dist/embeddings.js +16 -10
- package/dist/index.d.ts +3 -2
- package/dist/index.js +7 -4
- package/dist/mama-api.js +1 -1
- package/dist/memory/api.d.ts +6 -1
- package/dist/memory/api.js +223 -29
- package/dist/memory/provenance.d.ts +1 -0
- package/dist/memory/provenance.js +1 -0
- package/dist/memory-inject.js +1 -1
- package/dist/provenance/source-ref.d.ts +29 -0
- package/dist/provenance/source-ref.js +204 -0
- package/package.json +2 -1
|
@@ -88,6 +88,12 @@ class NodeSQLiteAdapter extends base_adapter_js_1.DatabaseAdapter {
|
|
|
88
88
|
_vectorSearchEnabled = true;
|
|
89
89
|
vectorCache = new Map();
|
|
90
90
|
topicCache = new Map();
|
|
91
|
+
// Effective status (status, falling back to outcome) per decision rowid. Used as a
|
|
92
|
+
// search-time optimization only - recallMemory's post-filter stays the authority
|
|
93
|
+
// (this cache can lag a status UPDATE until the next reloadVectorCache).
|
|
94
|
+
statusCache = new Map();
|
|
95
|
+
decisionsHasStatusColumns = false;
|
|
96
|
+
decisionsColumnInfoChecked = false;
|
|
91
97
|
constructor(config = {}) {
|
|
92
98
|
super();
|
|
93
99
|
this.config = config;
|
|
@@ -140,6 +146,46 @@ class NodeSQLiteAdapter extends base_adapter_js_1.DatabaseAdapter {
|
|
|
140
146
|
reloadVectorCache() {
|
|
141
147
|
this.loadVectorCache();
|
|
142
148
|
}
|
|
149
|
+
// Re-read one decision's effective status into the cache. MUST be called after
|
|
150
|
+
// any status transition that can move a row OUT of an excluded state (e.g.
|
|
151
|
+
// promoteMemoryStatus staging->active): the vectorSearch pre-filter drops
|
|
152
|
+
// excluded rowids before the api post-filter ever sees them, so a stale
|
|
153
|
+
// excluded entry would make an active row unrecallable until restart.
|
|
154
|
+
refreshDecisionStatusCache(rowid) {
|
|
155
|
+
if (!this.isConnected()) {
|
|
156
|
+
throw new Error('Database not connected');
|
|
157
|
+
}
|
|
158
|
+
if (!this.decisionsColumnInfoChecked) {
|
|
159
|
+
this.refreshDecisionColumnInfo();
|
|
160
|
+
}
|
|
161
|
+
const cacheSelect = this.decisionsHasStatusColumns
|
|
162
|
+
? 'SELECT topic, status, outcome FROM decisions WHERE rowid = ?'
|
|
163
|
+
: 'SELECT topic, NULL AS status, NULL AS outcome FROM decisions WHERE rowid = ?';
|
|
164
|
+
const row = this.prepare(cacheSelect).get(rowid);
|
|
165
|
+
if (!row) {
|
|
166
|
+
this.statusCache.delete(rowid);
|
|
167
|
+
this.topicCache.delete(rowid);
|
|
168
|
+
return;
|
|
169
|
+
}
|
|
170
|
+
this.topicCache.set(rowid, row.topic);
|
|
171
|
+
const effectiveStatus = row.status || row.outcome;
|
|
172
|
+
if (effectiveStatus) {
|
|
173
|
+
this.statusCache.set(rowid, effectiveStatus);
|
|
174
|
+
}
|
|
175
|
+
else {
|
|
176
|
+
this.statusCache.delete(rowid);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
refreshDecisionColumnInfo() {
|
|
180
|
+
if (!this.db)
|
|
181
|
+
return new Set();
|
|
182
|
+
const decisionCols = new Set(this.db.prepare('PRAGMA table_info(decisions)').all().map((c) => c.name));
|
|
183
|
+
this.decisionsHasStatusColumns = decisionCols.has('status') && decisionCols.has('outcome');
|
|
184
|
+
// Only latch "checked" once the decisions table actually exists - introspecting
|
|
185
|
+
// a not-yet-migrated DB must not stop later calls from re-checking.
|
|
186
|
+
this.decisionsColumnInfoChecked = decisionCols.size > 0;
|
|
187
|
+
return decisionCols;
|
|
188
|
+
}
|
|
143
189
|
loadVectorCache() {
|
|
144
190
|
if (!this.db)
|
|
145
191
|
return;
|
|
@@ -149,6 +195,7 @@ class NodeSQLiteAdapter extends base_adapter_js_1.DatabaseAdapter {
|
|
|
149
195
|
if (tableCheck.length === 0) {
|
|
150
196
|
this.vectorCache.clear();
|
|
151
197
|
this.topicCache.clear();
|
|
198
|
+
this.statusCache.clear();
|
|
152
199
|
return;
|
|
153
200
|
}
|
|
154
201
|
const start = Date.now();
|
|
@@ -161,11 +208,26 @@ class NodeSQLiteAdapter extends base_adapter_js_1.DatabaseAdapter {
|
|
|
161
208
|
this.vectorCache.set(row.rowid, vec);
|
|
162
209
|
}
|
|
163
210
|
}
|
|
164
|
-
// Load topic
|
|
211
|
+
// Load topic + effective-status caches for scoped/filtered vector search.
|
|
212
|
+
// Legacy partial schemas may lack status/outcome (they are added by later
|
|
213
|
+
// migrations, and loadVectorCache also runs at connect time, before
|
|
214
|
+
// runMigrations) - introspect columns so prepare() cannot crash on them.
|
|
215
|
+
// A missing column just leaves statusCache empty; the api-layer post-filter
|
|
216
|
+
// remains the authority.
|
|
165
217
|
this.topicCache.clear();
|
|
166
|
-
|
|
218
|
+
this.statusCache.clear();
|
|
219
|
+
const decisionCols = this.refreshDecisionColumnInfo();
|
|
220
|
+
const statusSelect = decisionCols.has('status') ? 'status' : 'NULL AS status';
|
|
221
|
+
const outcomeSelect = decisionCols.has('outcome') ? 'outcome' : 'NULL AS outcome';
|
|
222
|
+
const topicRows = this.db
|
|
223
|
+
.prepare(`SELECT rowid, topic, ${statusSelect}, ${outcomeSelect} FROM decisions`)
|
|
224
|
+
.all();
|
|
167
225
|
for (const row of topicRows) {
|
|
168
226
|
this.topicCache.set(row.rowid, row.topic);
|
|
227
|
+
const effectiveStatus = row.status || row.outcome;
|
|
228
|
+
if (effectiveStatus) {
|
|
229
|
+
this.statusCache.set(row.rowid, effectiveStatus);
|
|
230
|
+
}
|
|
169
231
|
}
|
|
170
232
|
const count = this.vectorCache.size;
|
|
171
233
|
const elapsed = Date.now() - start;
|
|
@@ -220,7 +282,7 @@ class NodeSQLiteAdapter extends base_adapter_js_1.DatabaseAdapter {
|
|
|
220
282
|
throw error;
|
|
221
283
|
}
|
|
222
284
|
}
|
|
223
|
-
vectorSearch(embedding, limit = 5, topicPrefix) {
|
|
285
|
+
vectorSearch(embedding, limit = 5, topicPrefix, excludeStatuses) {
|
|
224
286
|
if (!this.isConnected()) {
|
|
225
287
|
throw new Error('Database not connected');
|
|
226
288
|
}
|
|
@@ -228,6 +290,7 @@ class NodeSQLiteAdapter extends base_adapter_js_1.DatabaseAdapter {
|
|
|
228
290
|
const effectiveLimit = Math.max(limit, 1);
|
|
229
291
|
const bestMatches = [];
|
|
230
292
|
let minScore = -Infinity;
|
|
293
|
+
const excluded = excludeStatuses && excludeStatuses.length > 0 ? new Set(excludeStatuses) : null;
|
|
231
294
|
for (const [rowid, candidate] of this.vectorCache) {
|
|
232
295
|
if (candidate.length !== queryVector.length)
|
|
233
296
|
continue;
|
|
@@ -237,6 +300,13 @@ class NodeSQLiteAdapter extends base_adapter_js_1.DatabaseAdapter {
|
|
|
237
300
|
if (!topic || !topic.startsWith(topicPrefix))
|
|
238
301
|
continue;
|
|
239
302
|
}
|
|
303
|
+
// Pre-filter by effective status so superseded history does not occupy
|
|
304
|
+
// top-K slots (the api-layer post-filter remains the authority)
|
|
305
|
+
if (excluded) {
|
|
306
|
+
const status = this.statusCache.get(rowid);
|
|
307
|
+
if (status && excluded.has(status))
|
|
308
|
+
continue;
|
|
309
|
+
}
|
|
240
310
|
const similarity = (0, embeddings_js_1.cosineSimilarity)(candidate, queryVector);
|
|
241
311
|
if (bestMatches.length < effectiveLimit) {
|
|
242
312
|
bestMatches.push({ rowid, similarity, distance: 1 - similarity });
|
|
@@ -269,9 +339,7 @@ class NodeSQLiteAdapter extends base_adapter_js_1.DatabaseAdapter {
|
|
|
269
339
|
const result = stmt.run(rowid, buffer);
|
|
270
340
|
// Keep in-memory caches in sync
|
|
271
341
|
this.vectorCache.set(rowid, vec);
|
|
272
|
-
|
|
273
|
-
if (topicRow)
|
|
274
|
-
this.topicCache.set(rowid, topicRow.topic);
|
|
342
|
+
this.refreshDecisionStatusCache(rowid);
|
|
275
343
|
return result;
|
|
276
344
|
}
|
|
277
345
|
getLastInsertRowid() {
|
|
@@ -331,6 +399,11 @@ class NodeSQLiteAdapter extends base_adapter_js_1.DatabaseAdapter {
|
|
|
331
399
|
(0, debug_logger_js_1.info)(`[node-sqlite-adapter] Migration ${file} recovered successfully`);
|
|
332
400
|
continue;
|
|
333
401
|
}
|
|
402
|
+
if (message.includes('duplicate column') && version === 39) {
|
|
403
|
+
this.recoverConnectorEventOperatorSeqMigration039();
|
|
404
|
+
(0, debug_logger_js_1.info)(`[node-sqlite-adapter] Migration ${file} recovered successfully`);
|
|
405
|
+
continue;
|
|
406
|
+
}
|
|
334
407
|
if (message.includes('duplicate column')) {
|
|
335
408
|
(0, debug_logger_js_1.warn)(`[node-sqlite-adapter] Migration ${file} skipped (duplicate column - already applied)`);
|
|
336
409
|
this.prepare('INSERT OR IGNORE INTO schema_version (version) VALUES (?)').run(version);
|
|
@@ -395,6 +468,18 @@ class NodeSQLiteAdapter extends base_adapter_js_1.DatabaseAdapter {
|
|
|
395
468
|
this.recoverConnectorEventScopeMigration034();
|
|
396
469
|
(0, debug_logger_js_1.info)('[node-sqlite-adapter] Repaired skipped connector event scope migration');
|
|
397
470
|
}
|
|
471
|
+
const connectorColumnsAfterScopeRepair = this.tableColumns('connector_event_index');
|
|
472
|
+
const hasMissingOperatorSeqFeature = !connectorColumnsAfterScopeRepair.has('operator_ingest_seq') ||
|
|
473
|
+
!this.tableExists('connector_event_index_operator_seq_cursors') ||
|
|
474
|
+
!this.indexExists('idx_connector_event_index_operator_scope_seq') ||
|
|
475
|
+
!this.indexExists('idx_connector_event_index_operator_cursor_order') ||
|
|
476
|
+
!this.triggerExists('trg_connector_event_index_operator_ingest_seq_ai') ||
|
|
477
|
+
!this.triggerExists('trg_connector_event_index_operator_ingest_seq_explicit_ai') ||
|
|
478
|
+
!this.schemaVersionExists(39);
|
|
479
|
+
if (hasMissingOperatorSeqFeature) {
|
|
480
|
+
this.recoverConnectorEventOperatorSeqMigration039();
|
|
481
|
+
(0, debug_logger_js_1.info)('[node-sqlite-adapter] Repaired skipped connector event operator sequence migration');
|
|
482
|
+
}
|
|
398
483
|
}
|
|
399
484
|
if (!this.tableExists('twin_edges')) {
|
|
400
485
|
this.applyRepairMigration(migrationsDir, '035-create-twin-edges.sql', 'twin edge ledger');
|
|
@@ -406,6 +491,33 @@ class NodeSQLiteAdapter extends base_adapter_js_1.DatabaseAdapter {
|
|
|
406
491
|
if (!this.tableExists('context_packets')) {
|
|
407
492
|
this.applyRepairMigration(migrationsDir, '037-create-context-packets.sql', 'context packet store');
|
|
408
493
|
}
|
|
494
|
+
// TOMBSTONE (M6, 2026-07): the vnext_* operator tables (vnext_operator_cursors,
|
|
495
|
+
// vnext_operator_commits, operator_no_updates, worker_proposals) lost their last
|
|
496
|
+
// living reader/writer when the vNext parallel runtime was deleted in M4 (PR #120).
|
|
497
|
+
// They are intentionally KEPT: this repair path re-creates them on any DB that
|
|
498
|
+
// skipped migration 038, vnext_operator_commits holds an FK to
|
|
499
|
+
// vnext_operator_cursors, and shipped migrations are append-only. Do not drop
|
|
500
|
+
// them without also removing this repair block, the 040/041 repair/asserts below,
|
|
501
|
+
// and the schema-contract tests that pin them.
|
|
502
|
+
if (!this.tableExists('vnext_operator_cursors') ||
|
|
503
|
+
!this.tableExists('vnext_operator_commits') ||
|
|
504
|
+
!this.tableExists('operator_no_updates') ||
|
|
505
|
+
!this.tableExists('worker_proposals')) {
|
|
506
|
+
this.applyRepairMigration(migrationsDir, '038-create-vnext-operator-contracts.sql', 'vNext operator contracts');
|
|
507
|
+
}
|
|
508
|
+
// TOMBSTONE (M6, 2026-07): operator_memory_commit_intents (migrations 040/041)
|
|
509
|
+
// has no living reader/writer since M4 (PR #120). Kept for the same reasons as
|
|
510
|
+
// the 038 family above; the fail-loud asserts below still protect personal DBs
|
|
511
|
+
// that skipped or corrupted these migrations.
|
|
512
|
+
if (!this.tableExists('operator_memory_commit_intents') ||
|
|
513
|
+
!this.indexExists('idx_operator_memory_commit_intents_cursor_created')) {
|
|
514
|
+
this.applyRepairMigration(migrationsDir, '040-create-operator-memory-commit-intents.sql', 'operator memory commit intents');
|
|
515
|
+
}
|
|
516
|
+
this.assertMigration040BaseComplete();
|
|
517
|
+
if (!this.hasOperatorMemoryCommitIntentClaimInvariant()) {
|
|
518
|
+
this.applyRepairMigration(migrationsDir, '041-enforce-operator-memory-commit-claim-invariant.sql', 'operator memory commit claim invariant');
|
|
519
|
+
}
|
|
520
|
+
this.assertMigration041Complete();
|
|
409
521
|
}
|
|
410
522
|
applyRepairMigration(migrationsDir, fileName, label) {
|
|
411
523
|
const migrationPath = path_1.default.join(migrationsDir, fileName);
|
|
@@ -425,6 +537,64 @@ class NodeSQLiteAdapter extends base_adapter_js_1.DatabaseAdapter {
|
|
|
425
537
|
throw new Error(`Repair migration ${fileName} failed: ${message}`);
|
|
426
538
|
}
|
|
427
539
|
}
|
|
540
|
+
operatorMemoryCommitIntentTableSql() {
|
|
541
|
+
const tableDefinition = this.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name = 'operator_memory_commit_intents'").get();
|
|
542
|
+
return tableDefinition?.sql ?? '';
|
|
543
|
+
}
|
|
544
|
+
hasOperatorMemoryCommitIntentClaimInvariant() {
|
|
545
|
+
const sql = this.operatorMemoryCommitIntentTableSql();
|
|
546
|
+
return (sql.includes("(status = 'saving' AND claim_token IS NOT NULL)") &&
|
|
547
|
+
sql.includes("(status != 'saving' AND claim_token IS NULL)"));
|
|
548
|
+
}
|
|
549
|
+
assertMigration040BaseComplete() {
|
|
550
|
+
if (!this.tableExists('operator_memory_commit_intents')) {
|
|
551
|
+
throw new Error('Migration 040 recovery failed: missing table operator_memory_commit_intents');
|
|
552
|
+
}
|
|
553
|
+
const columns = this.tableColumns('operator_memory_commit_intents');
|
|
554
|
+
for (const column of [
|
|
555
|
+
'intent_id',
|
|
556
|
+
'cursor_name',
|
|
557
|
+
'idempotency_key',
|
|
558
|
+
'expected_memory_count',
|
|
559
|
+
'memory_payload_hash',
|
|
560
|
+
'memory_ids_json',
|
|
561
|
+
'source_refs_json',
|
|
562
|
+
'status',
|
|
563
|
+
'claim_token',
|
|
564
|
+
'created_at_ms',
|
|
565
|
+
'updated_at_ms',
|
|
566
|
+
]) {
|
|
567
|
+
if (!columns.has(column)) {
|
|
568
|
+
throw new Error(`Migration 040 recovery failed: missing operator_memory_commit_intents.${column}`);
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
for (const indexName of ['idx_operator_memory_commit_intents_cursor_created']) {
|
|
572
|
+
if (!this.indexExists(indexName)) {
|
|
573
|
+
throw new Error(`Migration 040 recovery failed: missing index ${indexName}`);
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
const sql = this.operatorMemoryCommitIntentTableSql();
|
|
577
|
+
for (const fragment of [
|
|
578
|
+
'idempotency_key TEXT NOT NULL UNIQUE',
|
|
579
|
+
'expected_memory_count INTEGER NOT NULL CHECK (expected_memory_count > 0)',
|
|
580
|
+
"memory_payload_hash TEXT NOT NULL CHECK (memory_payload_hash LIKE 'sha256:%')",
|
|
581
|
+
'memory_ids_json TEXT NOT NULL CHECK (json_valid(memory_ids_json))',
|
|
582
|
+
'source_refs_json TEXT NOT NULL CHECK (json_valid(source_refs_json))',
|
|
583
|
+
"status TEXT NOT NULL CHECK (status IN ('pending', 'saving', 'saved', 'promoted'))",
|
|
584
|
+
'created_at_ms INTEGER NOT NULL CHECK (created_at_ms >= 0)',
|
|
585
|
+
'updated_at_ms INTEGER NOT NULL CHECK (updated_at_ms >= created_at_ms)',
|
|
586
|
+
]) {
|
|
587
|
+
if (!sql.includes(fragment)) {
|
|
588
|
+
throw new Error(`Migration 040 recovery failed: incompatible operator_memory_commit_intents table definition missing ${fragment}`);
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
assertMigration041Complete() {
|
|
593
|
+
this.assertMigration040BaseComplete();
|
|
594
|
+
if (!this.hasOperatorMemoryCommitIntentClaimInvariant()) {
|
|
595
|
+
throw new Error('Migration 041 recovery failed: incompatible operator_memory_commit_intents table definition missing claim invariant');
|
|
596
|
+
}
|
|
597
|
+
}
|
|
428
598
|
recoverMemoryProvenanceMigration032() {
|
|
429
599
|
this.transaction(() => {
|
|
430
600
|
const expectedDecisionColumns = [
|
|
@@ -534,6 +704,142 @@ class NodeSQLiteAdapter extends base_adapter_js_1.DatabaseAdapter {
|
|
|
534
704
|
}
|
|
535
705
|
}
|
|
536
706
|
}
|
|
707
|
+
recoverConnectorEventOperatorSeqMigration039() {
|
|
708
|
+
this.transaction(() => {
|
|
709
|
+
const columns = this.tableColumns('connector_event_index');
|
|
710
|
+
if (!columns.has('operator_ingest_seq')) {
|
|
711
|
+
this.exec(`
|
|
712
|
+
ALTER TABLE connector_event_index
|
|
713
|
+
ADD COLUMN operator_ingest_seq INTEGER CHECK (
|
|
714
|
+
operator_ingest_seq IS NULL OR operator_ingest_seq >= 1
|
|
715
|
+
)
|
|
716
|
+
`);
|
|
717
|
+
}
|
|
718
|
+
this.exec(`
|
|
719
|
+
CREATE TABLE IF NOT EXISTS connector_event_index_operator_seq_cursors (
|
|
720
|
+
source_connector TEXT NOT NULL,
|
|
721
|
+
channel TEXT NOT NULL DEFAULT '',
|
|
722
|
+
next_seq INTEGER NOT NULL CHECK (next_seq >= 1),
|
|
723
|
+
PRIMARY KEY (source_connector, channel)
|
|
724
|
+
)
|
|
725
|
+
`);
|
|
726
|
+
this.exec(`
|
|
727
|
+
WITH ranked_events AS (
|
|
728
|
+
SELECT
|
|
729
|
+
event_index_id,
|
|
730
|
+
ROW_NUMBER() OVER (
|
|
731
|
+
PARTITION BY source_connector, COALESCE(channel, '')
|
|
732
|
+
ORDER BY rowid ASC
|
|
733
|
+
) AS operator_seq
|
|
734
|
+
FROM connector_event_index
|
|
735
|
+
)
|
|
736
|
+
UPDATE connector_event_index
|
|
737
|
+
SET operator_ingest_seq = (
|
|
738
|
+
SELECT operator_seq
|
|
739
|
+
FROM ranked_events
|
|
740
|
+
WHERE ranked_events.event_index_id = connector_event_index.event_index_id
|
|
741
|
+
)
|
|
742
|
+
WHERE operator_ingest_seq IS NULL
|
|
743
|
+
`);
|
|
744
|
+
this.exec(`
|
|
745
|
+
INSERT OR IGNORE INTO connector_event_index_operator_seq_cursors (
|
|
746
|
+
source_connector,
|
|
747
|
+
channel,
|
|
748
|
+
next_seq
|
|
749
|
+
)
|
|
750
|
+
SELECT
|
|
751
|
+
source_connector,
|
|
752
|
+
COALESCE(channel, ''),
|
|
753
|
+
COALESCE(MAX(operator_ingest_seq), 0) + 1
|
|
754
|
+
FROM connector_event_index
|
|
755
|
+
GROUP BY source_connector, COALESCE(channel, '')
|
|
756
|
+
`);
|
|
757
|
+
this.exec(`
|
|
758
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_connector_event_index_operator_scope_seq
|
|
759
|
+
ON connector_event_index(source_connector, COALESCE(channel, ''), operator_ingest_seq)
|
|
760
|
+
WHERE operator_ingest_seq IS NOT NULL
|
|
761
|
+
`);
|
|
762
|
+
this.exec(`
|
|
763
|
+
CREATE INDEX IF NOT EXISTS idx_connector_event_index_operator_cursor_order
|
|
764
|
+
ON connector_event_index(source_connector, channel, operator_ingest_seq)
|
|
765
|
+
`);
|
|
766
|
+
this.exec(`
|
|
767
|
+
CREATE TRIGGER IF NOT EXISTS trg_connector_event_index_operator_ingest_seq_ai
|
|
768
|
+
AFTER INSERT ON connector_event_index
|
|
769
|
+
WHEN NEW.operator_ingest_seq IS NULL
|
|
770
|
+
BEGIN
|
|
771
|
+
INSERT OR IGNORE INTO connector_event_index_operator_seq_cursors (
|
|
772
|
+
source_connector,
|
|
773
|
+
channel,
|
|
774
|
+
next_seq
|
|
775
|
+
)
|
|
776
|
+
VALUES (NEW.source_connector, COALESCE(NEW.channel, ''), 1);
|
|
777
|
+
|
|
778
|
+
UPDATE connector_event_index
|
|
779
|
+
SET operator_ingest_seq = (
|
|
780
|
+
SELECT next_seq
|
|
781
|
+
FROM connector_event_index_operator_seq_cursors
|
|
782
|
+
WHERE source_connector = NEW.source_connector
|
|
783
|
+
AND channel = COALESCE(NEW.channel, '')
|
|
784
|
+
)
|
|
785
|
+
WHERE event_index_id = NEW.event_index_id;
|
|
786
|
+
|
|
787
|
+
UPDATE connector_event_index_operator_seq_cursors
|
|
788
|
+
SET next_seq = next_seq + 1
|
|
789
|
+
WHERE source_connector = NEW.source_connector
|
|
790
|
+
AND channel = COALESCE(NEW.channel, '');
|
|
791
|
+
END
|
|
792
|
+
`);
|
|
793
|
+
this.exec(`
|
|
794
|
+
CREATE TRIGGER IF NOT EXISTS trg_connector_event_index_operator_ingest_seq_explicit_ai
|
|
795
|
+
AFTER INSERT ON connector_event_index
|
|
796
|
+
WHEN NEW.operator_ingest_seq IS NOT NULL
|
|
797
|
+
BEGIN
|
|
798
|
+
INSERT OR IGNORE INTO connector_event_index_operator_seq_cursors (
|
|
799
|
+
source_connector,
|
|
800
|
+
channel,
|
|
801
|
+
next_seq
|
|
802
|
+
)
|
|
803
|
+
VALUES (NEW.source_connector, COALESCE(NEW.channel, ''), 1);
|
|
804
|
+
|
|
805
|
+
UPDATE connector_event_index_operator_seq_cursors
|
|
806
|
+
SET next_seq = CASE
|
|
807
|
+
WHEN next_seq <= NEW.operator_ingest_seq THEN NEW.operator_ingest_seq + 1
|
|
808
|
+
ELSE next_seq
|
|
809
|
+
END
|
|
810
|
+
WHERE source_connector = NEW.source_connector
|
|
811
|
+
AND channel = COALESCE(NEW.channel, '');
|
|
812
|
+
END
|
|
813
|
+
`);
|
|
814
|
+
this.assertMigration039Complete();
|
|
815
|
+
this.prepare('INSERT OR IGNORE INTO schema_version (version, description) VALUES (?, ?)').run(39, 'Add connector event operator ingest sequence');
|
|
816
|
+
});
|
|
817
|
+
}
|
|
818
|
+
assertMigration039Complete() {
|
|
819
|
+
const columns = this.tableColumns('connector_event_index');
|
|
820
|
+
if (!columns.has('operator_ingest_seq')) {
|
|
821
|
+
throw new Error('Migration 039 recovery failed: missing connector_event_index.operator_ingest_seq');
|
|
822
|
+
}
|
|
823
|
+
if (!this.tableExists('connector_event_index_operator_seq_cursors')) {
|
|
824
|
+
throw new Error('Migration 039 recovery failed: missing connector_event_index_operator_seq_cursors');
|
|
825
|
+
}
|
|
826
|
+
for (const indexName of [
|
|
827
|
+
'idx_connector_event_index_operator_scope_seq',
|
|
828
|
+
'idx_connector_event_index_operator_cursor_order',
|
|
829
|
+
]) {
|
|
830
|
+
if (!this.indexExists(indexName)) {
|
|
831
|
+
throw new Error(`Migration 039 recovery failed: missing index ${indexName}`);
|
|
832
|
+
}
|
|
833
|
+
}
|
|
834
|
+
for (const triggerName of [
|
|
835
|
+
'trg_connector_event_index_operator_ingest_seq_ai',
|
|
836
|
+
'trg_connector_event_index_operator_ingest_seq_explicit_ai',
|
|
837
|
+
]) {
|
|
838
|
+
if (!this.triggerExists(triggerName)) {
|
|
839
|
+
throw new Error(`Migration 039 recovery failed: missing trigger ${triggerName}`);
|
|
840
|
+
}
|
|
841
|
+
}
|
|
842
|
+
}
|
|
537
843
|
tableColumns(tableName) {
|
|
538
844
|
if (!SQLITE_IDENTIFIER_PATTERN.test(tableName)) {
|
|
539
845
|
throw new Error('Invalid SQLite table identifier');
|
|
@@ -552,6 +858,17 @@ class NodeSQLiteAdapter extends base_adapter_js_1.DatabaseAdapter {
|
|
|
552
858
|
const row = this.prepare("SELECT name FROM sqlite_master WHERE type='index' AND name = ?").get(indexName);
|
|
553
859
|
return row?.name === indexName;
|
|
554
860
|
}
|
|
861
|
+
triggerExists(triggerName) {
|
|
862
|
+
const row = this.prepare("SELECT name FROM sqlite_master WHERE type='trigger' AND name = ?").get(triggerName);
|
|
863
|
+
return row?.name === triggerName;
|
|
864
|
+
}
|
|
865
|
+
schemaVersionExists(version) {
|
|
866
|
+
if (!this.tableExists('schema_version')) {
|
|
867
|
+
return false;
|
|
868
|
+
}
|
|
869
|
+
const row = this.prepare('SELECT version FROM schema_version WHERE version = ?').get(version);
|
|
870
|
+
return row?.version === version;
|
|
871
|
+
}
|
|
555
872
|
migrateFromVssMemories() {
|
|
556
873
|
try {
|
|
557
874
|
const vssTables = this.prepare(`SELECT name FROM sqlite_master WHERE name='vss_memories'`).all();
|
package/dist/db-manager.d.ts
CHANGED
|
@@ -26,9 +26,10 @@ export interface DatabaseAdapter {
|
|
|
26
26
|
prepare: (sql: string) => PreparedStatement;
|
|
27
27
|
transaction: <T>(fn: () => T) => T;
|
|
28
28
|
insertEmbedding: (rowid: number, embedding: Float32Array | number[]) => void;
|
|
29
|
-
vectorSearch: (embedding: Float32Array | number[], limit: number, topicPrefix?: string) => Promise<VectorSearchResult[] | null> | VectorSearchResult[] | null;
|
|
29
|
+
vectorSearch: (embedding: Float32Array | number[], limit: number, topicPrefix?: string, excludeStatuses?: readonly string[]) => Promise<VectorSearchResult[] | null> | VectorSearchResult[] | null;
|
|
30
30
|
vectorSearchEnabled: boolean;
|
|
31
31
|
reloadVectorCache?: () => void;
|
|
32
|
+
refreshDecisionStatusCache?: (rowid: number) => void;
|
|
32
33
|
getDbPath?: () => string;
|
|
33
34
|
dbPath?: string;
|
|
34
35
|
constructor: {
|
|
@@ -110,16 +111,14 @@ export interface SemanticEdges {
|
|
|
110
111
|
synthesized_by: SemanticEdgeItem[];
|
|
111
112
|
}
|
|
112
113
|
/**
|
|
113
|
-
*
|
|
114
|
+
* Fail loud if stored vectors predate the e5 query/passage prefix scheme.
|
|
114
115
|
*
|
|
115
|
-
*
|
|
116
|
-
*
|
|
117
|
-
*
|
|
118
|
-
*
|
|
119
|
-
* to prevent multiple adapters/migrations running simultaneously.
|
|
120
|
-
*
|
|
121
|
-
* @returns SQLite database connection
|
|
116
|
+
* Migration 042 records 'e5-prefixed-v1' on a fresh DB and 'legacy-unprefixed' on a DB
|
|
117
|
+
* that already held vectors at upgrade time. If the marker is not current AND legacy
|
|
118
|
+
* vectors exist, cosine search would be non-discriminative, so we throw with the exact
|
|
119
|
+
* re-embed command instead of silently serving degraded results. No-fallback by design.
|
|
122
120
|
*/
|
|
121
|
+
export declare function assertEmbeddingSchemeCurrent(adapter: DatabaseAdapter): void;
|
|
123
122
|
export declare function initDB(): Promise<unknown>;
|
|
124
123
|
export declare function assertTestProcessIsNotUsingRealDb(effectivePath?: string, effectivePathSource?: string): void;
|
|
125
124
|
/**
|
|
@@ -172,7 +171,7 @@ export declare function insertEmbedding(decisionRowid: number, embedding: Float3
|
|
|
172
171
|
* @param threshold - Minimum similarity threshold (default: 0.7)
|
|
173
172
|
* @returns Array of decisions with similarity scores, or empty array
|
|
174
173
|
*/
|
|
175
|
-
export declare function vectorSearch(queryEmbedding: Float32Array | number[], limit?: number, threshold?: number, topicPrefix?: string): Promise<DecisionRecord[]>;
|
|
174
|
+
export declare function vectorSearch(queryEmbedding: Float32Array | number[], limit?: number, threshold?: number, topicPrefix?: string, excludeStatuses?: readonly string[]): Promise<DecisionRecord[]>;
|
|
176
175
|
/**
|
|
177
176
|
* Decision input for storage
|
|
178
177
|
*/
|
package/dist/db-manager.js
CHANGED
|
@@ -22,6 +22,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
22
22
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
23
23
|
};
|
|
24
24
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
25
|
+
exports.assertEmbeddingSchemeCurrent = assertEmbeddingSchemeCurrent;
|
|
25
26
|
exports.initDB = initDB;
|
|
26
27
|
exports.assertTestProcessIsNotUsingRealDb = assertTestProcessIsNotUsingRealDb;
|
|
27
28
|
exports.getDB = getDB;
|
|
@@ -48,6 +49,7 @@ const os_1 = __importDefault(require("os"));
|
|
|
48
49
|
const debug_logger_js_1 = require("./debug-logger.js");
|
|
49
50
|
const progress_indicator_js_1 = require("./progress-indicator.js");
|
|
50
51
|
const index_js_1 = require("./db-adapter/index.js");
|
|
52
|
+
const embeddings_js_1 = require("./embeddings.js");
|
|
51
53
|
// Database adapter instance (singleton)
|
|
52
54
|
let dbAdapter = null;
|
|
53
55
|
let dbConnection = null;
|
|
@@ -67,6 +69,56 @@ const REAL_USER_DB_PATH = path_1.default.join(os_1.default.homedir(), '.claude',
|
|
|
67
69
|
*
|
|
68
70
|
* @returns SQLite database connection
|
|
69
71
|
*/
|
|
72
|
+
function countRows(adapter, table) {
|
|
73
|
+
try {
|
|
74
|
+
const row = adapter.prepare(`SELECT COUNT(*) AS n FROM ${table}`).get();
|
|
75
|
+
return row?.n ?? 0;
|
|
76
|
+
}
|
|
77
|
+
catch {
|
|
78
|
+
return 0; // table may not exist on very old DBs
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Fail loud if stored vectors predate the e5 query/passage prefix scheme.
|
|
83
|
+
*
|
|
84
|
+
* Migration 042 records 'e5-prefixed-v1' on a fresh DB and 'legacy-unprefixed' on a DB
|
|
85
|
+
* that already held vectors at upgrade time. If the marker is not current AND legacy
|
|
86
|
+
* vectors exist, cosine search would be non-discriminative, so we throw with the exact
|
|
87
|
+
* re-embed command instead of silently serving degraded results. No-fallback by design.
|
|
88
|
+
*/
|
|
89
|
+
function assertEmbeddingSchemeCurrent(adapter) {
|
|
90
|
+
let scheme = 'legacy-unprefixed';
|
|
91
|
+
try {
|
|
92
|
+
const row = adapter
|
|
93
|
+
.prepare("SELECT value FROM embedding_meta WHERE key = 'embedding_prefix_scheme'")
|
|
94
|
+
.get();
|
|
95
|
+
if (row?.value)
|
|
96
|
+
scheme = row.value;
|
|
97
|
+
}
|
|
98
|
+
catch {
|
|
99
|
+
// embedding_meta missing (pre-042). runMigrations creates it, so this path
|
|
100
|
+
// only hits if migrations did not run; fall through to the vector check.
|
|
101
|
+
}
|
|
102
|
+
if (scheme === embeddings_js_1.EMBEDDING_PREFIX_SCHEME)
|
|
103
|
+
return; // current
|
|
104
|
+
const legacyVectors = countRows(adapter, 'embeddings') + countRows(adapter, 'wiki_page_embeddings');
|
|
105
|
+
if (legacyVectors > 0) {
|
|
106
|
+
const dbPath = resolveAdapterDbPath(adapter) ?? '<db>';
|
|
107
|
+
throw new Error(`[embedding-guard] ${dbPath} holds ${legacyVectors} vectors written WITHOUT the e5 ` +
|
|
108
|
+
`query/passage prefix (scheme='${scheme}', code requires '${embeddings_js_1.EMBEDDING_PREFIX_SCHEME}'). ` +
|
|
109
|
+
`Cosine search would be non-discriminative. Re-embed before starting:\n` +
|
|
110
|
+
` MAMA_DB_PATH="${dbPath}" node packages/mama-core/scripts/re-embed-migration.mjs`);
|
|
111
|
+
}
|
|
112
|
+
// No legacy vectors present: safe. Upgrade the marker so we do not re-check.
|
|
113
|
+
try {
|
|
114
|
+
adapter
|
|
115
|
+
.prepare("INSERT OR REPLACE INTO embedding_meta (key, value) VALUES ('embedding_prefix_scheme', ?)")
|
|
116
|
+
.run(embeddings_js_1.EMBEDDING_PREFIX_SCHEME);
|
|
117
|
+
}
|
|
118
|
+
catch {
|
|
119
|
+
// best-effort; if embedding_meta truly missing, nothing to guard yet
|
|
120
|
+
}
|
|
121
|
+
}
|
|
70
122
|
async function initDB() {
|
|
71
123
|
assertTestProcessIsNotUsingRealDb();
|
|
72
124
|
// Already initialized - return immediately
|
|
@@ -92,6 +144,7 @@ async function initDB() {
|
|
|
92
144
|
if (typeof dbAdapter.reloadVectorCache === 'function') {
|
|
93
145
|
dbAdapter.reloadVectorCache();
|
|
94
146
|
}
|
|
147
|
+
assertEmbeddingSchemeCurrent(dbAdapter); // fail loud on legacy vectors + new code
|
|
95
148
|
isInitialized = true;
|
|
96
149
|
(0, debug_logger_js_1.info)(`[db-manager] Database initialized (${dbAdapter.constructor.name})`);
|
|
97
150
|
(0, progress_indicator_js_1.logComplete)('Database ready');
|
|
@@ -259,11 +312,11 @@ async function insertEmbedding(decisionRowid, embedding) {
|
|
|
259
312
|
* @param threshold - Minimum similarity threshold (default: 0.7)
|
|
260
313
|
* @returns Array of decisions with similarity scores, or empty array
|
|
261
314
|
*/
|
|
262
|
-
async function vectorSearch(queryEmbedding, limit = 5, threshold = 0.7, topicPrefix) {
|
|
315
|
+
async function vectorSearch(queryEmbedding, limit = 5, threshold = 0.7, topicPrefix, excludeStatuses) {
|
|
263
316
|
const adapter = getAdapter();
|
|
264
317
|
try {
|
|
265
|
-
// Brute-force cosine similarity over all embeddings (with optional topic pre-filter)
|
|
266
|
-
const results = await adapter.vectorSearch(queryEmbedding, limit * 3, topicPrefix);
|
|
318
|
+
// Brute-force cosine similarity over all embeddings (with optional topic/status pre-filter)
|
|
319
|
+
const results = await adapter.vectorSearch(queryEmbedding, limit * 3, topicPrefix, excludeStatuses);
|
|
267
320
|
if (!results || results.length === 0) {
|
|
268
321
|
return []; // No keyword fallback - fast fail
|
|
269
322
|
}
|
|
@@ -327,6 +380,7 @@ async function insertDecisionWithEmbedding(decision) {
|
|
|
327
380
|
(0, debug_logger_js_1.info)(`[db-manager] Generating embedding for decision (topic length: ${decision.topic?.length || 0})`);
|
|
328
381
|
let embedding = null;
|
|
329
382
|
try {
|
|
383
|
+
// e5 passage role (default) - stored vector
|
|
330
384
|
embedding = await generateEnhancedEmbedding({
|
|
331
385
|
topic: decision.topic,
|
|
332
386
|
decision: decision.decision,
|
|
@@ -570,7 +624,7 @@ async function queryVectorSearch(params) {
|
|
|
570
624
|
const { generateEmbedding } = await import('./embeddings.js');
|
|
571
625
|
try {
|
|
572
626
|
// Generate embedding for query
|
|
573
|
-
const embedding = await generateEmbedding(query);
|
|
627
|
+
const embedding = await generateEmbedding(query, 'query');
|
|
574
628
|
const cutoffTime = Date.now() - timeWindow;
|
|
575
629
|
const candidates = await adapter.vectorSearch(embedding, limit * 5);
|
|
576
630
|
if (!candidates || candidates.length === 0) {
|
|
@@ -777,6 +831,7 @@ async function reindexEmbeddings(onProgress) {
|
|
|
777
831
|
let completed = 0;
|
|
778
832
|
for (const row of decisions) {
|
|
779
833
|
try {
|
|
834
|
+
// e5 passage role (default) - stored vector
|
|
780
835
|
const embedding = await generateEnhancedEmbedding({
|
|
781
836
|
topic: row.topic,
|
|
782
837
|
decision: row.decision,
|
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
*
|
|
12
12
|
* @module embedding-client
|
|
13
13
|
*/
|
|
14
|
+
import type { EmbeddingRole } from './embeddings.js';
|
|
14
15
|
export declare const DEFAULT_PORT: number;
|
|
15
16
|
export declare const HOST = "127.0.0.1";
|
|
16
17
|
export declare const TIMEOUT_MS = 500;
|
|
@@ -33,9 +34,10 @@ export declare function isServerRunning(): Promise<boolean>;
|
|
|
33
34
|
* Generate embedding via HTTP server
|
|
34
35
|
*
|
|
35
36
|
* @param text - Text to embed
|
|
37
|
+
* @param role - e5 role: 'passage' (default, stored text) or 'query' (search)
|
|
36
38
|
* @returns Embedding or null if failed
|
|
37
39
|
*/
|
|
38
|
-
export declare function getEmbeddingFromServer(text: string): Promise<Float32Array | null>;
|
|
40
|
+
export declare function getEmbeddingFromServer(text: string, role?: EmbeddingRole): Promise<Float32Array | null>;
|
|
39
41
|
/**
|
|
40
42
|
* Get server status
|
|
41
43
|
*/
|
package/dist/embedding-client.js
CHANGED
|
@@ -76,9 +76,10 @@ async function isServerRunning() {
|
|
|
76
76
|
* Generate embedding via HTTP server
|
|
77
77
|
*
|
|
78
78
|
* @param text - Text to embed
|
|
79
|
+
* @param role - e5 role: 'passage' (default, stored text) or 'query' (search)
|
|
79
80
|
* @returns Embedding or null if failed
|
|
80
81
|
*/
|
|
81
|
-
async function getEmbeddingFromServer(text) {
|
|
82
|
+
async function getEmbeddingFromServer(text, role = 'passage') {
|
|
82
83
|
const port = getServerPort();
|
|
83
84
|
try {
|
|
84
85
|
const controller = new AbortController();
|
|
@@ -86,7 +87,7 @@ async function getEmbeddingFromServer(text) {
|
|
|
86
87
|
const response = await fetch(`http://${exports.HOST}:${port}/embed`, {
|
|
87
88
|
method: 'POST',
|
|
88
89
|
headers: { 'Content-Type': 'application/json' },
|
|
89
|
-
body: JSON.stringify({ text }),
|
|
90
|
+
body: JSON.stringify({ text, role }),
|
|
90
91
|
signal: controller.signal,
|
|
91
92
|
});
|
|
92
93
|
clearTimeout(timeout);
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
*/
|
|
15
15
|
import type { IncomingMessage, ServerResponse, Server as HTTPServer } from 'http';
|
|
16
16
|
export declare const SHUTDOWN_TOKEN: string;
|
|
17
|
+
import type { EmbeddingRole } from '../embeddings.js';
|
|
17
18
|
export declare const DEFAULT_PORT: number;
|
|
18
19
|
export declare const HOST = "127.0.0.1";
|
|
19
20
|
export declare const PORT_FILE: string;
|
|
@@ -42,6 +43,7 @@ export interface ServerOptions {
|
|
|
42
43
|
sessionStore?: SessionStore;
|
|
43
44
|
graphHandler?: RequestHandler;
|
|
44
45
|
}
|
|
46
|
+
export declare function resolveEmbeddingRole(raw: unknown): EmbeddingRole;
|
|
45
47
|
/**
|
|
46
48
|
* Clean up port file on exit
|
|
47
49
|
*/
|