@devflow-tools/memory-engine 0.16.10 → 0.16.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/embedding-model-store.d.ts +67 -0
- package/dist/embedding-model-store.d.ts.map +1 -0
- package/dist/embedding-model-store.js +350 -0
- package/dist/embedding-model-store.js.map +1 -0
- package/dist/embedding-provider.d.ts +8 -0
- package/dist/embedding-provider.d.ts.map +1 -1
- package/dist/embedding-provider.js +68 -11
- package/dist/embedding-provider.js.map +1 -1
- package/dist/hybrid-search.d.ts +8 -0
- package/dist/hybrid-search.d.ts.map +1 -1
- package/dist/hybrid-search.js +37 -10
- package/dist/hybrid-search.js.map +1 -1
- package/dist/index.d.ts +5 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -2
- package/dist/index.js.map +1 -1
- package/dist/memory-engine.d.ts +4 -0
- package/dist/memory-engine.d.ts.map +1 -1
- package/dist/memory-engine.js +62 -39
- package/dist/memory-engine.js.map +1 -1
- package/dist/memory-gate.d.ts +3 -1
- package/dist/memory-gate.d.ts.map +1 -1
- package/dist/memory-gate.js +22 -40
- package/dist/memory-gate.js.map +1 -1
- package/dist/memory-maintenance.d.ts.map +1 -1
- package/dist/memory-maintenance.js +4 -2
- package/dist/memory-maintenance.js.map +1 -1
- package/dist/memory-store.d.ts +38 -1
- package/dist/memory-store.d.ts.map +1 -1
- package/dist/memory-store.js +488 -5
- package/dist/memory-store.js.map +1 -1
- package/dist/migrations/v3-to-v4.d.ts.map +1 -1
- package/dist/migrations/v3-to-v4.js +4 -0
- package/dist/migrations/v3-to-v4.js.map +1 -1
- package/dist/migrations/v6-to-v7.d.ts +3 -0
- package/dist/migrations/v6-to-v7.d.ts.map +1 -0
- package/dist/migrations/v6-to-v7.js +18 -0
- package/dist/migrations/v6-to-v7.js.map +1 -0
- package/dist/migrations/v6-to-v7.test.d.ts +2 -0
- package/dist/migrations/v6-to-v7.test.d.ts.map +1 -0
- package/dist/migrations/v6-to-v7.test.js +33 -0
- package/dist/migrations/v6-to-v7.test.js.map +1 -0
- package/dist/observation-types.d.ts +2 -0
- package/dist/observation-types.d.ts.map +1 -1
- package/dist/observation-types.js +13 -0
- package/dist/observation-types.js.map +1 -1
- package/dist/preference-conflicts.d.ts +19 -0
- package/dist/preference-conflicts.d.ts.map +1 -0
- package/dist/preference-conflicts.js +293 -0
- package/dist/preference-conflicts.js.map +1 -0
- package/dist/preference-conflicts.test.d.ts +2 -0
- package/dist/preference-conflicts.test.d.ts.map +1 -0
- package/dist/preference-conflicts.test.js +72 -0
- package/dist/preference-conflicts.test.js.map +1 -0
- package/package.json +4 -4
package/dist/memory-store.js
CHANGED
|
@@ -2,11 +2,13 @@ import Database from 'better-sqlite3';
|
|
|
2
2
|
import * as sqliteVec from 'sqlite-vec';
|
|
3
3
|
import { dirname, join } from 'node:path';
|
|
4
4
|
import { existsSync, mkdirSync } from 'node:fs';
|
|
5
|
-
import { randomUUID } from 'node:crypto';
|
|
5
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
6
6
|
import { applyV2ToV3Migration } from './migrations/v2-to-v3.js';
|
|
7
7
|
import { applyV3ToV4Migration } from './migrations/v3-to-v4.js';
|
|
8
8
|
import { applyV4ToV5Migration } from './migrations/v4-to-v5.js';
|
|
9
|
+
import { applyV6ToV7Migration } from './migrations/v6-to-v7.js';
|
|
9
10
|
import { observationDedupKey, } from './observation-types.js';
|
|
11
|
+
import { analysesConflict, analyzePreference, analyzePreferenceObservation, } from './preference-conflicts.js';
|
|
10
12
|
// ── Schema ─────────────────────────────────────────────
|
|
11
13
|
const SCHEMA_V2 = `
|
|
12
14
|
-- ============================================================
|
|
@@ -169,7 +171,8 @@ CREATE TABLE IF NOT EXISTS observations (
|
|
|
169
171
|
source_event_ids TEXT NOT NULL DEFAULT '[]',
|
|
170
172
|
memory_id TEXT,
|
|
171
173
|
embedding BLOB,
|
|
172
|
-
timestamp INTEGER NOT NULL
|
|
174
|
+
timestamp INTEGER NOT NULL,
|
|
175
|
+
explicit_accepted_at INTEGER
|
|
173
176
|
);
|
|
174
177
|
CREATE INDEX IF NOT EXISTS idx_observations_session ON observations(session_id);
|
|
175
178
|
CREATE INDEX IF NOT EXISTS idx_observations_project ON observations(project);
|
|
@@ -256,6 +259,126 @@ CREATE TRIGGER IF NOT EXISTS memories_au AFTER UPDATE ON memories BEGIN
|
|
|
256
259
|
VALUES (new.rowid, new.id, new.content, new.scope);
|
|
257
260
|
END;
|
|
258
261
|
`;
|
|
262
|
+
const OBSERVATION_FTS_SCHEMA = `
|
|
263
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS observations_fts USING fts5(
|
|
264
|
+
id UNINDEXED,
|
|
265
|
+
title,
|
|
266
|
+
narrative,
|
|
267
|
+
facts,
|
|
268
|
+
concepts,
|
|
269
|
+
tokenize='trigram'
|
|
270
|
+
);
|
|
271
|
+
|
|
272
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS observations_terms_fts USING fts5(
|
|
273
|
+
id UNINDEXED,
|
|
274
|
+
title,
|
|
275
|
+
narrative,
|
|
276
|
+
facts,
|
|
277
|
+
concepts,
|
|
278
|
+
tokenize = "unicode61 tokenchars '#+'"
|
|
279
|
+
);
|
|
280
|
+
|
|
281
|
+
CREATE TRIGGER IF NOT EXISTS observations_fts_ai
|
|
282
|
+
AFTER INSERT ON observations
|
|
283
|
+
WHEN new.quality_status = 'accepted' AND new.resolution_state = 'current'
|
|
284
|
+
BEGIN
|
|
285
|
+
INSERT INTO observations_fts(rowid, id, title, narrative, facts, concepts)
|
|
286
|
+
VALUES (new.rowid, new.id, new.title, new.narrative, new.facts, new.concepts);
|
|
287
|
+
INSERT INTO observations_terms_fts(rowid, id, title, narrative, facts, concepts)
|
|
288
|
+
VALUES (new.rowid, new.id, new.title, new.narrative, new.facts, new.concepts);
|
|
289
|
+
END;
|
|
290
|
+
|
|
291
|
+
CREATE TRIGGER IF NOT EXISTS observations_fts_ad
|
|
292
|
+
AFTER DELETE ON observations
|
|
293
|
+
WHEN old.quality_status = 'accepted' AND old.resolution_state = 'current'
|
|
294
|
+
BEGIN
|
|
295
|
+
DELETE FROM observations_fts WHERE rowid = old.rowid;
|
|
296
|
+
DELETE FROM observations_terms_fts WHERE rowid = old.rowid;
|
|
297
|
+
END;
|
|
298
|
+
|
|
299
|
+
CREATE TRIGGER IF NOT EXISTS observations_fts_au
|
|
300
|
+
AFTER UPDATE OF title, narrative, facts, concepts, quality_status, resolution_state ON observations
|
|
301
|
+
BEGIN
|
|
302
|
+
DELETE FROM observations_fts WHERE rowid = old.rowid;
|
|
303
|
+
DELETE FROM observations_terms_fts WHERE rowid = old.rowid;
|
|
304
|
+
INSERT INTO observations_fts(rowid, id, title, narrative, facts, concepts)
|
|
305
|
+
SELECT new.rowid, new.id, new.title, new.narrative, new.facts, new.concepts
|
|
306
|
+
WHERE new.quality_status = 'accepted' AND new.resolution_state = 'current';
|
|
307
|
+
INSERT INTO observations_terms_fts(rowid, id, title, narrative, facts, concepts)
|
|
308
|
+
SELECT new.rowid, new.id, new.title, new.narrative, new.facts, new.concepts
|
|
309
|
+
WHERE new.quality_status = 'accepted' AND new.resolution_state = 'current';
|
|
310
|
+
END;
|
|
311
|
+
`;
|
|
312
|
+
const MAX_OBSERVATION_FTS_QUERY_LENGTH = 512;
|
|
313
|
+
const MAX_OBSERVATION_FTS_TERMS = 16;
|
|
314
|
+
const MAX_OBSERVATION_FTS_TERM_LENGTH = 64;
|
|
315
|
+
const OBSERVATION_FTS_SCHEMA_VERSION = 6;
|
|
316
|
+
const OBSERVATION_EXCLUSION_CTE = `
|
|
317
|
+
WITH excluded_observation_ids(id) AS (
|
|
318
|
+
SELECT CAST(value AS TEXT) FROM json_each(?)
|
|
319
|
+
)
|
|
320
|
+
`;
|
|
321
|
+
function serializeObservationSearchExcludedIds(excludedIds) {
|
|
322
|
+
return JSON.stringify(excludedIds ? [...excludedIds] : []);
|
|
323
|
+
}
|
|
324
|
+
function normalizeObservationLexicalQuery(query) {
|
|
325
|
+
const phrase = query
|
|
326
|
+
.slice(0, MAX_OBSERVATION_FTS_QUERY_LENGTH)
|
|
327
|
+
.normalize('NFKC')
|
|
328
|
+
.slice(0, MAX_OBSERVATION_FTS_QUERY_LENGTH)
|
|
329
|
+
.replace(/\s+/gu, ' ')
|
|
330
|
+
.trim();
|
|
331
|
+
if (!phrase)
|
|
332
|
+
return { trigramQuery: null, termQuery: null };
|
|
333
|
+
const lexicalTerms = phrase.match(/[\p{L}\p{N}]+/gu) ?? [];
|
|
334
|
+
const compact = lexicalTerms.join('');
|
|
335
|
+
const candidates = [compact, phrase, ...lexicalTerms];
|
|
336
|
+
const trigramTerms = [];
|
|
337
|
+
const termTerms = [];
|
|
338
|
+
const seenTrigrams = new Set();
|
|
339
|
+
const seenTerms = new Set();
|
|
340
|
+
for (const candidate of candidates) {
|
|
341
|
+
const term = [...candidate].slice(0, MAX_OBSERVATION_FTS_TERM_LENGTH).join('');
|
|
342
|
+
const searchableCharacters = term.match(/[\p{L}\p{N}]/gu) ?? [];
|
|
343
|
+
const normalized = term.toLocaleLowerCase('en-US');
|
|
344
|
+
const quoted = `"${term.replaceAll('"', '""')}"`;
|
|
345
|
+
if (searchableCharacters.length >= 3 && !seenTrigrams.has(normalized)) {
|
|
346
|
+
seenTrigrams.add(normalized);
|
|
347
|
+
trigramTerms.push(quoted);
|
|
348
|
+
}
|
|
349
|
+
if (searchableCharacters.length >= 1 && !seenTerms.has(normalized)) {
|
|
350
|
+
seenTerms.add(normalized);
|
|
351
|
+
termTerms.push(quoted);
|
|
352
|
+
}
|
|
353
|
+
if (trigramTerms.length >= MAX_OBSERVATION_FTS_TERMS
|
|
354
|
+
&& termTerms.length >= MAX_OBSERVATION_FTS_TERMS)
|
|
355
|
+
break;
|
|
356
|
+
}
|
|
357
|
+
return {
|
|
358
|
+
trigramQuery: trigramTerms.length > 0
|
|
359
|
+
? trigramTerms.slice(0, MAX_OBSERVATION_FTS_TERMS).join(' OR ')
|
|
360
|
+
: null,
|
|
361
|
+
termQuery: termTerms.length > 0
|
|
362
|
+
? termTerms.slice(0, MAX_OBSERVATION_FTS_TERMS).join(' OR ')
|
|
363
|
+
: null,
|
|
364
|
+
};
|
|
365
|
+
}
|
|
366
|
+
export function isRecoverableObservationFtsError(error) {
|
|
367
|
+
const candidate = error;
|
|
368
|
+
const code = typeof candidate?.code === 'string' ? candidate.code.toUpperCase() : '';
|
|
369
|
+
const message = typeof candidate?.message === 'string' ? candidate.message : '';
|
|
370
|
+
if (/^SQLITE_(?:BUSY|LOCKED|READONLY|IOERR|FULL|CANTOPEN|PERM)/.test(code))
|
|
371
|
+
return false;
|
|
372
|
+
if (code === 'SQLITE_CORRUPT_VTAB')
|
|
373
|
+
return true;
|
|
374
|
+
const referencesObservationFts = /observations_(?:terms_)?fts(?:_(?:data|idx|content|docsize|config))?/i
|
|
375
|
+
.test(message);
|
|
376
|
+
if (!referencesObservationFts)
|
|
377
|
+
return false;
|
|
378
|
+
if (/^SQLITE_CORRUPT(?:_|$)/.test(code))
|
|
379
|
+
return true;
|
|
380
|
+
return code === 'SQLITE_ERROR' && /no such table|malformed schema|vtable constructor failed/i.test(message);
|
|
381
|
+
}
|
|
259
382
|
// ── MemoryStore ─────────────────────────────────────────
|
|
260
383
|
export class MemoryStore {
|
|
261
384
|
db;
|
|
@@ -275,10 +398,13 @@ export class MemoryStore {
|
|
|
275
398
|
sqliteVec.load(this.db);
|
|
276
399
|
this.db.exec(SCHEMA_V2);
|
|
277
400
|
this.db.exec(FTS_TRIGGERS);
|
|
401
|
+
this.dropOrphanedObservationFtsTriggers();
|
|
278
402
|
this.migrateV1ToV2();
|
|
279
403
|
applyV2ToV3Migration(this.db);
|
|
280
404
|
applyV3ToV4Migration(this.db);
|
|
281
405
|
applyV4ToV5Migration(this.db);
|
|
406
|
+
this.ensureObservationFts();
|
|
407
|
+
applyV6ToV7Migration(this.db);
|
|
282
408
|
this.repairOrphanSessions(rootPath);
|
|
283
409
|
console.log('[MemoryStore] Schema initialized', {
|
|
284
410
|
dbPath: this.dbPath,
|
|
@@ -292,6 +418,66 @@ export class MemoryStore {
|
|
|
292
418
|
const row = this.db.prepare("SELECT COUNT(*) as c FROM sqlite_master WHERE type='table'").get();
|
|
293
419
|
return row.c;
|
|
294
420
|
}
|
|
421
|
+
dropOrphanedObservationFtsTriggers() {
|
|
422
|
+
const ftsTables = this.db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('observations_fts', 'observations_terms_fts')").all();
|
|
423
|
+
if (ftsTables.length === 2)
|
|
424
|
+
return;
|
|
425
|
+
this.db.exec(`
|
|
426
|
+
DROP TRIGGER IF EXISTS observations_fts_ai;
|
|
427
|
+
DROP TRIGGER IF EXISTS observations_fts_ad;
|
|
428
|
+
DROP TRIGGER IF EXISTS observations_fts_au;
|
|
429
|
+
`);
|
|
430
|
+
}
|
|
431
|
+
ensureObservationFts(forceRebuild = false) {
|
|
432
|
+
this.db.transaction(() => {
|
|
433
|
+
const ftsTableCount = this.db.prepare("SELECT COUNT(*) AS count FROM sqlite_master WHERE type = 'table' AND name IN ('observations_fts', 'observations_terms_fts')").get().count;
|
|
434
|
+
const migrationApplied = this.db.prepare('SELECT 1 FROM schema_versions WHERE version = ?').get(OBSERVATION_FTS_SCHEMA_VERSION) !== undefined;
|
|
435
|
+
const triggersCurrent = this.db.prepare(`
|
|
436
|
+
SELECT COUNT(*) AS count
|
|
437
|
+
FROM sqlite_master
|
|
438
|
+
WHERE type = 'trigger'
|
|
439
|
+
AND name IN ('observations_fts_ai', 'observations_fts_ad', 'observations_fts_au')
|
|
440
|
+
AND sql LIKE '%observations_terms_fts%'
|
|
441
|
+
`).get().count === 3;
|
|
442
|
+
const needsRebuild = forceRebuild
|
|
443
|
+
|| ftsTableCount !== 2
|
|
444
|
+
|| !migrationApplied
|
|
445
|
+
|| !triggersCurrent;
|
|
446
|
+
if (needsRebuild) {
|
|
447
|
+
this.db.exec(`
|
|
448
|
+
DROP TRIGGER IF EXISTS observations_fts_ai;
|
|
449
|
+
DROP TRIGGER IF EXISTS observations_fts_ad;
|
|
450
|
+
DROP TRIGGER IF EXISTS observations_fts_au;
|
|
451
|
+
`);
|
|
452
|
+
}
|
|
453
|
+
if (forceRebuild) {
|
|
454
|
+
this.db.exec(`
|
|
455
|
+
DROP TABLE IF EXISTS observations_fts;
|
|
456
|
+
DROP TABLE IF EXISTS observations_terms_fts;
|
|
457
|
+
`);
|
|
458
|
+
}
|
|
459
|
+
this.db.exec(OBSERVATION_FTS_SCHEMA);
|
|
460
|
+
if (!needsRebuild)
|
|
461
|
+
return;
|
|
462
|
+
this.db.exec(`
|
|
463
|
+
DELETE FROM observations_fts;
|
|
464
|
+
DELETE FROM observations_terms_fts;
|
|
465
|
+
|
|
466
|
+
INSERT INTO observations_fts(rowid, id, title, narrative, facts, concepts)
|
|
467
|
+
SELECT o.rowid, o.id, o.title, o.narrative, o.facts, o.concepts
|
|
468
|
+
FROM observations o
|
|
469
|
+
WHERE o.quality_status = 'accepted'
|
|
470
|
+
AND o.resolution_state = 'current';
|
|
471
|
+
|
|
472
|
+
INSERT INTO observations_terms_fts(rowid, id, title, narrative, facts, concepts)
|
|
473
|
+
SELECT o.rowid, o.id, o.title, o.narrative, o.facts, o.concepts
|
|
474
|
+
FROM observations o
|
|
475
|
+
WHERE o.quality_status = 'accepted'
|
|
476
|
+
AND o.resolution_state = 'current';
|
|
477
|
+
`);
|
|
478
|
+
this.db.prepare('INSERT OR REPLACE INTO schema_versions(version, applied_at) VALUES (?, ?)').run(OBSERVATION_FTS_SCHEMA_VERSION, Date.now());
|
|
479
|
+
})();
|
|
480
|
+
}
|
|
295
481
|
// ── V1 → V2 Migration ────────────────────────────
|
|
296
482
|
migrateV1ToV2() {
|
|
297
483
|
const tables = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table'").all().map(r => r.name);
|
|
@@ -503,16 +689,18 @@ export class MemoryStore {
|
|
|
503
689
|
const row = this.db.prepare('SELECT vector FROM embeddings WHERE source_type = ? AND source_id = ?').get(sourceType, sourceId);
|
|
504
690
|
return row?.vector ?? null;
|
|
505
691
|
}
|
|
506
|
-
searchObservationVector(embedding, limit = 20) {
|
|
692
|
+
searchObservationVector(embedding, limit = 20, excludedIds) {
|
|
507
693
|
const buffer = Buffer.from(embedding.buffer, embedding.byteOffset, embedding.byteLength);
|
|
508
|
-
const
|
|
694
|
+
const excludedJson = serializeObservationSearchExcludedIds(excludedIds);
|
|
695
|
+
const rows = this.db.prepare(`${OBSERVATION_EXCLUSION_CTE}
|
|
509
696
|
SELECT o.*, vec_distance_cosine(o.embedding, ?) AS distance
|
|
510
697
|
FROM observations o
|
|
511
698
|
WHERE o.embedding IS NOT NULL
|
|
512
699
|
AND o.quality_status = 'accepted'
|
|
513
700
|
AND o.resolution_state <> 'superseded'
|
|
701
|
+
AND o.id NOT IN (SELECT id FROM excluded_observation_ids)
|
|
514
702
|
ORDER BY distance LIMIT ?
|
|
515
|
-
`).all(buffer, limit);
|
|
703
|
+
`).all(excludedJson, buffer, limit);
|
|
516
704
|
return rows.map(row => ({ observation: row, distance: row.distance }));
|
|
517
705
|
}
|
|
518
706
|
// ── Search ───────────────────────────────────────
|
|
@@ -525,6 +713,66 @@ export class MemoryStore {
|
|
|
525
713
|
ORDER BY rank LIMIT ?
|
|
526
714
|
`).all(query, limit);
|
|
527
715
|
}
|
|
716
|
+
searchObservationBM25(query, limit = 20, excludedIds) {
|
|
717
|
+
const normalizedQuery = normalizeObservationLexicalQuery(query);
|
|
718
|
+
if ((!normalizedQuery.trigramQuery && !normalizedQuery.termQuery) || limit <= 0)
|
|
719
|
+
return [];
|
|
720
|
+
const boundedLimit = Number.isFinite(limit)
|
|
721
|
+
? Math.min(Math.max(Math.floor(limit), 1), 100)
|
|
722
|
+
: 20;
|
|
723
|
+
const excludedJson = serializeObservationSearchExcludedIds(excludedIds);
|
|
724
|
+
const scores = new Map();
|
|
725
|
+
const addRanked = (rows) => {
|
|
726
|
+
for (let index = 0; index < rows.length; index++) {
|
|
727
|
+
const id = rows[index].id;
|
|
728
|
+
scores.set(id, (scores.get(id) ?? 0) + 1 / (60 + index + 1));
|
|
729
|
+
}
|
|
730
|
+
};
|
|
731
|
+
if (normalizedQuery.trigramQuery) {
|
|
732
|
+
addRanked(this.searchObservationFtsWithRecovery('observations_fts', normalizedQuery.trigramQuery, boundedLimit, excludedJson));
|
|
733
|
+
}
|
|
734
|
+
if (normalizedQuery.termQuery) {
|
|
735
|
+
addRanked(this.searchObservationFtsWithRecovery('observations_terms_fts', normalizedQuery.termQuery, boundedLimit, excludedJson));
|
|
736
|
+
}
|
|
737
|
+
return [...scores]
|
|
738
|
+
.map(([id, score]) => ({ id, rank: -score }))
|
|
739
|
+
.sort((left, right) => left.rank - right.rank || left.id.localeCompare(right.id))
|
|
740
|
+
.slice(0, boundedLimit);
|
|
741
|
+
}
|
|
742
|
+
searchObservationFtsWithRecovery(table, query, limit, excludedJson) {
|
|
743
|
+
try {
|
|
744
|
+
return this.searchObservationFts(table, query, limit, excludedJson);
|
|
745
|
+
}
|
|
746
|
+
catch (error) {
|
|
747
|
+
if (!isRecoverableObservationFtsError(error))
|
|
748
|
+
throw error;
|
|
749
|
+
this.ensureObservationFts(true);
|
|
750
|
+
return this.searchObservationFts(table, query, limit, excludedJson);
|
|
751
|
+
}
|
|
752
|
+
}
|
|
753
|
+
searchObservationFts(table, query, limit, excludedJson) {
|
|
754
|
+
const sql = table === 'observations_fts' ? `${OBSERVATION_EXCLUSION_CTE}
|
|
755
|
+
SELECT o.id, bm25(observations_fts) AS rank
|
|
756
|
+
FROM observations_fts f
|
|
757
|
+
JOIN observations o ON o.rowid = f.rowid
|
|
758
|
+
WHERE observations_fts MATCH ?
|
|
759
|
+
AND o.quality_status = 'accepted'
|
|
760
|
+
AND o.resolution_state = 'current'
|
|
761
|
+
AND o.id NOT IN (SELECT id FROM excluded_observation_ids)
|
|
762
|
+
ORDER BY rank LIMIT ?
|
|
763
|
+
` : `${OBSERVATION_EXCLUSION_CTE}
|
|
764
|
+
SELECT o.id, bm25(observations_terms_fts) AS rank
|
|
765
|
+
FROM observations_terms_fts f
|
|
766
|
+
JOIN observations o ON o.rowid = f.rowid
|
|
767
|
+
WHERE observations_terms_fts MATCH ?
|
|
768
|
+
AND o.quality_status = 'accepted'
|
|
769
|
+
AND o.resolution_state = 'current'
|
|
770
|
+
AND o.id NOT IN (SELECT id FROM excluded_observation_ids)
|
|
771
|
+
ORDER BY rank, updated_at DESC
|
|
772
|
+
LIMIT ?
|
|
773
|
+
`;
|
|
774
|
+
return this.db.prepare(sql).all(excludedJson, query, limit);
|
|
775
|
+
}
|
|
528
776
|
searchVector(embedding, limit = 20) {
|
|
529
777
|
const buffer = Buffer.from(embedding.buffer);
|
|
530
778
|
return this.db.prepare(`
|
|
@@ -569,6 +817,123 @@ export class MemoryStore {
|
|
|
569
817
|
getSessionEvents(sessionId) {
|
|
570
818
|
return this.db.prepare('SELECT * FROM session_events WHERE session_id = ? ORDER BY created_at ASC').all(sessionId);
|
|
571
819
|
}
|
|
820
|
+
reconcileDecidedTurnEventWindows(sessionId, turns, sessionEndAt = Number.MAX_SAFE_INTEGER) {
|
|
821
|
+
const normalizedSessionId = sessionId.trim();
|
|
822
|
+
if (!normalizedSessionId)
|
|
823
|
+
throw new Error('Session event reconciliation requires a session ID');
|
|
824
|
+
if (!Number.isSafeInteger(sessionEndAt) || sessionEndAt < 0) {
|
|
825
|
+
throw new Error('Session event reconciliation requires a safe session end timestamp');
|
|
826
|
+
}
|
|
827
|
+
const orderedTurns = turns.filter(turn => turn.sessionId === normalizedSessionId);
|
|
828
|
+
if (orderedTurns.length !== turns.length) {
|
|
829
|
+
throw new Error('Memory turn session does not match event reconciliation session');
|
|
830
|
+
}
|
|
831
|
+
for (let index = 1; index < orderedTurns.length; index += 1) {
|
|
832
|
+
if (orderedTurns[index].createdAt < orderedTurns[index - 1].createdAt) {
|
|
833
|
+
throw new Error('Memory turns must be ordered by createdAt for event reconciliation');
|
|
834
|
+
}
|
|
835
|
+
}
|
|
836
|
+
const selectSessionEvents = this.db.prepare(`
|
|
837
|
+
SELECT id, created_at FROM session_events
|
|
838
|
+
WHERE session_id = ? AND processed = 0 AND maintenance_status = 'active'
|
|
839
|
+
ORDER BY created_at ASC, id ASC
|
|
840
|
+
`);
|
|
841
|
+
const markCommitted = this.db.prepare(`
|
|
842
|
+
UPDATE session_events SET processed = 1
|
|
843
|
+
WHERE id = ? AND session_id = ? AND processed = 0 AND maintenance_status = 'active'
|
|
844
|
+
`);
|
|
845
|
+
const markSkipped = this.db.prepare(`
|
|
846
|
+
UPDATE session_events
|
|
847
|
+
SET processed = 1, maintenance_status = 'discarded', quarantine_reason = ?
|
|
848
|
+
WHERE id = ? AND session_id = ? AND processed = 0 AND maintenance_status = 'active'
|
|
849
|
+
`);
|
|
850
|
+
const insertAudit = this.db.prepare(`
|
|
851
|
+
INSERT INTO event_processing_audit
|
|
852
|
+
(id, event_id, disposition, reason, run_id, timestamp)
|
|
853
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
854
|
+
`);
|
|
855
|
+
const countAuditsForRuns = this.db.prepare(`
|
|
856
|
+
WITH requested_runs(run_id) AS (
|
|
857
|
+
SELECT DISTINCT value FROM json_each(?)
|
|
858
|
+
)
|
|
859
|
+
SELECT audit.disposition, COUNT(*) AS count
|
|
860
|
+
FROM event_processing_audit audit
|
|
861
|
+
INNER JOIN requested_runs requested ON requested.run_id = audit.run_id
|
|
862
|
+
GROUP BY audit.disposition
|
|
863
|
+
`);
|
|
864
|
+
return this.db.transaction(() => {
|
|
865
|
+
const reconciledAt = Date.now();
|
|
866
|
+
const groups = [];
|
|
867
|
+
for (const turn of orderedTurns) {
|
|
868
|
+
const current = groups.at(-1);
|
|
869
|
+
if (current?.[0]?.createdAt === turn.createdAt)
|
|
870
|
+
current.push(turn);
|
|
871
|
+
else
|
|
872
|
+
groups.push([turn]);
|
|
873
|
+
}
|
|
874
|
+
let processedThisRun = 0;
|
|
875
|
+
let discardedThisRun = 0;
|
|
876
|
+
let decidedTurnCount = 0;
|
|
877
|
+
let eventIndex = 0;
|
|
878
|
+
const canonicalRunIds = [];
|
|
879
|
+
const events = selectSessionEvents.all(normalizedSessionId);
|
|
880
|
+
for (let groupIndex = 0; groupIndex < groups.length; groupIndex += 1) {
|
|
881
|
+
const group = groups[groupIndex];
|
|
882
|
+
const windowStart = group[0].createdAt;
|
|
883
|
+
const windowEnd = Math.min(groups[groupIndex + 1]?.[0]?.createdAt ?? sessionEndAt, sessionEndAt);
|
|
884
|
+
while (eventIndex < events.length && events[eventIndex].created_at < windowStart)
|
|
885
|
+
eventIndex += 1;
|
|
886
|
+
if (windowStart >= windowEnd)
|
|
887
|
+
continue;
|
|
888
|
+
const allCanonical = group.every(turn => turn.status !== 'pending' && turn.receiptId?.startsWith('memory-receipt:'));
|
|
889
|
+
const allCommitted = allCanonical && group.every(turn => turn.status === 'committed');
|
|
890
|
+
const allSkipped = allCanonical && group.every(turn => turn.status === 'skipped');
|
|
891
|
+
const disposition = allCommitted ? 'processed' : allSkipped ? 'discarded' : null;
|
|
892
|
+
const canonicalGroup = disposition
|
|
893
|
+
? group.map(turn => `${turn.turnId}=${turn.receiptId}`).join(',')
|
|
894
|
+
: '';
|
|
895
|
+
const reason = disposition
|
|
896
|
+
? `canonical_turn_group_${allCommitted ? 'committed' : 'skipped'}:${windowStart}:${canonicalGroup}`
|
|
897
|
+
: '';
|
|
898
|
+
const runId = disposition
|
|
899
|
+
? `turn-reconcile:${createHash('sha256')
|
|
900
|
+
.update(`${normalizedSessionId}\0${windowStart}\0${allCommitted ? 'committed' : 'skipped'}\0${canonicalGroup}`)
|
|
901
|
+
.digest('hex').slice(0, 32)}`
|
|
902
|
+
: '';
|
|
903
|
+
if (disposition) {
|
|
904
|
+
decidedTurnCount += group.length;
|
|
905
|
+
canonicalRunIds.push(runId);
|
|
906
|
+
}
|
|
907
|
+
while (eventIndex < events.length && events[eventIndex].created_at < windowEnd) {
|
|
908
|
+
const event = events[eventIndex++];
|
|
909
|
+
if (!disposition)
|
|
910
|
+
continue;
|
|
911
|
+
const changed = disposition === 'processed'
|
|
912
|
+
? markCommitted.run(event.id, normalizedSessionId).changes
|
|
913
|
+
: markSkipped.run(reason, event.id, normalizedSessionId).changes;
|
|
914
|
+
if (changed === 0)
|
|
915
|
+
continue;
|
|
916
|
+
if (disposition === 'processed')
|
|
917
|
+
processedThisRun += changed;
|
|
918
|
+
else
|
|
919
|
+
discardedThisRun += changed;
|
|
920
|
+
insertAudit.run(`epa:${reconciledAt}:${randomUUID().slice(0, 8)}`, event.id, disposition, reason, runId, reconciledAt);
|
|
921
|
+
}
|
|
922
|
+
}
|
|
923
|
+
const historicalCounts = canonicalRunIds.length > 0
|
|
924
|
+
? countAuditsForRuns.all(JSON.stringify(canonicalRunIds))
|
|
925
|
+
: [];
|
|
926
|
+
const countByDisposition = new Map(historicalCounts.map(row => [row.disposition, Number(row.count)]));
|
|
927
|
+
return {
|
|
928
|
+
sessionId: normalizedSessionId,
|
|
929
|
+
turnCount: orderedTurns.length,
|
|
930
|
+
decidedTurnCount,
|
|
931
|
+
committedProcessed: countByDisposition.get('processed') ?? 0,
|
|
932
|
+
skippedDiscarded: countByDisposition.get('discarded') ?? 0,
|
|
933
|
+
remainingUndecided: events.length - processedThisRun - discardedThisRun,
|
|
934
|
+
};
|
|
935
|
+
})();
|
|
936
|
+
}
|
|
572
937
|
getUnprocessedEvents() {
|
|
573
938
|
const events = this.db.prepare(`SELECT * FROM session_events
|
|
574
939
|
WHERE processed = 0 AND maintenance_status = 'active'
|
|
@@ -895,6 +1260,124 @@ export class MemoryStore {
|
|
|
895
1260
|
return this.getObservation(row.id);
|
|
896
1261
|
})();
|
|
897
1262
|
}
|
|
1263
|
+
acceptExplicitPreference(input) {
|
|
1264
|
+
return this.db.transaction(() => {
|
|
1265
|
+
this.ensureSession(input.sessionId, input.project);
|
|
1266
|
+
const latest = this.db.prepare(`
|
|
1267
|
+
SELECT MAX(COALESCE(explicit_accepted_at, timestamp)) AS latest
|
|
1268
|
+
FROM observations
|
|
1269
|
+
WHERE project = ? AND type = 'user_preference' AND source = 'explicit_intent'
|
|
1270
|
+
`).get(input.project);
|
|
1271
|
+
const acceptedAt = Math.max(Date.now(), (latest.latest ?? 0) + 1);
|
|
1272
|
+
const dedupKey = observationDedupKey('user_preference', input.title);
|
|
1273
|
+
const exact = this.db.prepare('SELECT * FROM observations WHERE dedup_key = ? AND project = ?').get(dedupKey, input.project) ?? null;
|
|
1274
|
+
let row;
|
|
1275
|
+
if (exact) {
|
|
1276
|
+
const contentChanged = exact.title !== input.title || exact.narrative !== input.content;
|
|
1277
|
+
const sourceEventIds = new Set(parseStringArray(exact.source_event_ids));
|
|
1278
|
+
if (input.sourceEventId)
|
|
1279
|
+
sourceEventIds.add(input.sourceEventId);
|
|
1280
|
+
this.db.prepare(`
|
|
1281
|
+
UPDATE observations
|
|
1282
|
+
SET title = ?, narrative = ?, source = 'explicit_intent',
|
|
1283
|
+
confidence = MAX(confidence, 0.95),
|
|
1284
|
+
importance = MAX(importance, 10), quality_status = 'accepted',
|
|
1285
|
+
quarantine_reason = NULL, quality_details = NULL, quarantined_at = NULL,
|
|
1286
|
+
resolution_state = 'current', superseded_by_observation = NULL,
|
|
1287
|
+
source_event_ids = ?, updated_at = ?, explicit_accepted_at = ?,
|
|
1288
|
+
embedding = CASE WHEN ? = 1 THEN NULL ELSE embedding END
|
|
1289
|
+
WHERE id = ?
|
|
1290
|
+
`).run(input.title, input.content, JSON.stringify([...sourceEventIds]), acceptedAt, acceptedAt, contentChanged ? 1 : 0, exact.id);
|
|
1291
|
+
if (contentChanged) {
|
|
1292
|
+
this.db.prepare("DELETE FROM embeddings WHERE source_type = 'observation' AND source_id = ?").run(exact.id);
|
|
1293
|
+
}
|
|
1294
|
+
this.incrementObservationAccess(exact.id, acceptedAt);
|
|
1295
|
+
row = this.getObservation(exact.id);
|
|
1296
|
+
}
|
|
1297
|
+
else {
|
|
1298
|
+
row = this.addObservation({
|
|
1299
|
+
sessionId: input.sessionId,
|
|
1300
|
+
project: input.project,
|
|
1301
|
+
type: 'user_preference',
|
|
1302
|
+
title: input.title,
|
|
1303
|
+
narrative: input.content,
|
|
1304
|
+
importance: 10,
|
|
1305
|
+
confidence: 0.95,
|
|
1306
|
+
source: 'explicit_intent',
|
|
1307
|
+
sourceEventIds: input.sourceEventId ? [input.sourceEventId] : [],
|
|
1308
|
+
timestamp: acceptedAt,
|
|
1309
|
+
qualityStatus: 'accepted',
|
|
1310
|
+
});
|
|
1311
|
+
this.db.prepare('UPDATE observations SET explicit_accepted_at = ? WHERE id = ?').run(acceptedAt, row.id);
|
|
1312
|
+
row = this.getObservation(row.id);
|
|
1313
|
+
}
|
|
1314
|
+
this.supersedeConflictingExplicitPreferences(row, analyzePreference(input.content));
|
|
1315
|
+
if (input.sourceEventId)
|
|
1316
|
+
this.markEventsProcessed([input.sourceEventId]);
|
|
1317
|
+
return this.getObservation(row.id);
|
|
1318
|
+
})();
|
|
1319
|
+
}
|
|
1320
|
+
reconcileExplicitPreferenceConflicts(project = this.defaultProjectRoot) {
|
|
1321
|
+
return this.db.transaction(() => {
|
|
1322
|
+
const rows = this.listCurrentExplicitPreferences(project).sort((left, right) => this.explicitPreferenceAcceptanceTime(left)
|
|
1323
|
+
- this.explicitPreferenceAcceptanceTime(right)
|
|
1324
|
+
|| left.timestamp - right.timestamp
|
|
1325
|
+
|| left.id.localeCompare(right.id));
|
|
1326
|
+
let superseded = 0;
|
|
1327
|
+
const active = [];
|
|
1328
|
+
for (const row of rows) {
|
|
1329
|
+
const analysis = analyzePreferenceObservation(row);
|
|
1330
|
+
if (analysis.status !== 'high_confidence') {
|
|
1331
|
+
active.push({ row, analysis });
|
|
1332
|
+
continue;
|
|
1333
|
+
}
|
|
1334
|
+
for (let index = active.length - 1; index >= 0; index--) {
|
|
1335
|
+
const previous = active[index];
|
|
1336
|
+
if (!analysesConflict(previous.analysis, analysis))
|
|
1337
|
+
continue;
|
|
1338
|
+
this.db.prepare(`
|
|
1339
|
+
UPDATE observations
|
|
1340
|
+
SET resolution_state = 'superseded', superseded_by_observation = ?
|
|
1341
|
+
WHERE id = ? AND resolution_state = 'current'
|
|
1342
|
+
`).run(row.id, previous.row.id);
|
|
1343
|
+
active.splice(index, 1);
|
|
1344
|
+
superseded += 1;
|
|
1345
|
+
}
|
|
1346
|
+
active.push({ row, analysis });
|
|
1347
|
+
}
|
|
1348
|
+
return superseded;
|
|
1349
|
+
})();
|
|
1350
|
+
}
|
|
1351
|
+
listCurrentExplicitPreferences(project = this.defaultProjectRoot) {
|
|
1352
|
+
return this.db.prepare(`
|
|
1353
|
+
SELECT * FROM observations
|
|
1354
|
+
WHERE project = ? AND type = 'user_preference' AND source = 'explicit_intent'
|
|
1355
|
+
AND quality_status = 'accepted' AND resolution_state = 'current'
|
|
1356
|
+
ORDER BY COALESCE(explicit_accepted_at, timestamp) DESC, timestamp DESC, id DESC
|
|
1357
|
+
`).all(project);
|
|
1358
|
+
}
|
|
1359
|
+
explicitPreferenceAcceptanceTime(row) {
|
|
1360
|
+
return row.explicit_accepted_at ?? row.timestamp;
|
|
1361
|
+
}
|
|
1362
|
+
supersedeConflictingExplicitPreferences(winner, winnerAnalysis) {
|
|
1363
|
+
if (winnerAnalysis.status !== 'high_confidence')
|
|
1364
|
+
return 0;
|
|
1365
|
+
const update = this.db.prepare(`
|
|
1366
|
+
UPDATE observations
|
|
1367
|
+
SET resolution_state = 'superseded', superseded_by_observation = ?
|
|
1368
|
+
WHERE id = ? AND resolution_state = 'current'
|
|
1369
|
+
`);
|
|
1370
|
+
let superseded = 0;
|
|
1371
|
+
for (const candidate of this.listCurrentExplicitPreferences(winner.project)) {
|
|
1372
|
+
if (candidate.id === winner.id)
|
|
1373
|
+
continue;
|
|
1374
|
+
const analysis = analyzePreferenceObservation(candidate);
|
|
1375
|
+
if (!analysesConflict(analysis, winnerAnalysis))
|
|
1376
|
+
continue;
|
|
1377
|
+
superseded += update.run(winner.id, candidate.id).changes;
|
|
1378
|
+
}
|
|
1379
|
+
return superseded;
|
|
1380
|
+
}
|
|
898
1381
|
getObservation(id) {
|
|
899
1382
|
return this.db.prepare('SELECT * FROM observations WHERE id = ?').get(id) ?? null;
|
|
900
1383
|
}
|