@slorenzot/memento-core 0.7.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -35,41 +35,36 @@ var __importStar = (this && this.__importStar) || (function () {
35
35
  Object.defineProperty(exports, "__esModule", { value: true });
36
36
  exports.MemoryEngine = void 0;
37
37
  const crypto = __importStar(require("crypto"));
38
+ const EmbeddingService_js_1 = require("./EmbeddingService.js");
38
39
  const bun_sqlite_1 = require("bun:sqlite");
39
40
  const fs_1 = require("fs");
40
41
  const path_1 = require("path");
41
42
  class MemoryEngine {
43
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Bun Database type is incompatible with mock pattern
42
44
  db;
43
45
  dbPath;
44
46
  initError = null;
47
+ embeddingService;
45
48
  constructor(dbPath = './data/memento.db') {
46
- this.dbPath = dbPath;
49
+ this.dbPath = (0, path_1.resolve)(dbPath);
50
+ this.embeddingService = new EmbeddingService_js_1.EmbeddingService();
47
51
  try {
48
- const dbDir = (0, path_1.dirname)(dbPath);
49
- // Create directory structure if it doesn't exist
52
+ const dbDir = (0, path_1.dirname)(this.dbPath);
50
53
  (0, fs_1.mkdirSync)(dbDir, { recursive: true });
51
- // Create database connection
52
- this.db = new bun_sqlite_1.Database(dbPath, { create: true });
54
+ this.db = new bun_sqlite_1.Database(this.dbPath, { create: true });
53
55
  this.initializeDatabase();
54
- console.error(`✓ Database initialized successfully at: ${dbPath}`);
55
56
  }
56
57
  catch (error) {
57
- this.initError = error;
58
- console.error(`✗ Failed to initialize database at ${dbPath}:`, error.message);
59
- console.error(' The server will start but database operations will fail until this is resolved.');
60
- // Create a mock database object to prevent null reference errors
58
+ this.initError = error instanceof Error ? error : new Error(String(error));
61
59
  this.db = this.createMockDatabase();
62
60
  }
63
61
  }
62
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Mock fallback for failed DB init
64
63
  createMockDatabase() {
65
64
  const throwError = () => {
66
65
  throw new Error(`Database not initialized: ${this.initError?.message || 'Unknown error'}`);
67
66
  };
68
- return {
69
- prepare: throwError,
70
- exec: throwError,
71
- close: () => { },
72
- };
67
+ return { prepare: throwError, exec: throwError, close: () => { }, transaction: throwError };
73
68
  }
74
69
  isHealthy() {
75
70
  return this.initError === null;
@@ -83,6 +78,7 @@ class MemoryEngine {
83
78
  initializeDatabase() {
84
79
  this.db.exec('PRAGMA journal_mode = WAL');
85
80
  this.db.exec('PRAGMA foreign_keys = ON');
81
+ this.db.exec('PRAGMA busy_timeout = 5000');
86
82
  this.db.exec(`
87
83
  CREATE TABLE IF NOT EXISTS sessions (
88
84
  id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -103,6 +99,7 @@ class MemoryEngine {
103
99
  topic_key TEXT,
104
100
  project_id TEXT NOT NULL,
105
101
  created_at INTEGER NOT NULL,
102
+ deleted_at INTEGER DEFAULT NULL,
106
103
  metadata TEXT,
107
104
  FOREIGN KEY (session_id) REFERENCES sessions(id)
108
105
  );
@@ -124,27 +121,186 @@ class MemoryEngine {
124
121
  created_at INTEGER NOT NULL,
125
122
  metadata TEXT
126
123
  );
127
-
124
+ `);
125
+ // Migrate: add working_dir and aliases columns to projects table (Issue #177)
126
+ try {
127
+ this.db.exec('SELECT working_dir FROM projects LIMIT 0');
128
+ }
129
+ catch {
130
+ try {
131
+ this.db.exec('ALTER TABLE projects ADD COLUMN working_dir TEXT');
132
+ console.error('✓ Migration: added working_dir column to projects');
133
+ }
134
+ catch {
135
+ // Column may already exist
136
+ }
137
+ }
138
+ try {
139
+ this.db.exec('SELECT aliases FROM projects LIMIT 0');
140
+ }
141
+ catch {
142
+ try {
143
+ this.db.exec("ALTER TABLE projects ADD COLUMN aliases TEXT DEFAULT '[]'");
144
+ console.error('✓ Migration: added aliases column to projects');
145
+ }
146
+ catch {
147
+ // Column may already exist
148
+ }
149
+ }
150
+ // Migrate: add deleted_at column if missing
151
+ try {
152
+ this.db.exec('SELECT deleted_at FROM observations LIMIT 0');
153
+ }
154
+ catch {
155
+ try {
156
+ this.db.exec('ALTER TABLE observations ADD COLUMN deleted_at INTEGER DEFAULT NULL');
157
+ console.error('✓ Migration: added deleted_at column to observations');
158
+ }
159
+ catch {
160
+ // Column may already exist in a concurrent scenario
161
+ }
162
+ }
163
+ // Create index for deleted_at if not exists
164
+ this.db.exec('CREATE INDEX IF NOT EXISTS idx_observations_deleted ON observations(deleted_at)');
165
+ this.db.exec('CREATE INDEX IF NOT EXISTS idx_observations_created ON observations(created_at)');
166
+ this.db.exec('CREATE INDEX IF NOT EXISTS idx_observations_type ON observations(type)');
167
+ this.db.exec('CREATE INDEX IF NOT EXISTS idx_observations_project ON observations(project_id)');
168
+ this.db.exec('CREATE INDEX IF NOT EXISTS idx_observations_topic ON observations(topic_key)');
169
+ // Migrate: add scope column if missing
170
+ try {
171
+ this.db.exec('SELECT scope FROM observations LIMIT 0');
172
+ }
173
+ catch {
174
+ try {
175
+ this.db.exec("ALTER TABLE observations ADD COLUMN scope TEXT NOT NULL DEFAULT 'project'");
176
+ console.error('✓ Migration: added scope column to observations');
177
+ }
178
+ catch {
179
+ // Column may already exist in a concurrent scenario
180
+ }
181
+ }
182
+ this.db.exec('CREATE INDEX IF NOT EXISTS idx_observations_scope ON observations(scope)');
183
+ // Migrate: add revision_count column if missing
184
+ try {
185
+ this.db.exec('SELECT revision_count FROM observations LIMIT 0');
186
+ }
187
+ catch {
188
+ try {
189
+ this.db.exec('ALTER TABLE observations ADD COLUMN revision_count INTEGER NOT NULL DEFAULT 0');
190
+ console.error('✓ Migration: added revision_count column to observations');
191
+ }
192
+ catch {
193
+ // Column may already exist in a concurrent scenario
194
+ }
195
+ }
196
+ // Migrate: add pinned column if missing (Issue #50)
197
+ try {
198
+ this.db.exec('SELECT pinned FROM observations LIMIT 0');
199
+ }
200
+ catch {
201
+ try {
202
+ this.db.exec('ALTER TABLE observations ADD COLUMN pinned INTEGER NOT NULL DEFAULT 0');
203
+ console.error('✓ Migration: added pinned column to observations');
204
+ }
205
+ catch {
206
+ // Column may already exist in a concurrent scenario
207
+ }
208
+ }
209
+ this.db.exec('CREATE INDEX IF NOT EXISTS idx_observations_pinned ON observations(pinned) WHERE pinned = 1');
210
+ // Migrate: add read_only column if missing (Issue #54)
211
+ try {
212
+ this.db.exec('SELECT read_only FROM observations LIMIT 0');
213
+ }
214
+ catch {
215
+ try {
216
+ this.db.exec('ALTER TABLE observations ADD COLUMN read_only INTEGER NOT NULL DEFAULT 0');
217
+ console.error('✓ Migration: added read_only column to observations');
218
+ }
219
+ catch {
220
+ // Column may already exist in a concurrent scenario
221
+ }
222
+ }
223
+ // Migrate: clean up empty string topic_key → NULL (Issue #69)
224
+ try {
225
+ const result = this.db.prepare("UPDATE observations SET topic_key = NULL WHERE topic_key = ''").run();
226
+ if (result.changes > 0) {
227
+ console.error(`✓ Migration: cleaned up ${result.changes} empty topic_key value(s)`);
228
+ }
229
+ }
230
+ catch {
231
+ // Table may not have topic_key column yet
232
+ }
233
+ // FTS5 — standalone mode (no content= parameter).
234
+ // FTS5 owns its own data copy, supports full CRUD (INSERT, DELETE, UPDATE),
235
+ // and avoids SQLITE_CORRUPT_VTAB that occurred with content='observations' mode
236
+ // under bun:sqlite + WAL + concurrent access. See: GitHub Issue #68.
237
+ this.db.exec(`
128
238
  CREATE VIRTUAL TABLE IF NOT EXISTS observations_fts USING fts5(
129
239
  title, content, topic_key, project_id,
130
- content='observations'
240
+ tokenize='porter unicode61'
241
+ );
242
+ `);
243
+ // FTS5 sync is managed at application-level in each CRUD method.
244
+ // Previous trigger-based approach caused SQLITE_CORRUPT_VTAB under
245
+ // concurrent access (DELETE+INSERT on FTS5 virtual table within a trigger).
246
+ // See: GitHub Issue #68
247
+ // ─── Journal tables (append-only evidence) ────────────────
248
+ this.db.exec(`
249
+ CREATE TABLE IF NOT EXISTS journal (
250
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
251
+ uuid TEXT UNIQUE NOT NULL,
252
+ project_id TEXT NOT NULL,
253
+ session_id INTEGER REFERENCES sessions(id),
254
+ title TEXT NOT NULL,
255
+ body TEXT NOT NULL,
256
+ model TEXT,
257
+ provider TEXT,
258
+ agent TEXT,
259
+ superseded_by INTEGER REFERENCES journal(id),
260
+ invalidated_at INTEGER,
261
+ metadata TEXT,
262
+ created_at INTEGER NOT NULL
131
263
  );
132
264
 
133
- CREATE TRIGGER IF NOT EXISTS observations_ai AFTER INSERT ON observations BEGIN
134
- INSERT INTO observations_fts(rowid, title, content, topic_key, project_id)
135
- VALUES (new.id, new.title, new.content, new.topic_key, new.project_id);
136
- END;
137
-
138
- CREATE TRIGGER IF NOT EXISTS observations_ad AFTER DELETE ON observations BEGIN
139
- DELETE FROM observations_fts WHERE rowid = old.id;
140
- END;
141
-
142
- CREATE TRIGGER IF NOT EXISTS observations_au AFTER UPDATE ON observations BEGIN
143
- DELETE FROM observations_fts WHERE rowid = old.id;
144
- INSERT INTO observations_fts(rowid, title, content, topic_key, project_id)
145
- VALUES (new.id, new.title, new.content, new.topic_key, new.project_id);
265
+ CREATE TABLE IF NOT EXISTS journal_tags (
266
+ journal_id INTEGER NOT NULL REFERENCES journal(id) ON DELETE CASCADE,
267
+ tag TEXT NOT NULL,
268
+ PRIMARY KEY (journal_id, tag)
269
+ );
270
+ `);
271
+ // Journal indexes
272
+ this.db.exec('CREATE INDEX IF NOT EXISTS idx_journal_project ON journal(project_id)');
273
+ this.db.exec('CREATE INDEX IF NOT EXISTS idx_journal_session ON journal(session_id)');
274
+ this.db.exec('CREATE INDEX IF NOT EXISTS idx_journal_created ON journal(created_at)');
275
+ this.db.exec('CREATE INDEX IF NOT EXISTS idx_journal_superseded ON journal(superseded_by)');
276
+ this.db.exec('CREATE INDEX IF NOT EXISTS idx_journal_tags_tag ON journal_tags(tag)');
277
+ // Journal FTS5
278
+ this.db.exec(`
279
+ CREATE VIRTUAL TABLE IF NOT EXISTS journal_fts USING fts5(
280
+ title, body, project_id,
281
+ content='journal',
282
+ tokenize='porter unicode61'
283
+ );
284
+ `);
285
+ // Journal FTS trigger — insert only (append-only: no update/delete triggers)
286
+ this.db.exec('DROP TRIGGER IF EXISTS journal_ai');
287
+ this.db.exec(`
288
+ CREATE TRIGGER journal_ai AFTER INSERT ON journal BEGIN
289
+ INSERT INTO journal_fts(rowid, title, body, project_id)
290
+ VALUES (new.id, new.title, new.body, new.project_id);
146
291
  END;
147
292
  `);
293
+ // ─── Embeddings table (semantic search) ──────────────────────
294
+ this.db.exec(`
295
+ CREATE TABLE IF NOT EXISTS embeddings (
296
+ observation_id INTEGER PRIMARY KEY REFERENCES observations(id) ON DELETE CASCADE,
297
+ embedding BLOB NOT NULL,
298
+ model TEXT NOT NULL DEFAULT 'all-MiniLM-L6-v2',
299
+ dimensions INTEGER NOT NULL DEFAULT 384,
300
+ generated_at INTEGER NOT NULL
301
+ );
302
+ `);
303
+ this.db.exec('CREATE INDEX IF NOT EXISTS idx_embeddings_obs ON embeddings(observation_id)');
148
304
  }
149
305
  serialize(value) {
150
306
  return JSON.stringify(value);
@@ -164,26 +320,70 @@ class MemoryEngine {
164
320
  throw this.initError || new Error('Database not initialized');
165
321
  }
166
322
  }
323
+ /**
324
+ * Retry wrapper with exponential backoff for SQLITE_BUSY errors.
325
+ * Dual defense: PRAGMA busy_timeout (5s) catches most contention,
326
+ * this catches the rest with retries.
327
+ */
328
+ async withRetry(operation, maxRetries = 3, baseDelay = 100) {
329
+ for (let attempt = 0; attempt <= maxRetries; attempt++) {
330
+ try {
331
+ return operation();
332
+ }
333
+ catch (error) {
334
+ const message = error instanceof Error ? error.message : String(error);
335
+ const isBusy = message.includes('SQLITE_BUSY') || message.includes('database is locked');
336
+ if (!isBusy || attempt === maxRetries) {
337
+ throw error;
338
+ }
339
+ const delay = baseDelay * Math.pow(2, attempt);
340
+ await new Promise((resolve) => setTimeout(resolve, delay));
341
+ }
342
+ }
343
+ throw new Error('Max retries exceeded');
344
+ }
345
+ // ─── Observations CRUD ─────────────────────────────────────
167
346
  async createObservation(data) {
168
347
  this.checkHealth();
169
348
  const uuid = crypto.randomUUID();
170
349
  const createdAt = new Date();
171
350
  const metadata = this.serialize(data.metadata);
172
- const result = this.db
173
- .prepare(`INSERT INTO observations (uuid, session_id, title, content, type, topic_key, project_id, created_at, metadata) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`)
174
- .run(uuid, data.sessionId, data.title, data.content, data.type, data.topicKey ?? null, data.projectId, createdAt.getTime(), metadata);
175
- const id = typeof result.lastInsertRowid === 'bigint'
176
- ? Number(result.lastInsertRowid)
177
- : result.lastInsertRowid;
178
- const observation = await this.getObservationById(id);
351
+ const scope = data.scope || 'project';
352
+ const pinned = data.pinned ? 1 : 0;
353
+ const readOnly = data.readOnly ? 1 : 0;
354
+ const id = await this.withRetry(() => {
355
+ const result = this.db
356
+ .prepare(`INSERT INTO observations (uuid, session_id, title, content, type, topic_key, project_id, created_at, deleted_at, metadata, scope, pinned, read_only) VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?, ?, ?)`)
357
+ .run(uuid, data.sessionId, data.title, data.content, data.type, data.topicKey ?? null, data.projectId, createdAt.getTime(), metadata, scope, pinned, readOnly);
358
+ const insertId = typeof result.lastInsertRowid === 'bigint'
359
+ ? Number(result.lastInsertRowid)
360
+ : result.lastInsertRowid;
361
+ // Application-level FTS5 sync (was trigger observations_ai)
362
+ this.db.prepare('INSERT INTO observations_fts(rowid, title, content, topic_key, project_id) VALUES (?, ?, ?, ?, ?)').run(insertId, data.title, data.content, data.topicKey ?? '', data.projectId);
363
+ return insertId;
364
+ });
365
+ const observation = await this.getObservationById(id, true);
179
366
  if (!observation)
180
367
  throw new Error('Failed to retrieve created observation');
368
+ // Generate embedding asynchronously (non-blocking, fire-and-forget)
369
+ this.generateEmbedding(id).catch(() => {
370
+ // Silently ignore — embedding generation failure doesn't affect save
371
+ });
181
372
  return observation;
182
373
  }
183
374
  async updateObservation(id, updates) {
184
375
  const current = await this.getObservationById(id);
185
376
  if (!current)
186
377
  throw new Error('Observation not found');
378
+ if (current.deletedAt)
379
+ throw new Error('Cannot update a soft-deleted observation');
380
+ // Read-only protection: only allow changing readOnly itself
381
+ if (current.readOnly) {
382
+ const nonReadOnlyFields = Object.keys(updates).filter(k => k !== 'readOnly');
383
+ if (nonReadOnlyFields.length > 0) {
384
+ throw new Error(`Observation #${id} is read-only. Only readOnly flag can be changed.`);
385
+ }
386
+ }
187
387
  const fields = [];
188
388
  const values = [];
189
389
  if (updates.title !== undefined) {
@@ -200,35 +400,501 @@ class MemoryEngine {
200
400
  }
201
401
  if (updates.topicKey !== undefined) {
202
402
  fields.push('topic_key = ?');
203
- values.push(updates.topicKey || '');
403
+ values.push(updates.topicKey ?? null);
204
404
  }
205
405
  if (updates.metadata !== undefined) {
206
406
  fields.push('metadata = ?');
207
407
  values.push(this.serialize(updates.metadata));
208
408
  }
409
+ if (updates.pinned !== undefined) {
410
+ fields.push('pinned = ?');
411
+ values.push(updates.pinned ? 1 : 0);
412
+ }
413
+ if (updates.readOnly !== undefined) {
414
+ fields.push('read_only = ?');
415
+ values.push(updates.readOnly ? 1 : 0);
416
+ }
209
417
  if (fields.length === 0)
210
418
  return current;
419
+ fields.push('revision_count = revision_count + 1');
211
420
  values.push(id);
212
- this.db.prepare(`UPDATE observations SET ${fields.join(', ')} WHERE id = ?`).run(...values);
213
- const updated = await this.getObservationById(id);
421
+ // Check if any FTS5-indexed fields changed
422
+ const hasFtsUpdate = updates.title !== undefined ||
423
+ updates.content !== undefined ||
424
+ updates.topicKey !== undefined;
425
+ if (hasFtsUpdate) {
426
+ // Atomic transaction: UPDATE + FTS5 rebuild
427
+ // Replaces trigger observations_au that caused SQLITE_CORRUPT_VTAB (#68)
428
+ await this.withRetry(() => {
429
+ this.db.transaction(() => {
430
+ this.db.prepare(`UPDATE observations SET ${fields.join(', ')} WHERE id = ?`).run(...values);
431
+ // Re-read updated row for FTS5 sync
432
+ const row = this.db
433
+ .prepare('SELECT title, content, topic_key, project_id FROM observations WHERE id = ?')
434
+ .get(id);
435
+ if (row) {
436
+ this.db.prepare('DELETE FROM observations_fts WHERE rowid = ?').run(id);
437
+ this.db.prepare('INSERT INTO observations_fts(rowid, title, content, topic_key, project_id) VALUES (?, ?, ?, ?, ?)').run(id, row.title, row.content, row.topic_key, row.project_id);
438
+ }
439
+ })();
440
+ });
441
+ }
442
+ else {
443
+ // No FTS5 fields changed — just UPDATE
444
+ await this.withRetry(() => this.db.prepare(`UPDATE observations SET ${fields.join(', ')} WHERE id = ?`).run(...values));
445
+ }
446
+ const updated = await this.getObservationById(id, true);
214
447
  if (!updated)
215
448
  throw new Error('Failed to update observation');
449
+ // Regenerate embedding if content-affecting fields changed
450
+ if (hasFtsUpdate) {
451
+ this.generateEmbedding(id).catch(() => {
452
+ // Silently ignore — embedding regeneration failure doesn't affect update
453
+ });
454
+ }
216
455
  return updated;
217
456
  }
218
- async deleteObservation(id) {
219
- this.db.prepare('DELETE FROM observations WHERE id = ?').run(id);
457
+ // ─── Soft Delete / Restore / Purge ─────────────────────────
458
+ async deleteObservation(id, reason) {
459
+ this.checkHealth();
460
+ const obs = await this.getObservationById(id, true);
461
+ if (!obs)
462
+ throw new Error('Observation not found');
463
+ if (obs.deletedAt)
464
+ throw new Error('Observation already deleted');
465
+ if (obs.readOnly)
466
+ throw new Error(`Observation #${id} is read-only. Cannot delete.`);
467
+ const now = Date.now();
468
+ // If there's a reason, store it in metadata
469
+ if (reason) {
470
+ const meta = { ...obs.metadata, deleteReason: reason };
471
+ await this.withRetry(() => {
472
+ this.db
473
+ .prepare('UPDATE observations SET deleted_at = ?, metadata = ? WHERE id = ?')
474
+ .run(now, this.serialize(meta), id);
475
+ // Application-level FTS5 sync (was trigger observations_soft_delete)
476
+ this.db.prepare('DELETE FROM observations_fts WHERE rowid = ?').run(id);
477
+ });
478
+ }
479
+ else {
480
+ await this.withRetry(() => {
481
+ this.db.prepare('UPDATE observations SET deleted_at = ? WHERE id = ?').run(now, id);
482
+ // Application-level FTS5 sync (was trigger observations_soft_delete)
483
+ this.db.prepare('DELETE FROM observations_fts WHERE rowid = ?').run(id);
484
+ });
485
+ }
486
+ }
487
+ async restoreObservation(id) {
488
+ this.checkHealth();
489
+ const obs = await this.getObservationById(id, true);
490
+ if (!obs)
491
+ throw new Error('Observation not found');
492
+ if (!obs.deletedAt)
493
+ throw new Error('Observation is not deleted');
494
+ await this.withRetry(() => {
495
+ this.db.prepare('UPDATE observations SET deleted_at = NULL WHERE id = ?').run(id);
496
+ // Application-level FTS5 sync (was trigger observations_undelete)
497
+ const row = this.db
498
+ .prepare('SELECT title, content, topic_key, project_id FROM observations WHERE id = ?')
499
+ .get(id);
500
+ if (row) {
501
+ this.db.prepare('INSERT INTO observations_fts(rowid, title, content, topic_key, project_id) VALUES (?, ?, ?, ?, ?)').run(id, row.title, row.content, row.topic_key, row.project_id);
502
+ }
503
+ });
504
+ const restored = await this.getObservationById(id, true);
505
+ if (!restored)
506
+ throw new Error('Failed to restore observation');
507
+ return restored;
508
+ }
509
+ async purgeObservations(params) {
510
+ this.checkHealth();
511
+ let sql = 'SELECT id FROM observations WHERE deleted_at IS NOT NULL';
512
+ const values = [];
513
+ if (params.observationIds && params.observationIds.length > 0) {
514
+ const placeholders = params.observationIds.map(() => '?').join(',');
515
+ sql += ` AND id IN (${placeholders})`;
516
+ values.push(...params.observationIds);
517
+ }
518
+ if (params.projectId) {
519
+ sql += ' AND project_id = ?';
520
+ values.push(params.projectId);
521
+ }
522
+ const rows = this.db.prepare(sql).all(...values);
523
+ const ids = rows.map((r) => r.id);
524
+ if (ids.length === 0)
525
+ return { purgedCount: 0, purgedIds: [] };
526
+ const placeholders = ids.map(() => '?').join(',');
527
+ // Application-level FTS5 sync (was trigger observations_ad)
528
+ this.db.prepare(`DELETE FROM observations_fts WHERE rowid IN (${placeholders})`).run(...ids);
529
+ this.db.prepare(`DELETE FROM observations WHERE id IN (${placeholders})`).run(...ids);
530
+ return { purgedCount: ids.length, purgedIds: ids };
531
+ }
532
+ async listDeleted(params) {
533
+ this.checkHealth();
534
+ const { projectId, limit = 20 } = params;
535
+ let countSql = 'SELECT COUNT(*) as count FROM observations WHERE deleted_at IS NOT NULL';
536
+ let sql = 'SELECT * FROM observations WHERE deleted_at IS NOT NULL';
537
+ const values = [];
538
+ if (projectId) {
539
+ countSql += ' AND project_id = ?';
540
+ sql += ' AND project_id = ?';
541
+ values.push(projectId);
542
+ }
543
+ const countResult = this.db.prepare(countSql).get(...values);
544
+ const total = countResult ? countResult.count : 0;
545
+ sql += ' ORDER BY deleted_at DESC, id DESC LIMIT ?';
546
+ const rows = this.db.prepare(sql).all(...values, limit);
547
+ const observations = rows.map((row) => this.mapObservation(row));
548
+ return { observations, total };
220
549
  }
221
- async getObservation(id) {
222
- return await this.getObservationById(id);
550
+ // ─── Merge ─────────────────────────────────────────────────
551
+ async findMergeCandidates(params) {
552
+ this.checkHealth();
553
+ const { projectId, strategy, similarityThreshold = 0.85 } = params;
554
+ const groups = [];
555
+ if (strategy === 'by_topic') {
556
+ const rows = this.db
557
+ .prepare(`SELECT topic_key, COUNT(*) as cnt FROM observations
558
+ WHERE project_id = ? AND deleted_at IS NULL AND topic_key IS NOT NULL AND topic_key != ''
559
+ GROUP BY topic_key HAVING cnt >= 2
560
+ ORDER BY cnt DESC`)
561
+ .all(projectId);
562
+ for (const row of rows) {
563
+ const obs = this.db
564
+ .prepare('SELECT * FROM observations WHERE project_id = ? AND topic_key = ? AND deleted_at IS NULL ORDER BY created_at ASC, id ASC')
565
+ .all(projectId, row.topic_key);
566
+ groups.push({
567
+ reason: `topic_key: ${row.topic_key}`,
568
+ observations: obs.map((o) => this.mapObservation(o)),
569
+ estimatedReduction: obs.length - 1,
570
+ });
571
+ }
572
+ }
573
+ else {
574
+ // by_similarity — compare recent observations pairwise
575
+ const allObs = this.db
576
+ .prepare('SELECT * FROM observations WHERE project_id = ? AND deleted_at IS NULL ORDER BY created_at DESC, id DESC LIMIT 200')
577
+ .all(projectId);
578
+ const mapped = allObs.map((o) => this.mapObservation(o));
579
+ const visited = new Set();
580
+ for (let i = 0; i < mapped.length; i++) {
581
+ if (visited.has(mapped[i].id))
582
+ continue;
583
+ const group = [mapped[i]];
584
+ for (let j = i + 1; j < mapped.length; j++) {
585
+ if (visited.has(mapped[j].id))
586
+ continue;
587
+ const sim = this.jaccardSimilarity(mapped[i].content, mapped[j].content);
588
+ if (sim >= similarityThreshold) {
589
+ group.push(mapped[j]);
590
+ visited.add(mapped[j].id);
591
+ }
592
+ }
593
+ if (group.length >= 2) {
594
+ visited.add(mapped[i].id);
595
+ groups.push({
596
+ reason: `similarity >= ${similarityThreshold}`,
597
+ observations: group,
598
+ estimatedReduction: group.length - 1,
599
+ });
600
+ }
601
+ }
602
+ }
603
+ const totalCandidates = groups.reduce((acc, g) => acc + g.observations.length, 0);
604
+ const estimatedReduction = groups.reduce((acc, g) => acc + g.estimatedReduction, 0);
605
+ return { groups, totalCandidates, estimatedReduction };
223
606
  }
224
- async search(params) {
225
- const { query, type, projectId, topicKey, limit = 100, offset = 0 } = params;
607
+ async mergeObservations(params) {
608
+ this.checkHealth();
609
+ const { projectId, topicKey, observationIds, strategy, dryRun = false } = params;
610
+ let groups = [];
611
+ if (strategy === 'by_ids' && observationIds && observationIds.length >= 2) {
612
+ const obs = [];
613
+ for (const id of observationIds) {
614
+ const o = await this.getObservationById(id);
615
+ if (!o)
616
+ throw new Error(`Observation ${id} not found`);
617
+ if (o.deletedAt)
618
+ throw new Error(`Observation ${id} is soft-deleted`);
619
+ if (o.projectId !== projectId) {
620
+ throw new Error(`Observation ${id} belongs to project '${o.projectId}', not '${projectId}'`);
621
+ }
622
+ obs.push(o);
623
+ }
624
+ groups = [{ observations: obs }];
625
+ }
626
+ else if (strategy === 'by_topic') {
627
+ const candidates = await this.findMergeCandidates({
628
+ projectId,
629
+ strategy: 'by_topic',
630
+ });
631
+ if (topicKey) {
632
+ groups = candidates.groups.filter((g) => g.reason === `topic_key: ${topicKey}`);
633
+ }
634
+ else {
635
+ groups = candidates.groups;
636
+ }
637
+ }
638
+ else {
639
+ const candidates = await this.findMergeCandidates({
640
+ projectId,
641
+ strategy: 'by_similarity',
642
+ });
643
+ groups = candidates.groups;
644
+ }
645
+ if (groups.length === 0)
646
+ return [];
647
+ // Read-only protection: reject merge if any source observation is read-only
648
+ for (const group of groups) {
649
+ for (const obs of group.observations) {
650
+ if (obs.readOnly) {
651
+ throw new Error(`Cannot merge: observation #${obs.id} is read-only.`);
652
+ }
653
+ }
654
+ }
655
+ if (dryRun) {
656
+ return groups.map((g) => ({
657
+ mergedObservation: g.observations[0], // placeholder
658
+ deletedIds: g.observations.map((o) => o.id),
659
+ originalCount: g.observations.length,
660
+ strategy,
661
+ }));
662
+ }
663
+ const results = [];
664
+ const doMerge = this.db.transaction(() => {
665
+ for (const group of groups) {
666
+ const obs = group.observations.sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime());
667
+ // Synthesize content
668
+ const synthTitle = obs[obs.length - 1].title; // most recent title
669
+ const synthContent = obs
670
+ .map((o) => `--- [${o.createdAt.toISOString()}] ${o.title} ---\n${o.content}`)
671
+ .join('\n\n');
672
+ // Most frequent type
673
+ const typeCounts = {};
674
+ for (const o of obs) {
675
+ typeCounts[o.type] = (typeCounts[o.type] || 0) + 1;
676
+ }
677
+ const synthType = Object.entries(typeCounts).sort((a, b) => b[1] - a[1])[0][0];
678
+ const synthTopicKey = obs.find((o) => o.topicKey)?.topicKey ?? null;
679
+ const sourceIds = obs.map((o) => o.id);
680
+ const synthMetadata = this.serialize({
681
+ merged: true,
682
+ sourceIds,
683
+ mergedAt: new Date().toISOString(),
684
+ originalCount: obs.length,
685
+ });
686
+ const uuid = crypto.randomUUID();
687
+ const createdAt = Date.now();
688
+ const sessionId = obs[obs.length - 1].sessionId;
689
+ const insertResult = this.db
690
+ .prepare(`INSERT INTO observations (uuid, session_id, title, content, type, topic_key, project_id, created_at, deleted_at, metadata)
691
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL, ?)`)
692
+ .run(uuid, sessionId, synthTitle, synthContent, synthType, synthTopicKey, projectId, createdAt, synthMetadata);
693
+ const newId = typeof insertResult.lastInsertRowid === 'bigint'
694
+ ? Number(insertResult.lastInsertRowid)
695
+ : insertResult.lastInsertRowid;
696
+ // Application-level FTS5 sync: insert merged, delete originals
697
+ this.db.prepare('INSERT INTO observations_fts(rowid, title, content, topic_key, project_id) VALUES (?, ?, ?, ?, ?)').run(newId, synthTitle, synthContent, synthTopicKey ?? '', projectId);
698
+ // Delete originals (from both observations and observations_fts)
699
+ const placeholders = sourceIds.map(() => '?').join(',');
700
+ this.db.prepare(`DELETE FROM observations_fts WHERE rowid IN (${placeholders})`).run(...sourceIds);
701
+ this.db.prepare(`DELETE FROM observations WHERE id IN (${placeholders})`).run(...sourceIds);
702
+ // Fetch the newly created merged observation
703
+ const merged = this.db.prepare('SELECT * FROM observations WHERE id = ?').get(newId);
704
+ results.push({
705
+ mergedObservation: this.mapObservation(merged),
706
+ deletedIds: sourceIds,
707
+ originalCount: obs.length,
708
+ strategy,
709
+ });
710
+ }
711
+ });
712
+ doMerge();
713
+ return results;
714
+ }
715
+ jaccardSimilarity(text1, text2) {
716
+ const tokenize = (t) => new Set(t
717
+ .toLowerCase()
718
+ .split(/\s+/)
719
+ .filter((w) => w.length > 2));
720
+ const set1 = tokenize(text1);
721
+ const set2 = tokenize(text2);
722
+ const intersection = [...set1].filter((x) => set2.has(x)).length;
723
+ const union = new Set([...set1, ...set2]).size;
724
+ return union === 0 ? 0 : intersection / union;
725
+ }
726
+ // ─── Export ────────────────────────────────────────────────
727
+ async exportObservations(params) {
728
+ this.checkHealth();
729
+ const { format, projectId, type, topicKey, dateFrom, dateTo, includeDeleted = false } = params;
226
730
  let sql = 'SELECT * FROM observations WHERE 1=1';
227
731
  const values = [];
228
- if (query) {
732
+ if (!includeDeleted) {
733
+ sql += ' AND deleted_at IS NULL';
734
+ }
735
+ if (projectId) {
736
+ sql += ' AND project_id = ?';
737
+ values.push(projectId);
738
+ }
739
+ if (type) {
740
+ sql += ' AND type = ?';
741
+ values.push(type);
742
+ }
743
+ if (topicKey) {
744
+ sql += ' AND topic_key = ?';
745
+ values.push(topicKey);
746
+ }
747
+ if (dateFrom) {
748
+ sql += ' AND created_at >= ?';
749
+ values.push(dateFrom.getTime());
750
+ }
751
+ if (dateTo) {
752
+ sql += ' AND created_at <= ?';
753
+ values.push(dateTo.getTime());
754
+ }
755
+ sql += ' ORDER BY created_at ASC, id ASC';
756
+ const rows = this.db.prepare(sql).all(...values);
757
+ const observations = rows.map((row) => this.mapObservation(row));
758
+ const exportedAt = new Date();
759
+ let content;
760
+ switch (format) {
761
+ case 'json':
762
+ content = this.exportToJSON(observations, exportedAt, params);
763
+ break;
764
+ case 'xml':
765
+ content = this.exportToXML(observations, exportedAt, params);
766
+ break;
767
+ case 'txt':
768
+ content = this.exportToTXT(observations, exportedAt, params);
769
+ break;
770
+ default:
771
+ throw new Error(`Unsupported export format: ${format}`);
772
+ }
773
+ return {
774
+ content,
775
+ format,
776
+ recordCount: observations.length,
777
+ exportedAt,
778
+ };
779
+ }
780
+ exportToJSON(observations, exportedAt, params) {
781
+ const data = {
782
+ exportedAt: exportedAt.toISOString(),
783
+ version: '1.0',
784
+ project: params.projectId || 'all',
785
+ filters: {
786
+ ...(params.type && { type: params.type }),
787
+ ...(params.topicKey && { topicKey: params.topicKey }),
788
+ ...(params.dateFrom && { dateFrom: params.dateFrom.toISOString() }),
789
+ ...(params.dateTo && { dateTo: params.dateTo.toISOString() }),
790
+ ...(params.includeDeleted && { includeDeleted: true }),
791
+ },
792
+ totalRecords: observations.length,
793
+ observations: observations.map((o) => ({
794
+ id: o.id,
795
+ uuid: o.uuid,
796
+ title: o.title,
797
+ content: o.content,
798
+ type: o.type,
799
+ topicKey: o.topicKey,
800
+ projectId: o.projectId,
801
+ createdAt: o.createdAt.toISOString(),
802
+ ...(o.deletedAt && { deletedAt: o.deletedAt.toISOString() }),
803
+ metadata: o.metadata,
804
+ })),
805
+ };
806
+ return JSON.stringify(data, null, 2);
807
+ }
808
+ exportToXML(observations, exportedAt, params) {
809
+ const escapeXml = (s) => s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;');
810
+ let xml = `<?xml version="1.0" encoding="UTF-8"?>\n`;
811
+ xml += `<memento-export version="1.0" exportedAt="${exportedAt.toISOString()}">\n`;
812
+ xml += ` <project>${escapeXml(params.projectId || 'all')}</project>\n`;
813
+ xml += ` <observations count="${observations.length}">\n`;
814
+ for (const o of observations) {
815
+ xml += ` <observation id="${o.id}" uuid="${escapeXml(o.uuid)}">\n`;
816
+ xml += ` <title>${escapeXml(o.title)}</title>\n`;
817
+ xml += ` <content><![CDATA[${o.content}]]></content>\n`;
818
+ xml += ` <type>${escapeXml(o.type)}</type>\n`;
819
+ if (o.topicKey)
820
+ xml += ` <topicKey>${escapeXml(o.topicKey)}</topicKey>\n`;
821
+ xml += ` <projectId>${escapeXml(o.projectId)}</projectId>\n`;
822
+ xml += ` <createdAt>${o.createdAt.toISOString()}</createdAt>\n`;
823
+ if (o.deletedAt)
824
+ xml += ` <deletedAt>${o.deletedAt.toISOString()}</deletedAt>\n`;
825
+ xml += ` </observation>\n`;
826
+ }
827
+ xml += ` </observations>\n`;
828
+ xml += `</memento-export>\n`;
829
+ return xml;
830
+ }
831
+ exportToTXT(observations, exportedAt, params) {
832
+ const sep = '═'.repeat(50);
833
+ const thinSep = '─'.repeat(50);
834
+ const dateStr = exportedAt.toISOString().split('T')[0];
835
+ let txt = `${sep}\n`;
836
+ txt += `MEMENTO EXPORT — Project: ${params.projectId || 'all'}\n`;
837
+ txt += `Exported: ${dateStr} | Records: ${observations.length}\n`;
838
+ txt += `${sep}\n\n`;
839
+ for (const o of observations) {
840
+ txt += `[#${o.id}] ${o.title}\n`;
841
+ txt += `Type: ${o.type}`;
842
+ if (o.topicKey)
843
+ txt += ` | Topic: ${o.topicKey}`;
844
+ txt += `\nDate: ${o.createdAt.toISOString().split('T')[0]}`;
845
+ if (o.deletedAt)
846
+ txt += ` | DELETED: ${o.deletedAt.toISOString().split('T')[0]}`;
847
+ txt += `\n\n ${o.content}\n\n${thinSep}\n\n`;
848
+ }
849
+ return txt;
850
+ }
851
+ // ─── Search ────────────────────────────────────────────────
852
+ /**
853
+ * Sanitize a user query for safe use in FTS5 MATCH.
854
+ * Strips FTS5 operators and special characters that would cause syntax errors.
855
+ * Returns empty string if nothing survives sanitization.
856
+ */
857
+ sanitizeFTS5Query(input) {
858
+ return input
859
+ .replace(/[^a-zA-Z0-9\s]/g, ' ') // Keep only alphanumeric and whitespace
860
+ .replace(/\s+/g, ' ') // Collapse whitespace
861
+ .trim();
862
+ }
863
+ async getObservation(id, includeDeleted = false) {
864
+ return await this.getObservationById(id, includeDeleted);
865
+ }
866
+ async search(params) {
867
+ const mode = params.mode || 'keyword';
868
+ // Semantic and hybrid modes require a query string
869
+ if ((mode === 'semantic' || mode === 'hybrid') && !params.query) {
870
+ // Fall back to keyword mode if no query provided
871
+ return this.searchKeyword(params);
872
+ }
873
+ switch (mode) {
874
+ case 'semantic':
875
+ return this.searchSemantic(params);
876
+ case 'hybrid':
877
+ return this.searchHybrid(params);
878
+ case 'keyword':
879
+ default:
880
+ return this.searchKeyword(params);
881
+ }
882
+ }
883
+ /**
884
+ * Keyword search using existing FTS5 (unchanged behavior).
885
+ */
886
+ async searchKeyword(params) {
887
+ const { query, type, projectId, topicKey, limit = 100, offset = 0, includeDeleted = false, scope, sessionId, } = params;
888
+ let sql;
889
+ const values = [];
890
+ const sanitizedQuery = query ? this.sanitizeFTS5Query(query) : '';
891
+ if (sanitizedQuery) {
229
892
  sql =
230
893
  'SELECT observations.* FROM observations JOIN observations_fts ON observations.id = observations_fts.rowid WHERE observations_fts MATCH ?';
231
- values.push(query);
894
+ values.push(sanitizedQuery);
895
+ if (!includeDeleted) {
896
+ sql += ' AND observations.deleted_at IS NULL';
897
+ }
232
898
  if (type) {
233
899
  sql += ' AND observations.type = ?';
234
900
  values.push(type);
@@ -241,8 +907,20 @@ class MemoryEngine {
241
907
  sql += ' AND observations.topic_key = ?';
242
908
  values.push(topicKey);
243
909
  }
910
+ if (scope) {
911
+ sql += ' AND observations.scope = ?';
912
+ values.push(scope);
913
+ }
914
+ if (sessionId) {
915
+ sql += ' AND observations.session_id = ?';
916
+ values.push(sessionId);
917
+ }
244
918
  }
245
919
  else {
920
+ sql = 'SELECT * FROM observations WHERE 1=1';
921
+ if (!includeDeleted) {
922
+ sql += ' AND deleted_at IS NULL';
923
+ }
246
924
  if (type) {
247
925
  sql += ' AND type = ?';
248
926
  values.push(type);
@@ -255,16 +933,196 @@ class MemoryEngine {
255
933
  sql += ' AND topic_key = ?';
256
934
  values.push(topicKey);
257
935
  }
936
+ if (scope) {
937
+ sql += ' AND scope = ?';
938
+ values.push(scope);
939
+ }
940
+ if (sessionId) {
941
+ sql += ' AND session_id = ?';
942
+ values.push(sessionId);
943
+ }
258
944
  }
259
945
  const countSql = sql.replace(/SELECT.*?FROM/, 'SELECT COUNT(*) as count FROM');
260
946
  const countResult = this.db.prepare(countSql).get(...values);
261
947
  const total = countResult ? countResult.count : 0;
262
- sql += ' ORDER BY created_at DESC LIMIT ? OFFSET ?';
948
+ sql += ' ORDER BY created_at DESC, id DESC LIMIT ? OFFSET ?';
263
949
  values.push(limit, offset);
264
950
  const rows = this.db.prepare(sql).all(...values);
265
951
  const observations = rows.map((row) => this.mapObservation(row));
266
952
  return { observations, total };
267
953
  }
954
+ /**
955
+ * Semantic search using cosine similarity on embeddings.
956
+ * Falls back to keyword search if embeddings are unavailable.
957
+ */
958
+ async searchSemantic(params) {
959
+ this.checkHealth();
960
+ const { query, type, projectId, topicKey, limit = 10, scope } = params;
961
+ if (!query)
962
+ return { observations: [], total: 0 };
963
+ // Generate query embedding
964
+ const queryEmbedding = await this.embeddingService.generate(query);
965
+ if (!queryEmbedding) {
966
+ // Fall back to keyword search if embedding service unavailable
967
+ return this.searchKeyword(params);
968
+ }
969
+ // Build candidate filter
970
+ let candidateSql = 'SELECT o.* FROM observations o WHERE o.deleted_at IS NULL';
971
+ const values = [];
972
+ if (type) {
973
+ candidateSql += ' AND o.type = ?';
974
+ values.push(type);
975
+ }
976
+ if (projectId) {
977
+ candidateSql += ' AND o.project_id = ?';
978
+ values.push(projectId);
979
+ }
980
+ if (topicKey) {
981
+ candidateSql += ' AND o.topic_key = ?';
982
+ values.push(topicKey);
983
+ }
984
+ if (scope) {
985
+ candidateSql += ' AND o.scope = ?';
986
+ values.push(scope);
987
+ }
988
+ if (params.sessionId) {
989
+ candidateSql += ' AND o.session_id = ?';
990
+ values.push(params.sessionId);
991
+ }
992
+ // Only observations that have embeddings
993
+ candidateSql += ' AND EXISTS (SELECT 1 FROM embeddings e WHERE e.observation_id = o.id)';
994
+ const rows = this.db.prepare(candidateSql).all(...values);
995
+ const candidates = rows.map((row) => this.mapObservation(row));
996
+ // Score each candidate by cosine similarity
997
+ const scored = [];
998
+ for (const obs of candidates) {
999
+ const embeddingRow = this.db
1000
+ .prepare('SELECT embedding, dimensions FROM embeddings WHERE observation_id = ?')
1001
+ .get(obs.id);
1002
+ if (!embeddingRow)
1003
+ continue;
1004
+ const storedEmbedding = EmbeddingService_js_1.EmbeddingService.deserializeEmbedding(embeddingRow.embedding, embeddingRow.dimensions);
1005
+ const score = EmbeddingService_js_1.EmbeddingService.cosineSimilarity(queryEmbedding.embedding, storedEmbedding);
1006
+ scored.push({ observation: obs, score });
1007
+ }
1008
+ // Sort by score descending
1009
+ scored.sort((a, b) => b.score - a.score);
1010
+ const total = scored.length;
1011
+ const results = scored.slice(0, limit);
1012
+ const scores = new Map(results.map((r) => [r.observation.id, r.score]));
1013
+ const observations = results.map((r) => r.observation);
1014
+ return { observations, total, scores };
1015
+ }
1016
+ /**
1017
+ * Hybrid search combining FTS5 BM25 + cosine similarity.
1018
+ * Score = 0.4 * bm25_normalized + 0.6 * cosine_similarity
1019
+ */
1020
+ async searchHybrid(params) {
1021
+ this.checkHealth();
1022
+ const { query, limit = 10 } = params;
1023
+ if (!query)
1024
+ return { observations: [], total: 0 };
1025
+ // Run both searches in parallel
1026
+ const [keywordResult, semanticResult] = await Promise.all([
1027
+ this.searchKeyword({ ...params, limit: limit * 3 }), // Get more candidates for re-ranking
1028
+ this.searchSemantic({ ...params, limit: limit * 3 }),
1029
+ ]);
1030
+ // Normalize BM25 scores (use position-based ranking as proxy)
1031
+ const maxBm25Rank = keywordResult.observations.length;
1032
+ const bm25Scores = new Map();
1033
+ keywordResult.observations.forEach((obs, index) => {
1034
+ // Higher rank (earlier) = higher score
1035
+ bm25Scores.set(obs.id, maxBm25Rank > 0 ? 1 - index / maxBm25Rank : 0);
1036
+ });
1037
+ // Get semantic scores
1038
+ const semanticScores = semanticResult.scores || new Map();
1039
+ // Combine all unique observation IDs
1040
+ const allIds = new Set([...bm25Scores.keys(), ...semanticScores.keys()]);
1041
+ // Calculate hybrid scores
1042
+ const hybridWeight = { bm25: 0.4, cosine: 0.6 };
1043
+ const scored = [];
1044
+ for (const id of allIds) {
1045
+ const bm25 = bm25Scores.get(id) ?? 0;
1046
+ const cosine = semanticScores.get(id) ?? 0;
1047
+ const hybridScore = hybridWeight.bm25 * bm25 + hybridWeight.cosine * cosine;
1048
+ scored.push({ observationId: id, score: hybridScore });
1049
+ }
1050
+ scored.sort((a, b) => b.score - a.score);
1051
+ const topResults = scored.slice(0, limit);
1052
+ // Fetch full observations
1053
+ const observations = [];
1054
+ const scores = new Map();
1055
+ for (const result of topResults) {
1056
+ const obs = await this.getObservation(result.observationId);
1057
+ if (obs) {
1058
+ observations.push(obs);
1059
+ scores.set(obs.id, result.score);
1060
+ }
1061
+ }
1062
+ return { observations, total: scored.length, scores };
1063
+ }
1064
+ // ─── Embedding Management ──────────────────────────────────
1065
+ /**
1066
+ * Generate and store embedding for an observation.
1067
+ * Non-blocking: does not throw on failure.
1068
+ */
1069
+ async generateEmbedding(observationId) {
1070
+ this.checkHealth();
1071
+ const obs = await this.getObservationById(observationId);
1072
+ if (!obs)
1073
+ return false;
1074
+ const text = `${obs.title} ${obs.content}`;
1075
+ const result = await this.embeddingService.generate(text);
1076
+ if (!result)
1077
+ return false;
1078
+ const blob = EmbeddingService_js_1.EmbeddingService.serializeEmbedding(result.embedding);
1079
+ const now = Date.now();
1080
+ await this.withRetry(() => {
1081
+ this.db.prepare(`INSERT OR REPLACE INTO embeddings (observation_id, embedding, model, dimensions, generated_at)
1082
+ VALUES (?, ?, ?, ?, ?)`).run(observationId, blob, result.model, result.dimensions, now);
1083
+ });
1084
+ return true;
1085
+ }
1086
+ /**
1087
+ * Delete embedding for an observation.
1088
+ */
1089
+ async deleteEmbedding(observationId) {
1090
+ this.checkHealth();
1091
+ this.db.prepare('DELETE FROM embeddings WHERE observation_id = ?').run(observationId);
1092
+ }
1093
+ /**
1094
+ * Get embedding status (is the service available?).
1095
+ */
1096
+ async getEmbeddingStatus() {
1097
+ return this.embeddingService.status;
1098
+ }
1099
+ /**
1100
+ * Backfill embeddings for observations that don't have them.
1101
+ * Processes in batches. Returns count of embeddings generated.
1102
+ */
1103
+ async backfillEmbeddings(batchSize = 50) {
1104
+ this.checkHealth();
1105
+ const rows = this.db.prepare(`SELECT o.id FROM observations o
1106
+ WHERE o.deleted_at IS NULL
1107
+ AND NOT EXISTS (SELECT 1 FROM embeddings e WHERE e.observation_id = o.id)
1108
+ ORDER BY o.id ASC`).all();
1109
+ let processed = 0;
1110
+ let failed = 0;
1111
+ for (let i = 0; i < rows.length; i += batchSize) {
1112
+ const batch = rows.slice(i, i + batchSize);
1113
+ for (const row of batch) {
1114
+ const success = await this.generateEmbedding(row.id);
1115
+ if (success) {
1116
+ processed++;
1117
+ }
1118
+ else {
1119
+ failed++;
1120
+ }
1121
+ }
1122
+ }
1123
+ return { processed, failed };
1124
+ }
1125
+ // ─── Sessions ──────────────────────────────────────────────
268
1126
  async createSession(data) {
269
1127
  const uuid = crypto.randomUUID();
270
1128
  const startedAt = new Date();
@@ -278,8 +1136,62 @@ class MemoryEngine {
278
1136
  const session = await this.getSessionById(id);
279
1137
  if (!session)
280
1138
  throw new Error('Failed to retrieve created session');
1139
+ // Seed default observations if requested and this is the first session for the project
1140
+ if (data.seedIfEmpty) {
1141
+ await this.seedIfEmpty(data.projectId, id);
1142
+ }
281
1143
  return session;
282
1144
  }
1145
+ /**
1146
+ * Seed default observations if project has no observations yet.
1147
+ * Creates 3 seeds: persona (personal scope), human (personal), project (project scope).
1148
+ * Public so that `memento init` can call it directly.
1149
+ */
1150
+ async seedIfEmpty(projectId, sessionId) {
1151
+ // Check if project already has observations
1152
+ const existing = this.db
1153
+ .prepare('SELECT COUNT(*) as count FROM observations WHERE project_id = ? AND deleted_at IS NULL')
1154
+ .get(projectId);
1155
+ if (existing && existing.count > 0)
1156
+ return;
1157
+ // Seed personal-scope observations (only if no personal observations exist)
1158
+ const personalExisting = this.db
1159
+ .prepare("SELECT COUNT(*) as count FROM observations WHERE scope = 'personal' AND deleted_at IS NULL")
1160
+ .get();
1161
+ if (!personalExisting || personalExisting.count === 0) {
1162
+ await this.createObservation({
1163
+ sessionId,
1164
+ title: 'Agent Persona',
1165
+ content: 'Instructions for how the agent should behave and respond. Fill this in with your preferred communication style, tone, and approach.',
1166
+ type: 'preference',
1167
+ topicKey: 'persona',
1168
+ projectId,
1169
+ metadata: { seed: true },
1170
+ scope: 'personal',
1171
+ });
1172
+ await this.createObservation({
1173
+ sessionId,
1174
+ title: 'User Preferences',
1175
+ content: 'Key details about the user: preferred language, frameworks, communication style, constraints, and habits.',
1176
+ type: 'preference',
1177
+ topicKey: 'human',
1178
+ projectId,
1179
+ metadata: { seed: true },
1180
+ scope: 'personal',
1181
+ });
1182
+ }
1183
+ // Seed project-scope observation
1184
+ await this.createObservation({
1185
+ sessionId,
1186
+ title: 'Project Knowledge',
1187
+ content: 'Durable, high-signal information about this codebase: commands, architecture notes, conventions, and gotchas.',
1188
+ type: 'note',
1189
+ topicKey: 'project',
1190
+ projectId,
1191
+ metadata: { seed: true },
1192
+ scope: 'project',
1193
+ });
1194
+ }
283
1195
  async endSession(id) {
284
1196
  const session = await this.getSessionById(id);
285
1197
  if (!session)
@@ -291,9 +1203,69 @@ class MemoryEngine {
291
1203
  throw new Error('Failed to update session');
292
1204
  return updated;
293
1205
  }
1206
+ /**
1207
+ * Close stale (orphaned) sessions for a specific project.
1208
+ * Sessions are considered stale if they have no ended_at AND started_at is older than maxAgeMs.
1209
+ * Auto-closed sessions receive metadata: { auto_closed: true, reason: 'stale', closed_at: <timestamp> }.
1210
+ * Returns the count of closed sessions.
1211
+ */
1212
+ closeStaleSessionsForProject(projectId, maxAgeMs) {
1213
+ this.checkHealth();
1214
+ const maxAge = maxAgeMs ?? 24 * 60 * 60 * 1000; // default 24h
1215
+ const cutoff = Date.now() - maxAge;
1216
+ const now = Date.now();
1217
+ // Get sessions to close (need to update metadata individually)
1218
+ const staleRows = this.db.prepare('SELECT id, metadata FROM sessions WHERE ended_at IS NULL AND project_id = ? AND started_at < ?').all(projectId, cutoff);
1219
+ if (staleRows.length === 0)
1220
+ return { closed: 0 };
1221
+ const updateStmt = this.db.prepare('UPDATE sessions SET ended_at = ?, metadata = ? WHERE id = ?');
1222
+ let closed = 0;
1223
+ for (const row of staleRows) {
1224
+ const existingMeta = this.deserialize(row.metadata);
1225
+ const mergedMeta = {
1226
+ ...existingMeta,
1227
+ auto_closed: true,
1228
+ reason: 'stale',
1229
+ closed_at: now,
1230
+ };
1231
+ updateStmt.run(now, this.serialize(mergedMeta), row.id);
1232
+ closed++;
1233
+ }
1234
+ return { closed };
1235
+ }
1236
+ /**
1237
+ * Close ALL stale (orphaned) sessions across all projects.
1238
+ * Sessions are considered stale if they have no ended_at AND started_at is older than maxAgeMs.
1239
+ * Returns the count of closed sessions.
1240
+ */
1241
+ closeStaleSessions(maxAgeMs) {
1242
+ this.checkHealth();
1243
+ const maxAge = maxAgeMs ?? 24 * 60 * 60 * 1000; // default 24h
1244
+ const cutoff = Date.now() - maxAge;
1245
+ const now = Date.now();
1246
+ // Get sessions to close
1247
+ const staleRows = this.db.prepare('SELECT id, metadata FROM sessions WHERE ended_at IS NULL AND started_at < ?').all(cutoff);
1248
+ if (staleRows.length === 0)
1249
+ return { closed: 0 };
1250
+ const updateStmt = this.db.prepare('UPDATE sessions SET ended_at = ?, metadata = ? WHERE id = ?');
1251
+ let closed = 0;
1252
+ for (const row of staleRows) {
1253
+ const existingMeta = this.deserialize(row.metadata);
1254
+ const mergedMeta = {
1255
+ ...existingMeta,
1256
+ auto_closed: true,
1257
+ reason: 'stale',
1258
+ closed_at: now,
1259
+ };
1260
+ updateStmt.run(now, this.serialize(mergedMeta), row.id);
1261
+ closed++;
1262
+ }
1263
+ return { closed };
1264
+ }
294
1265
  async getSession(id) {
295
1266
  return await this.getSessionById(id);
296
1267
  }
1268
+ // ─── Prompts ───────────────────────────────────────────────
297
1269
  async savePrompt(data) {
298
1270
  const uuid = crypto.randomUUID();
299
1271
  const createdAt = new Date();
@@ -309,11 +1281,140 @@ class MemoryEngine {
309
1281
  throw new Error('Failed to retrieve saved prompt');
310
1282
  return prompt;
311
1283
  }
312
- async getObservationById(id) {
313
- const row = this.db.prepare('SELECT * FROM observations WHERE id = ?').get(id);
1284
+ // ─── Prompt Injection (OpenCode Plugin) ──────────────────────
1285
+ /**
1286
+ * Select observations for prompt injection based on config.
1287
+ * Returns observations sorted deterministically for prompt caching:
1288
+ * 1. Pinned observations (by ID ascending)
1289
+ * 2. Non-pinned observations (by ID ascending)
1290
+ */
1291
+ selectForPrompt(config) {
1292
+ this.checkHealth();
1293
+ const types = config.types.map((t) => `'${t}'`).join(',');
1294
+ const values = [];
1295
+ let sql = `SELECT * FROM observations WHERE deleted_at IS NULL AND type IN (${types})`;
1296
+ if (config.projectId) {
1297
+ sql += ' AND project_id = ?';
1298
+ values.push(config.projectId);
1299
+ }
1300
+ if (config.strategy === 'pinned-only') {
1301
+ sql += ' AND pinned = 1';
1302
+ }
1303
+ // Deterministic sort: pinned first (by ID ASC), then non-pinned (by ID ASC)
1304
+ sql += ' ORDER BY pinned DESC, id ASC';
1305
+ // Fetch more than needed to allow for token budget trimming
1306
+ const rows = this.db.prepare(sql).all(...values);
1307
+ const observations = rows.map((row) => this.mapObservation(row));
1308
+ // Apply maxObservations limit
1309
+ const limited = observations.slice(0, config.maxObservations);
1310
+ // Apply token budget
1311
+ const maxChars = config.maxTokens * 4; // approximate: chars / 4 = tokens
1312
+ let totalChars = 0;
1313
+ const result = [];
1314
+ for (const obs of limited) {
1315
+ const obsChars = this.estimateObservationChars(obs);
1316
+ if (totalChars + obsChars > maxChars && result.length > 0) {
1317
+ break;
1318
+ }
1319
+ totalChars += obsChars;
1320
+ result.push(obs);
1321
+ }
1322
+ // Re-sort: pinned first by ID ASC, then non-pinned by ID ASC
1323
+ // (ensures deterministic order even after budget trim)
1324
+ result.sort((a, b) => {
1325
+ if (a.pinned !== b.pinned)
1326
+ return a.pinned ? -1 : 1;
1327
+ return a.id - b.id;
1328
+ });
1329
+ return result;
1330
+ }
1331
+ /**
1332
+ * Render observations as compact XML for system prompt injection.
1333
+ */
1334
+ renderPromptContext(observations, config) {
1335
+ if (observations.length === 0) {
1336
+ return { xml: '', observationCount: 0, tokenCount: 0, budgetExceeded: false };
1337
+ }
1338
+ const parts = ['<memento_context>'];
1339
+ let totalChars = 0;
1340
+ const maxChars = config.maxTokens * 4;
1341
+ let budgetExceeded = false;
1342
+ for (const obs of observations) {
1343
+ const xmlNode = this.renderObservationXml(obs);
1344
+ const nodeChars = xmlNode.length;
1345
+ if (totalChars + nodeChars > maxChars && parts.length > 1) {
1346
+ budgetExceeded = true;
1347
+ break;
1348
+ }
1349
+ parts.push(xmlNode);
1350
+ totalChars += nodeChars;
1351
+ }
1352
+ parts.push('</memento_context>');
1353
+ const xml = parts.join('\n');
1354
+ return {
1355
+ xml,
1356
+ observationCount: parts.length - 2, // subtract opening and closing tags
1357
+ tokenCount: Math.ceil(xml.length / 4),
1358
+ budgetExceeded,
1359
+ };
1360
+ }
1361
+ renderObservationXml(obs) {
1362
+ const pinnedAttr = obs.pinned ? ' pinned="true"' : '';
1363
+ const projectAttr = obs.projectId ? ` project="${obs.projectId}"` : '';
1364
+ // Compact: title as attribute, content as text (trimmed to first 500 chars for budget)
1365
+ const truncatedContent = obs.content.length > 500
1366
+ ? obs.content.slice(0, 500) + '...'
1367
+ : obs.content;
1368
+ return `<observation id="${obs.id}" type="${obs.type}"${projectAttr}${pinnedAttr}>\n${obs.title}\n${truncatedContent}\n</observation>`;
1369
+ }
1370
+ estimateObservationChars(obs) {
1371
+ // Estimate XML overhead: ~100 chars for tags + attributes
1372
+ // Plus title + first 500 chars of content
1373
+ const contentLen = Math.min(obs.content.length, 500);
1374
+ return 100 + obs.title.length + contentLen;
1375
+ }
1376
+ /**
1377
+ * Pin an observation (always include in prompt injection).
1378
+ */
1379
+ async pinObservation(id) {
1380
+ return this.updateObservation(id, { pinned: true });
1381
+ }
1382
+ /**
1383
+ * Unpin an observation.
1384
+ */
1385
+ async unpinObservation(id) {
1386
+ return this.updateObservation(id, { pinned: false });
1387
+ }
1388
+ /**
1389
+ * Lock an observation (set read-only). Prevents all modifications.
1390
+ */
1391
+ async lockObservation(id) {
1392
+ return this.updateObservation(id, { readOnly: true });
1393
+ }
1394
+ /**
1395
+ * Unlock an observation (remove read-only protection).
1396
+ */
1397
+ async unlockObservation(id) {
1398
+ return this.updateObservation(id, { readOnly: false });
1399
+ }
1400
+ // ─── Private helpers ───────────────────────────────────────
1401
+ async getObservationById(id, includeDeleted = false) {
1402
+ let sql = 'SELECT * FROM observations WHERE id = ?';
1403
+ if (!includeDeleted) {
1404
+ sql += ' AND deleted_at IS NULL';
1405
+ }
1406
+ const row = this.db.prepare(sql).get(id);
314
1407
  if (!row)
315
1408
  return null;
316
- return this.mapObservation(row);
1409
+ const observation = this.mapObservation(row);
1410
+ // Calculate duplicatesCount when topicKey is present
1411
+ if (observation.topicKey) {
1412
+ const dupResult = this.db
1413
+ .prepare('SELECT COUNT(*) as count FROM observations WHERE topic_key = ? AND deleted_at IS NULL')
1414
+ .get(observation.topicKey);
1415
+ observation.duplicatesCount = dupResult?.count ?? 0;
1416
+ }
1417
+ return observation;
317
1418
  }
318
1419
  async getSessionById(id) {
319
1420
  const row = this.db.prepare('SELECT * FROM sessions WHERE id = ?').get(id);
@@ -339,7 +1440,12 @@ class MemoryEngine {
339
1440
  topicKey: r.topic_key,
340
1441
  projectId: r.project_id,
341
1442
  createdAt: new Date(r.created_at),
1443
+ deletedAt: r.deleted_at ? new Date(r.deleted_at) : null,
342
1444
  metadata: this.deserialize(r.metadata),
1445
+ scope: r.scope || 'project',
1446
+ pinned: Boolean(r.pinned),
1447
+ readOnly: Boolean(r.read_only),
1448
+ revisionCount: r.revision_count ?? 0,
343
1449
  };
344
1450
  }
345
1451
  mapSession(row) {
@@ -351,6 +1457,7 @@ class MemoryEngine {
351
1457
  startedAt: new Date(r.started_at),
352
1458
  endedAt: r.ended_at ? new Date(r.ended_at) : null,
353
1459
  metadata: this.deserialize(r.metadata),
1460
+ observationCount: r.observation_count ?? 0,
354
1461
  };
355
1462
  }
356
1463
  mapPrompt(row) {
@@ -365,6 +1472,1114 @@ class MemoryEngine {
365
1472
  metadata: this.deserialize(r.metadata),
366
1473
  };
367
1474
  }
1475
+ // ─── Timeline & Context ──────────────────────────────────────
1476
+ async getTimeline(params) {
1477
+ this.checkHealth();
1478
+ const { projectId, scope, limit = 50, offset = 0 } = params;
1479
+ let countSql = 'SELECT COUNT(*) as count FROM observations WHERE deleted_at IS NULL';
1480
+ let sql = 'SELECT * FROM observations WHERE deleted_at IS NULL';
1481
+ const values = [];
1482
+ if (projectId) {
1483
+ countSql += ' AND project_id = ?';
1484
+ sql += ' AND project_id = ?';
1485
+ values.push(projectId);
1486
+ }
1487
+ if (scope) {
1488
+ countSql += ' AND scope = ?';
1489
+ sql += ' AND scope = ?';
1490
+ values.push(scope);
1491
+ }
1492
+ const countResult = this.db.prepare(countSql).get(...values);
1493
+ const total = countResult?.count ?? 0;
1494
+ sql += ' ORDER BY created_at DESC, id DESC LIMIT ? OFFSET ?';
1495
+ const rows = this.db.prepare(sql).all(...values, limit, offset);
1496
+ const observations = rows.map((row) => this.mapObservation(row));
1497
+ return { observations, total };
1498
+ }
1499
+ async getRecentContext(params) {
1500
+ this.checkHealth();
1501
+ const { projectId, limit = 20, scope } = params;
1502
+ let sql = 'SELECT * FROM observations WHERE deleted_at IS NULL';
1503
+ const values = [];
1504
+ if (projectId) {
1505
+ sql += ' AND project_id = ?';
1506
+ values.push(projectId);
1507
+ }
1508
+ if (scope) {
1509
+ sql += ' AND scope = ?';
1510
+ values.push(scope);
1511
+ }
1512
+ sql += ' ORDER BY created_at DESC, id DESC LIMIT ?';
1513
+ const rows = this.db.prepare(sql).all(...values, limit);
1514
+ const observations = rows.map((row) => this.mapObservation(row));
1515
+ // Get total for context
1516
+ let countSql = 'SELECT COUNT(*) as count FROM observations WHERE deleted_at IS NULL';
1517
+ const countValues = [];
1518
+ if (projectId) {
1519
+ countSql += ' AND project_id = ?';
1520
+ countValues.push(projectId);
1521
+ }
1522
+ const countResult = this.db.prepare(countSql).get(...countValues);
1523
+ const total = countResult?.count ?? 0;
1524
+ return { observations, total };
1525
+ }
1526
+ // ─── TUI Explorer API ──────────────────────────────────────
1527
+ async listSessions(params) {
1528
+ this.checkHealth();
1529
+ const { projectId, activeOnly = false, limit = 50, offset = 0 } = params;
1530
+ // Build WHERE conditions once, reuse for count and fetch
1531
+ let whereClause = ' WHERE 1=1';
1532
+ const values = [];
1533
+ if (projectId) {
1534
+ whereClause += ' AND project_id = ?';
1535
+ values.push(projectId);
1536
+ }
1537
+ if (activeOnly) {
1538
+ whereClause += ' AND ended_at IS NULL';
1539
+ }
1540
+ // Count (no subquery needed)
1541
+ const countResult = this.db.prepare(`SELECT COUNT(*) as count FROM sessions${whereClause}`).get(...values);
1542
+ const total = countResult?.count ?? 0;
1543
+ // Fetch with observation count subquery
1544
+ const sql = `SELECT s.*,
1545
+ (SELECT COUNT(*) FROM observations o WHERE o.session_id = s.id AND o.deleted_at IS NULL) as observation_count
1546
+ FROM sessions s${whereClause}
1547
+ ORDER BY s.started_at DESC, s.id DESC LIMIT ? OFFSET ?`;
1548
+ values.push(limit, offset);
1549
+ const rows = this.db.prepare(sql).all(...values);
1550
+ const sessions = rows.map((row) => this.mapSession(row));
1551
+ return { sessions, total };
1552
+ }
1553
+ async listProjects() {
1554
+ this.checkHealth();
1555
+ // Get all distinct project_ids with their stats
1556
+ const rows = this.db
1557
+ .prepare(`
1558
+ SELECT
1559
+ project_id,
1560
+ SUM(CASE WHEN deleted_at IS NULL THEN 1 ELSE 0 END) as active_count,
1561
+ SUM(CASE WHEN deleted_at IS NOT NULL THEN 1 ELSE 0 END) as deleted_count,
1562
+ MAX(created_at) as last_activity,
1563
+ MAX(id) as max_id
1564
+ FROM observations
1565
+ GROUP BY project_id
1566
+ ORDER BY last_activity DESC, max_id DESC
1567
+ `)
1568
+ .all();
1569
+ const projects = rows.map((row) => {
1570
+ const projectId = row.project_id;
1571
+ // Get type distribution for this project
1572
+ const typeRows = this.db
1573
+ .prepare(`
1574
+ SELECT type, COUNT(*) as count
1575
+ FROM observations
1576
+ WHERE project_id = ?
1577
+ GROUP BY type
1578
+ `)
1579
+ .all(projectId);
1580
+ const byType = {
1581
+ decision: 0,
1582
+ bug: 0,
1583
+ discovery: 0,
1584
+ note: 0,
1585
+ summary: 0,
1586
+ learning: 0,
1587
+ pattern: 0,
1588
+ architecture: 0,
1589
+ config: 0,
1590
+ preference: 0,
1591
+ };
1592
+ for (const tr of typeRows) {
1593
+ byType[tr.type] = tr.count;
1594
+ }
1595
+ return {
1596
+ name: projectId,
1597
+ activeCount: row.active_count,
1598
+ deletedCount: row.deleted_count,
1599
+ lastActivity: row.last_activity ? new Date(row.last_activity) : null,
1600
+ byType,
1601
+ };
1602
+ });
1603
+ return projects;
1604
+ }
1605
+ // ─── Project Registry (Issue #177) ────────────────────────
1606
+ /**
1607
+ * Register a project in the projects table.
1608
+ * Uses INSERT OR IGNORE to be idempotent.
1609
+ * Returns the registered project name (normalized).
1610
+ */
1611
+ registerProject(name, workingDir) {
1612
+ this.checkHealth();
1613
+ const { normalizeProjectId } = require('./ConfigManager');
1614
+ const normalizedName = normalizeProjectId(name);
1615
+ const now = Date.now();
1616
+ this.db
1617
+ .prepare(`INSERT OR IGNORE INTO projects (name, created_at, metadata, working_dir, aliases) VALUES (?, ?, ?, ?, ?)`)
1618
+ .run(normalizedName, now, '{}', workingDir || null, '[]');
1619
+ return normalizedName;
1620
+ }
1621
+ /**
1622
+ * Get all registered project names from the projects table.
1623
+ */
1624
+ listRegisteredProjects() {
1625
+ this.checkHealth();
1626
+ const rows = this.db
1627
+ .prepare('SELECT name, working_dir, aliases FROM projects ORDER BY name')
1628
+ .all();
1629
+ return rows.map((row) => ({
1630
+ name: row.name,
1631
+ workingDir: row.working_dir,
1632
+ aliases: row.aliases ? JSON.parse(row.aliases) : [],
1633
+ }));
1634
+ }
1635
+ /**
1636
+ * Merge all observations and sessions from sourceProject into targetProject.
1637
+ * Also updates FTS index. Returns count of affected records.
1638
+ *
1639
+ * Tries both the raw source name and the normalized version to handle
1640
+ * un-normalized data in the database.
1641
+ */
1642
+ mergeProject(sourceProjectId, targetProjectId) {
1643
+ this.checkHealth();
1644
+ const { normalizeProjectId } = require('./ConfigManager');
1645
+ const normalizedSource = normalizeProjectId(sourceProjectId);
1646
+ const target = normalizeProjectId(targetProjectId);
1647
+ // Check same-name BEFORE searching for data
1648
+ if (normalizedSource === target) {
1649
+ throw new Error(`Source and target are the same after normalization: "${normalizedSource}"`);
1650
+ }
1651
+ // Resolve source: try raw name first, then normalized
1652
+ const candidates = [sourceProjectId, normalizedSource];
1653
+ let source = null;
1654
+ for (const candidate of candidates) {
1655
+ if (candidate === target)
1656
+ continue; // skip if same as target
1657
+ const count = this.db
1658
+ .prepare('SELECT COUNT(*) as count FROM observations WHERE project_id = ?')
1659
+ .get(candidate);
1660
+ if (count.count > 0) {
1661
+ source = candidate;
1662
+ break;
1663
+ }
1664
+ // Also check sessions and journal
1665
+ const sessionCount = this.db
1666
+ .prepare('SELECT COUNT(*) as count FROM sessions WHERE project_id = ?')
1667
+ .get(candidate);
1668
+ const journalCount = this.db
1669
+ .prepare('SELECT COUNT(*) as count FROM journal WHERE project_id = ?')
1670
+ .get(candidate);
1671
+ if (sessionCount.count > 0 || journalCount.count > 0) {
1672
+ source = candidate;
1673
+ break;
1674
+ }
1675
+ }
1676
+ if (!source) {
1677
+ throw new Error(`No data found for source project: "${sourceProjectId}" (also tried "${normalizedSource}")`);
1678
+ }
1679
+ // Move observations
1680
+ const obsResult = this.db
1681
+ .prepare('UPDATE observations SET project_id = ? WHERE project_id = ?')
1682
+ .run(target, source);
1683
+ // Move sessions
1684
+ const sessionResult = this.db
1685
+ .prepare('UPDATE sessions SET project_id = ? WHERE project_id = ?')
1686
+ .run(target, source);
1687
+ // Move journal entries
1688
+ const journalResult = this.db
1689
+ .prepare('UPDATE journal SET project_id = ? WHERE project_id = ?')
1690
+ .run(target, source);
1691
+ // Move prompts
1692
+ const promptsResult = this.db
1693
+ .prepare('UPDATE prompts SET project_id = ? WHERE project_id = ?')
1694
+ .run(target, source);
1695
+ // Re-index FTS for moved observations (delete old + insert new)
1696
+ this.db.prepare('DELETE FROM observations_fts WHERE project_id = ?').run(source);
1697
+ const movedObs = this.db
1698
+ .prepare('SELECT id, title, content, topic_key, project_id FROM observations WHERE project_id = ?')
1699
+ .all(target);
1700
+ for (const obs of movedObs) {
1701
+ this.db
1702
+ .prepare('INSERT OR REPLACE INTO observations_fts(rowid, title, content, topic_key, project_id) VALUES (?, ?, ?, ?, ?)')
1703
+ .run(obs.id, obs.title, obs.content, obs.topic_key ?? '', obs.project_id);
1704
+ }
1705
+ // Add source as alias of target in projects table
1706
+ try {
1707
+ const existing = this.db
1708
+ .prepare('SELECT aliases FROM projects WHERE name = ?')
1709
+ .get(target);
1710
+ if (existing) {
1711
+ const aliases = existing.aliases ? JSON.parse(existing.aliases) : [];
1712
+ if (!aliases.includes(source)) {
1713
+ aliases.push(source);
1714
+ this.db.prepare('UPDATE projects SET aliases = ? WHERE name = ?').run(JSON.stringify(aliases), target);
1715
+ }
1716
+ }
1717
+ }
1718
+ catch {
1719
+ // Non-critical — alias tracking is best-effort
1720
+ }
1721
+ return {
1722
+ observationsMoved: obsResult.changes,
1723
+ sessionsMoved: sessionResult.changes,
1724
+ journalMoved: journalResult.changes,
1725
+ promptsMoved: promptsResult.changes,
1726
+ };
1727
+ }
1728
+ /**
1729
+ * Normalize all project_id values in the database.
1730
+ * After normalization, merges projects that became identical.
1731
+ * Returns details of what was changed.
1732
+ */
1733
+ normalizeAllProjectIds() {
1734
+ this.checkHealth();
1735
+ const { normalizeProjectId } = require('./ConfigManager');
1736
+ // Get all distinct project_ids
1737
+ const obsProjects = this.db
1738
+ .prepare('SELECT DISTINCT project_id FROM observations')
1739
+ .all();
1740
+ const sessionProjects = this.db
1741
+ .prepare('SELECT DISTINCT project_id FROM sessions')
1742
+ .all();
1743
+ const allProjects = new Set([
1744
+ ...obsProjects.map((r) => r.project_id),
1745
+ ...sessionProjects.map((r) => r.project_id),
1746
+ ]);
1747
+ // Build normalization map: original → normalized
1748
+ const normalizationMap = new Map();
1749
+ for (const original of allProjects) {
1750
+ const normalized = normalizeProjectId(original);
1751
+ if (normalized !== original) {
1752
+ normalizationMap.set(original, normalized);
1753
+ }
1754
+ }
1755
+ // Apply normalizations
1756
+ let normalizedCount = 0;
1757
+ for (const [original, normalized] of normalizationMap) {
1758
+ // Update observations
1759
+ const obsResult = this.db
1760
+ .prepare('UPDATE observations SET project_id = ? WHERE project_id = ?')
1761
+ .run(normalized, original);
1762
+ normalizedCount += obsResult.changes;
1763
+ // Update sessions
1764
+ const sessionResult = this.db
1765
+ .prepare('UPDATE sessions SET project_id = ? WHERE project_id = ?')
1766
+ .run(normalized, original);
1767
+ normalizedCount += sessionResult.changes;
1768
+ // Update journal
1769
+ const journalResult = this.db
1770
+ .prepare('UPDATE journal SET project_id = ? WHERE project_id = ?')
1771
+ .run(normalized, original);
1772
+ normalizedCount += journalResult.changes;
1773
+ // Update prompts
1774
+ const promptsResult = this.db
1775
+ .prepare('UPDATE prompts SET project_id = ? WHERE project_id = ?')
1776
+ .run(normalized, original);
1777
+ normalizedCount += promptsResult.changes;
1778
+ }
1779
+ // After normalization, find duplicates (same normalized name) and merge
1780
+ const currentProjects = this.db
1781
+ .prepare('SELECT DISTINCT project_id FROM observations')
1782
+ .all();
1783
+ // Group by normalized name to find duplicates — but since we already normalized,
1784
+ // just check for remaining duplicates (shouldn't be any after normalization above)
1785
+ // The main purpose of mergeProject is for MANUAL consolidation of differently-named projects
1786
+ // that happen to represent the same real-world project
1787
+ // Re-index FTS completely for affected projects
1788
+ if (normalizedCount > 0) {
1789
+ // Delete and rebuild FTS for all changed projects
1790
+ for (const normalized of new Set(normalizationMap.values())) {
1791
+ this.db.prepare('DELETE FROM observations_fts WHERE project_id = ?').run(normalized);
1792
+ const obs = this.db
1793
+ .prepare('SELECT id, title, content, topic_key, project_id FROM observations WHERE project_id = ?')
1794
+ .all(normalized);
1795
+ for (const o of obs) {
1796
+ this.db
1797
+ .prepare('INSERT OR REPLACE INTO observations_fts(rowid, title, content, topic_key, project_id) VALUES (?, ?, ?, ?, ?)')
1798
+ .run(o.id, o.title, o.content, o.topic_key ?? '', o.project_id);
1799
+ }
1800
+ }
1801
+ }
1802
+ return {
1803
+ normalized: normalizedCount,
1804
+ merged: [],
1805
+ };
1806
+ }
1807
+ async getDashboardStats() {
1808
+ this.checkHealth();
1809
+ // Total counts
1810
+ const countRow = this.db
1811
+ .prepare(`
1812
+ SELECT
1813
+ COUNT(*) as total,
1814
+ SUM(CASE WHEN deleted_at IS NULL THEN 1 ELSE 0 END) as active,
1815
+ SUM(CASE WHEN deleted_at IS NOT NULL THEN 1 ELSE 0 END) as deleted
1816
+ FROM observations
1817
+ `)
1818
+ .get();
1819
+ // By type
1820
+ const typeRows = this.db
1821
+ .prepare(`
1822
+ SELECT type, COUNT(*) as count
1823
+ FROM observations
1824
+ WHERE deleted_at IS NULL
1825
+ GROUP BY type
1826
+ `)
1827
+ .all();
1828
+ const byType = { decision: 0, bug: 0, discovery: 0, note: 0, summary: 0, learning: 0, pattern: 0, architecture: 0, config: 0, preference: 0 };
1829
+ for (const tr of typeRows) {
1830
+ byType[tr.type] = tr.count;
1831
+ }
1832
+ // By project (active only)
1833
+ const projectRows = this.db
1834
+ .prepare(`
1835
+ SELECT project_id, COUNT(*) as count
1836
+ FROM observations
1837
+ WHERE deleted_at IS NULL
1838
+ GROUP BY project_id
1839
+ ORDER BY count DESC
1840
+ `)
1841
+ .all();
1842
+ const byProject = {};
1843
+ for (const pr of projectRows) {
1844
+ byProject[pr.project_id] = pr.count;
1845
+ }
1846
+ // Active sessions
1847
+ const sessionRow = this.db
1848
+ .prepare('SELECT COUNT(*) as count FROM sessions WHERE ended_at IS NULL')
1849
+ .get();
1850
+ // Recent 5 observations (active only)
1851
+ const recentRows = this.db
1852
+ .prepare(`
1853
+ SELECT * FROM observations
1854
+ WHERE deleted_at IS NULL
1855
+ ORDER BY created_at DESC, id DESC
1856
+ LIMIT 5
1857
+ `)
1858
+ .all();
1859
+ const recentObservations = recentRows.map((row) => this.mapObservation(row));
1860
+ return {
1861
+ totalObservations: countRow.total ?? 0,
1862
+ activeObservations: countRow.active ?? 0,
1863
+ deletedObservations: countRow.deleted ?? 0,
1864
+ byType,
1865
+ byProject,
1866
+ activeSessions: sessionRow.count,
1867
+ recentObservations,
1868
+ };
1869
+ }
1870
+ // ─── Import/Export ──────────────────────────────────────────
1871
+ async exportToJson(options) {
1872
+ this.checkHealth();
1873
+ const { projectId, includeSessions = false } = options || {};
1874
+ const searchResult = await this.search({ projectId, limit: 100000 });
1875
+ const observations = searchResult.observations.map((obs) => ({
1876
+ uuid: obs.uuid,
1877
+ title: obs.title,
1878
+ content: obs.content,
1879
+ type: obs.type,
1880
+ topicKey: obs.topicKey,
1881
+ projectId: obs.projectId,
1882
+ metadata: obs.metadata,
1883
+ createdAt: obs.createdAt.toISOString(),
1884
+ }));
1885
+ let sessions;
1886
+ if (includeSessions) {
1887
+ const sessionIds = [...new Set(searchResult.observations.map((o) => o.sessionId))];
1888
+ const sessionResults = await Promise.all(sessionIds.map((id) => this.getSession(id)));
1889
+ sessions = sessionResults
1890
+ .filter((s) => s !== null)
1891
+ .map((s) => ({
1892
+ uuid: s.uuid,
1893
+ projectId: s.projectId,
1894
+ startedAt: s.startedAt.toISOString(),
1895
+ endedAt: s.endedAt?.toISOString() || null,
1896
+ metadata: s.metadata,
1897
+ }));
1898
+ }
1899
+ return {
1900
+ version: '1.0',
1901
+ exportedAt: new Date().toISOString(),
1902
+ project: projectId,
1903
+ observations,
1904
+ ...(includeSessions && sessions && { sessions }),
1905
+ };
1906
+ }
1907
+ async importFromJson(data, options) {
1908
+ this.checkHealth();
1909
+ const { conflictStrategy = 'skip', dryRun = false, projectId } = options || {};
1910
+ // Validate schema
1911
+ if (!data.version || !Array.isArray(data.observations)) {
1912
+ throw new Error('Invalid import data: missing version or observations array');
1913
+ }
1914
+ const validTypes = ['decision', 'bug', 'discovery', 'note', 'summary', 'learning', 'pattern', 'architecture', 'config', 'preference'];
1915
+ const targetProject = projectId || data.project || 'import';
1916
+ const result = {
1917
+ imported: 0,
1918
+ skipped: 0,
1919
+ overwritten: 0,
1920
+ failed: 0,
1921
+ errors: [],
1922
+ observations: [],
1923
+ };
1924
+ if (data.observations.length === 0) {
1925
+ return result;
1926
+ }
1927
+ // Create import session
1928
+ const session = await this.createSession({
1929
+ projectId: targetProject,
1930
+ endedAt: null,
1931
+ metadata: { type: 'import', source: 'mem_import', timestamp: new Date().toISOString() },
1932
+ });
1933
+ // Use db.transaction() for safe atomic import (#70)
1934
+ // Replaces manual BEGIN/COMMIT which was fragile — if createObservation's
1935
+ // withRetry() hit SQLITE_BUSY inside a manual transaction, the retry would
1936
+ // fail because the transaction was already in error state.
1937
+ const doImport = this.db.transaction(() => {
1938
+ for (const obs of data.observations) {
1939
+ // Validate required fields
1940
+ if (!obs.title || !obs.content || !obs.type) {
1941
+ result.errors.push(`Invalid observation: missing required fields (title, content, type)`);
1942
+ if (conflictStrategy === 'fail') {
1943
+ result.failed++;
1944
+ throw new Error('IMPORT_ABORT');
1945
+ }
1946
+ result.failed++;
1947
+ continue;
1948
+ }
1949
+ // Validate type
1950
+ if (!validTypes.includes(obs.type)) {
1951
+ result.errors.push(`Invalid type: "${obs.type}". Valid types: ${validTypes.join(', ')}`);
1952
+ if (conflictStrategy === 'fail') {
1953
+ result.failed++;
1954
+ throw new Error('IMPORT_ABORT');
1955
+ }
1956
+ result.failed++;
1957
+ continue;
1958
+ }
1959
+ // Check for duplicates by uuid
1960
+ if (obs.uuid) {
1961
+ const existing = this.findObservationByUuid(obs.uuid);
1962
+ if (existing) {
1963
+ if (conflictStrategy === 'skip') {
1964
+ result.skipped++;
1965
+ continue;
1966
+ }
1967
+ if (conflictStrategy === 'overwrite') {
1968
+ if (!dryRun) {
1969
+ // Direct UPDATE + FTS5 sync (avoids withRetry inside transaction)
1970
+ const fields = [];
1971
+ const values = [];
1972
+ if (obs.title !== undefined) {
1973
+ fields.push('title = ?');
1974
+ values.push(obs.title);
1975
+ }
1976
+ if (obs.content !== undefined) {
1977
+ fields.push('content = ?');
1978
+ values.push(obs.content);
1979
+ }
1980
+ if (obs.type !== undefined) {
1981
+ fields.push('type = ?');
1982
+ values.push(obs.type);
1983
+ }
1984
+ if (obs.topicKey !== undefined) {
1985
+ fields.push('topic_key = ?');
1986
+ values.push(obs.topicKey ?? null);
1987
+ }
1988
+ if (obs.metadata !== undefined) {
1989
+ fields.push('metadata = ?');
1990
+ values.push(this.serialize(obs.metadata || {}));
1991
+ }
1992
+ fields.push('revision_count = revision_count + 1');
1993
+ values.push(existing.id);
1994
+ this.db.prepare(`UPDATE observations SET ${fields.join(', ')} WHERE id = ?`).run(...values);
1995
+ // FTS5 sync — delete + re-insert
1996
+ this.db.prepare('DELETE FROM observations_fts WHERE rowid = ?').run(existing.id);
1997
+ const row = this.db
1998
+ .prepare('SELECT title, content, topic_key, project_id FROM observations WHERE id = ?')
1999
+ .get(existing.id);
2000
+ if (row) {
2001
+ this.db.prepare('INSERT INTO observations_fts(rowid, title, content, topic_key, project_id) VALUES (?, ?, ?, ?, ?)').run(existing.id, row.title, row.content, row.topic_key, row.project_id);
2002
+ }
2003
+ }
2004
+ result.overwritten++;
2005
+ continue;
2006
+ }
2007
+ if (conflictStrategy === 'fail') {
2008
+ result.errors.push(`Duplicate uuid: ${obs.uuid}`);
2009
+ result.failed++;
2010
+ throw new Error('IMPORT_ABORT');
2011
+ }
2012
+ }
2013
+ }
2014
+ if (!dryRun) {
2015
+ // Direct INSERT + FTS5 sync (avoids withRetry inside transaction)
2016
+ const uuid = obs.uuid || crypto.randomUUID();
2017
+ const metadata = this.serialize(obs.metadata || {});
2018
+ const createdAt = Date.now();
2019
+ const insertResult = this.db.prepare(`INSERT INTO observations (uuid, session_id, title, content, type, topic_key, project_id, created_at, deleted_at, metadata, scope) VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, 'project')`).run(uuid, session.id, obs.title, obs.content, obs.type, obs.topicKey ?? null, targetProject, createdAt, metadata);
2020
+ const insertId = typeof insertResult.lastInsertRowid === 'bigint'
2021
+ ? Number(insertResult.lastInsertRowid)
2022
+ : insertResult.lastInsertRowid;
2023
+ // FTS5 sync
2024
+ this.db.prepare('INSERT INTO observations_fts(rowid, title, content, topic_key, project_id) VALUES (?, ?, ?, ?, ?)').run(insertId, obs.title, obs.content, obs.topicKey ?? '', targetProject);
2025
+ // Fetch created observation for result
2026
+ const importedRow = this.db.prepare('SELECT * FROM observations WHERE id = ?').get(insertId);
2027
+ if (importedRow) {
2028
+ result.observations.push(this.mapObservation(importedRow));
2029
+ }
2030
+ }
2031
+ result.imported++;
2032
+ }
2033
+ });
2034
+ try {
2035
+ doImport();
2036
+ }
2037
+ catch (error) {
2038
+ if (error instanceof Error && error.message === 'IMPORT_ABORT') {
2039
+ // Transaction auto-rolled back; return partial result with errors
2040
+ return result;
2041
+ }
2042
+ throw error;
2043
+ }
2044
+ return result;
2045
+ }
2046
+ findObservationByUuid(uuid) {
2047
+ const row = this.db.prepare('SELECT * FROM observations WHERE uuid = ?').get(uuid);
2048
+ if (!row)
2049
+ return null;
2050
+ return this.mapObservation(row);
2051
+ }
2052
+ async resetFull() {
2053
+ this.checkHealth();
2054
+ // Count before reset
2055
+ const countResult = this.db.prepare('SELECT COUNT(*) as count FROM observations').get();
2056
+ const deleted = countResult?.count || 0;
2057
+ // Drop everything (no observation triggers — FTS5 sync is application-level)
2058
+ this.db.exec('DROP TABLE IF EXISTS observations_fts');
2059
+ this.db.exec('DROP TRIGGER IF EXISTS journal_ai');
2060
+ this.db.exec('DROP TABLE IF EXISTS journal_fts');
2061
+ this.db.exec('DROP TABLE IF EXISTS journal_tags');
2062
+ this.db.exec('DROP TABLE IF EXISTS journal');
2063
+ this.db.exec('DROP TABLE IF EXISTS observations');
2064
+ this.db.exec('DROP TABLE IF EXISTS prompts');
2065
+ this.db.exec('DROP TABLE IF EXISTS sessions');
2066
+ this.db.exec('DROP TABLE IF EXISTS projects');
2067
+ // Recreate schema
2068
+ this.initializeDatabase();
2069
+ return { deleted };
2070
+ }
2071
+ async resetProject(projectId) {
2072
+ this.checkHealth();
2073
+ // Count observations to delete
2074
+ const obsCount = this.db
2075
+ .prepare('SELECT COUNT(*) as count FROM observations WHERE project_id = ?')
2076
+ .get(projectId);
2077
+ const promptCount = this.db
2078
+ .prepare('SELECT COUNT(*) as count FROM prompts WHERE project_id = ?')
2079
+ .get(projectId);
2080
+ const deleted = obsCount.count + promptCount.count;
2081
+ // Delete observations (FTS5 first, then main table) and prompts for this project
2082
+ // Application-level FTS5 sync (was trigger observations_ad)
2083
+ this.db.prepare('DELETE FROM observations_fts WHERE rowid IN (SELECT id FROM observations WHERE project_id = ?)').run(projectId);
2084
+ this.db.prepare('DELETE FROM observations WHERE project_id = ?').run(projectId);
2085
+ this.db.prepare('DELETE FROM prompts WHERE project_id = ?').run(projectId);
2086
+ // Clean up orphan sessions (no observations AND no prompts)
2087
+ const orphanResult = this.db.prepare(`
2088
+ DELETE FROM sessions WHERE id NOT IN (
2089
+ SELECT DISTINCT session_id FROM observations
2090
+ UNION
2091
+ SELECT DISTINCT session_id FROM prompts
2092
+ )
2093
+ `).run();
2094
+ const orphanSessions = typeof orphanResult.changes === 'bigint'
2095
+ ? Number(orphanResult.changes)
2096
+ : orphanResult.changes;
2097
+ return { deleted, orphanSessions };
2098
+ }
2099
+ async countByProject(projectId) {
2100
+ this.checkHealth();
2101
+ const obsCount = this.db
2102
+ .prepare('SELECT COUNT(*) as count FROM observations WHERE project_id = ?')
2103
+ .get(projectId);
2104
+ const promptCount = this.db
2105
+ .prepare('SELECT COUNT(*) as count FROM prompts WHERE project_id = ?')
2106
+ .get(projectId);
2107
+ const sessionCount = this.db
2108
+ .prepare('SELECT COUNT(DISTINCT session_id) as count FROM observations WHERE project_id = ?')
2109
+ .get(projectId);
2110
+ return {
2111
+ observations: obsCount.count,
2112
+ prompts: promptCount.count,
2113
+ sessions: sessionCount.count,
2114
+ };
2115
+ }
2116
+ // ─── Journal (append-only evidence) ──────────────────────────
2117
+ async writeJournal(data) {
2118
+ this.checkHealth();
2119
+ const uuid = crypto.randomUUID();
2120
+ const createdAt = Date.now();
2121
+ const metadata = this.serialize(data.metadata || {});
2122
+ // If supersedes is set, validate and invalidate the old entry
2123
+ if (data.supersedes) {
2124
+ const existing = await this.readJournal(data.supersedes);
2125
+ if (!existing)
2126
+ throw new Error(`Journal entry ${data.supersedes} not found`);
2127
+ if (existing.invalidatedAt)
2128
+ throw new Error(`Journal entry ${data.supersedes} is already invalidated`);
2129
+ }
2130
+ const id = await this.withRetry(() => {
2131
+ const result = this.db
2132
+ .prepare(`INSERT INTO journal (uuid, project_id, session_id, title, body, model, provider, agent, superseded_by, invalidated_at, metadata, created_at)
2133
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, ?, ?)`)
2134
+ .run(uuid, data.projectId, data.sessionId ?? null, data.title, data.body, data.model ?? null, data.provider ?? null, data.agent ?? null, metadata, createdAt);
2135
+ return typeof result.lastInsertRowid === 'bigint'
2136
+ ? Number(result.lastInsertRowid)
2137
+ : result.lastInsertRowid;
2138
+ });
2139
+ // Insert tags
2140
+ if (data.tags && data.tags.length > 0) {
2141
+ const insertTag = this.db.prepare('INSERT OR IGNORE INTO journal_tags (journal_id, tag) VALUES (?, ?)');
2142
+ for (const tag of data.tags) {
2143
+ insertTag.run(id, tag);
2144
+ }
2145
+ }
2146
+ // If supersedes, mark the old entry
2147
+ if (data.supersedes) {
2148
+ await this.withRetry(() => this.db
2149
+ .prepare('UPDATE journal SET superseded_by = ?, invalidated_at = ? WHERE id = ?')
2150
+ .run(id, createdAt, data.supersedes));
2151
+ }
2152
+ const entry = await this.readJournal(id);
2153
+ if (!entry)
2154
+ throw new Error('Failed to retrieve created journal entry');
2155
+ return entry;
2156
+ }
2157
+ async readJournal(id) {
2158
+ this.checkHealth();
2159
+ const row = this.db.prepare('SELECT * FROM journal WHERE id = ?').get(id);
2160
+ if (!row)
2161
+ return null;
2162
+ // Fetch tags
2163
+ const tagRows = this.db
2164
+ .prepare('SELECT tag FROM journal_tags WHERE journal_id = ?')
2165
+ .all(id);
2166
+ const tags = tagRows.map((t) => t.tag);
2167
+ return this.mapJournalEntry(row, tags);
2168
+ }
2169
+ async searchJournal(params) {
2170
+ this.checkHealth();
2171
+ const { query, tags, projectId, sessionId, dateFrom, dateTo, activeOnly = false, limit = 50, offset = 0, } = params;
2172
+ let sql;
2173
+ const values = [];
2174
+ const sanitizedQuery = query ? this.sanitizeFTS5Query(query) : '';
2175
+ if (sanitizedQuery) {
2176
+ // FTS5 search — qualify column names to avoid ambiguity with journal_fts
2177
+ sql =
2178
+ 'SELECT journal.* FROM journal JOIN journal_fts ON journal.id = journal_fts.rowid WHERE journal_fts MATCH ?';
2179
+ values.push(sanitizedQuery);
2180
+ }
2181
+ else {
2182
+ sql = 'SELECT * FROM journal WHERE 1=1';
2183
+ }
2184
+ if (activeOnly) {
2185
+ sql += ' AND journal.invalidated_at IS NULL';
2186
+ }
2187
+ if (projectId) {
2188
+ sql += ' AND journal.project_id = ?';
2189
+ values.push(projectId);
2190
+ }
2191
+ if (sessionId) {
2192
+ sql += ' AND journal.session_id = ?';
2193
+ values.push(sessionId);
2194
+ }
2195
+ if (dateFrom) {
2196
+ sql += ' AND journal.created_at >= ?';
2197
+ values.push(dateFrom.getTime());
2198
+ }
2199
+ if (dateTo) {
2200
+ sql += ' AND journal.created_at <= ?';
2201
+ values.push(dateTo.getTime());
2202
+ }
2203
+ // Tag filtering — if tags specified, intersect with journal_tags
2204
+ if (tags && tags.length > 0) {
2205
+ // Find journal entries that have ALL specified tags
2206
+ const tagPlaceholders = tags.map(() => '?').join(',');
2207
+ const tagSubquery = `
2208
+ AND journal.id IN (
2209
+ SELECT jt.journal_id FROM journal_tags jt
2210
+ WHERE jt.tag IN (${tagPlaceholders})
2211
+ GROUP BY jt.journal_id
2212
+ HAVING COUNT(DISTINCT jt.tag) = ${tags.length}
2213
+ )
2214
+ `;
2215
+ sql += tagSubquery;
2216
+ values.push(...tags);
2217
+ }
2218
+ // Count
2219
+ const countSql = sql.replace(/SELECT.*?FROM/, 'SELECT COUNT(*) as count FROM');
2220
+ const countResult = this.db.prepare(countSql).get(...values);
2221
+ const total = countResult?.count ?? 0;
2222
+ // Fetch
2223
+ sql += ' ORDER BY journal.created_at DESC, journal.id DESC LIMIT ? OFFSET ?';
2224
+ values.push(limit, offset);
2225
+ const rows = this.db.prepare(sql).all(...values);
2226
+ // Fetch tags for each entry
2227
+ const entries = [];
2228
+ for (const row of rows) {
2229
+ const entryId = row.id;
2230
+ const tagRows = this.db
2231
+ .prepare('SELECT tag FROM journal_tags WHERE journal_id = ?')
2232
+ .all(entryId);
2233
+ entries.push(this.mapJournalEntry(row, tagRows.map((t) => t.tag)));
2234
+ }
2235
+ return { entries, total };
2236
+ }
2237
+ async invalidateJournal(id, supersededById) {
2238
+ this.checkHealth();
2239
+ const entry = await this.readJournal(id);
2240
+ if (!entry)
2241
+ throw new Error(`Journal entry ${id} not found`);
2242
+ if (entry.invalidatedAt)
2243
+ throw new Error(`Journal entry ${id} is already invalidated`);
2244
+ // Validate the superseding entry exists
2245
+ const superseding = await this.readJournal(supersededById);
2246
+ if (!superseding)
2247
+ throw new Error(`Superseding journal entry ${supersededById} not found`);
2248
+ const now = Date.now();
2249
+ await this.withRetry(() => this.db
2250
+ .prepare('UPDATE journal SET superseded_by = ?, invalidated_at = ? WHERE id = ?')
2251
+ .run(supersededById, now, id));
2252
+ }
2253
+ mapJournalEntry(row, tags) {
2254
+ const r = row;
2255
+ return {
2256
+ id: r.id,
2257
+ uuid: r.uuid,
2258
+ projectId: r.project_id,
2259
+ sessionId: r.session_id,
2260
+ title: r.title,
2261
+ body: r.body,
2262
+ tags,
2263
+ model: r.model,
2264
+ provider: r.provider,
2265
+ agent: r.agent,
2266
+ supersededBy: r.superseded_by,
2267
+ invalidatedAt: r.invalidated_at ? new Date(r.invalidated_at) : null,
2268
+ metadata: this.deserialize(r.metadata),
2269
+ createdAt: new Date(r.created_at),
2270
+ };
2271
+ }
2272
+ // ─── Full Export/Import (v2.0) ────────────────────────────────
2273
+ async listPrompts(params) {
2274
+ this.checkHealth();
2275
+ const { projectId, sessionId, limit = 50, offset = 0 } = params || {};
2276
+ let whereClause = ' WHERE 1=1';
2277
+ const values = [];
2278
+ if (projectId) {
2279
+ whereClause += ' AND project_id = ?';
2280
+ values.push(projectId);
2281
+ }
2282
+ if (sessionId) {
2283
+ whereClause += ' AND session_id = ?';
2284
+ values.push(sessionId);
2285
+ }
2286
+ const countResult = this.db.prepare(`SELECT COUNT(*) as count FROM prompts${whereClause}`).get(...values);
2287
+ const total = countResult?.count ?? 0;
2288
+ const rows = this.db.prepare(`SELECT * FROM prompts${whereClause} ORDER BY created_at ASC LIMIT ? OFFSET ?`).all(...values, limit, offset);
2289
+ const prompts = rows.map((row) => this.mapPrompt(row));
2290
+ return { prompts, total };
2291
+ }
2292
+ async exportProject(projectId) {
2293
+ this.checkHealth();
2294
+ // Fetch all data in parallel using Promise.all
2295
+ const [observationsResult, sessionsResult, journalResult, promptsResult,] = await Promise.all([
2296
+ this.search({ projectId, limit: 100000, includeDeleted: true }),
2297
+ this.listSessions({ projectId, limit: 100000 }),
2298
+ this.searchJournal({ projectId, limit: 100000 }),
2299
+ this.listPrompts({ projectId, limit: 100000 }),
2300
+ ]);
2301
+ // Fetch project data
2302
+ const projectRow = this.db
2303
+ .prepare('SELECT * FROM projects WHERE name = ?')
2304
+ .get(projectId);
2305
+ const projects = projectRow
2306
+ ? [{
2307
+ name: projectRow.name,
2308
+ createdAt: new Date(projectRow.created_at).toISOString(),
2309
+ metadata: this.deserialize(projectRow.metadata),
2310
+ }]
2311
+ : [{
2312
+ name: projectId,
2313
+ createdAt: new Date().toISOString(),
2314
+ metadata: {},
2315
+ }];
2316
+ const sessions = sessionsResult.sessions.map((s) => ({
2317
+ uuid: s.uuid,
2318
+ projectId: s.projectId,
2319
+ startedAt: s.startedAt.toISOString(),
2320
+ endedAt: s.endedAt ? s.endedAt.toISOString() : null,
2321
+ metadata: s.metadata,
2322
+ }));
2323
+ const observations = observationsResult.observations.map((o) => ({
2324
+ uuid: o.uuid,
2325
+ title: o.title,
2326
+ content: o.content,
2327
+ type: o.type,
2328
+ topicKey: o.topicKey,
2329
+ projectId: o.projectId,
2330
+ scope: o.scope,
2331
+ pinned: o.pinned,
2332
+ readOnly: o.readOnly,
2333
+ revisionCount: o.revisionCount,
2334
+ createdAt: o.createdAt.toISOString(),
2335
+ deletedAt: o.deletedAt ? o.deletedAt.toISOString() : null,
2336
+ metadata: o.metadata,
2337
+ }));
2338
+ const prompts = promptsResult.prompts.map((p) => ({
2339
+ uuid: p.uuid,
2340
+ content: p.content,
2341
+ projectId: p.projectId,
2342
+ createdAt: p.createdAt.toISOString(),
2343
+ metadata: p.metadata,
2344
+ }));
2345
+ const journal = journalResult.entries.map((j) => ({
2346
+ uuid: j.uuid,
2347
+ title: j.title,
2348
+ body: j.body,
2349
+ tags: j.tags,
2350
+ projectId: j.projectId,
2351
+ model: j.model,
2352
+ provider: j.provider,
2353
+ agent: j.agent,
2354
+ supersededByUuid: null, // Resolved below
2355
+ invalidatedAt: j.invalidatedAt ? j.invalidatedAt.toISOString() : null,
2356
+ metadata: j.metadata,
2357
+ createdAt: j.createdAt.toISOString(),
2358
+ }));
2359
+ // Resolve journal supersedes references from ID to UUID
2360
+ const journalIdToUuid = new Map();
2361
+ for (const entry of journalResult.entries) {
2362
+ journalIdToUuid.set(entry.id, entry.uuid);
2363
+ }
2364
+ for (let i = 0; i < journal.length; i++) {
2365
+ const supersededById = journalResult.entries[i].supersededBy;
2366
+ if (supersededById !== null) {
2367
+ journal[i].supersededByUuid = journalIdToUuid.get(supersededById) ?? null;
2368
+ }
2369
+ }
2370
+ return {
2371
+ version: '2.0',
2372
+ exportedAt: new Date().toISOString(),
2373
+ source: {
2374
+ project: projectId,
2375
+ allProjects: false,
2376
+ },
2377
+ stats: {
2378
+ totalProjects: projects.length,
2379
+ totalSessions: sessions.length,
2380
+ totalObservations: observations.length,
2381
+ totalPrompts: prompts.length,
2382
+ totalJournalEntries: journal.length,
2383
+ },
2384
+ projects,
2385
+ sessions,
2386
+ observations,
2387
+ prompts,
2388
+ journal,
2389
+ };
2390
+ }
2391
+ async importProject(data, options) {
2392
+ this.checkHealth();
2393
+ const { conflictStrategy = 'skip', dryRun = false } = options || {};
2394
+ const result = {
2395
+ imported: { projects: 0, sessions: 0, observations: 0, prompts: 0, journalEntries: 0 },
2396
+ skipped: { observations: 0, sessions: 0, journalEntries: 0 },
2397
+ overwritten: { observations: 0 },
2398
+ failed: 0,
2399
+ errors: [],
2400
+ };
2401
+ if (!data.version) {
2402
+ throw new Error('Invalid import data: missing version field');
2403
+ }
2404
+ // Determine target project
2405
+ const targetProject = options?.projectId || data.source?.project || 'import';
2406
+ // Validate observations types
2407
+ const validTypes = ['decision', 'bug', 'discovery', 'note', 'summary', 'learning', 'pattern', 'architecture', 'config', 'preference'];
2408
+ // UUID → new ID mappings for referential integrity
2409
+ const sessionUuidToId = new Map();
2410
+ // Create import session
2411
+ const importSession = await this.createSession({
2412
+ projectId: targetProject,
2413
+ endedAt: null,
2414
+ metadata: { type: 'full-import', source: 'web-ui', timestamp: new Date().toISOString() },
2415
+ });
2416
+ const doImport = this.db.transaction(() => {
2417
+ // 1. Import projects (register if not exists)
2418
+ if (data.projects && data.projects.length > 0) {
2419
+ for (const proj of data.projects) {
2420
+ if (!dryRun) {
2421
+ this.db.prepare('INSERT OR IGNORE INTO projects (name, created_at, metadata) VALUES (?, ?, ?)').run(proj.name, Date.now(), this.serialize(proj.metadata || {}));
2422
+ }
2423
+ result.imported.projects++;
2424
+ }
2425
+ }
2426
+ // 2. Import sessions (create new, map UUID → new ID)
2427
+ if (data.sessions && data.sessions.length > 0) {
2428
+ for (const sess of data.sessions) {
2429
+ // Check if session with this UUID already exists
2430
+ const existing = this.db
2431
+ .prepare('SELECT id FROM sessions WHERE uuid = ?')
2432
+ .get(sess.uuid);
2433
+ if (existing) {
2434
+ sessionUuidToId.set(sess.uuid, existing.id);
2435
+ result.skipped.sessions++;
2436
+ continue;
2437
+ }
2438
+ if (!dryRun) {
2439
+ const insertResult = this.db.prepare('INSERT INTO sessions (uuid, project_id, started_at, ended_at, metadata) VALUES (?, ?, ?, ?, ?)').run(sess.uuid, targetProject, new Date(sess.startedAt).getTime(), sess.endedAt ? new Date(sess.endedAt).getTime() : null, this.serialize(sess.metadata || {}));
2440
+ const newId = typeof insertResult.lastInsertRowid === 'bigint'
2441
+ ? Number(insertResult.lastInsertRowid)
2442
+ : insertResult.lastInsertRowid;
2443
+ sessionUuidToId.set(sess.uuid, newId);
2444
+ }
2445
+ else {
2446
+ // For dry run, assign placeholder IDs
2447
+ sessionUuidToId.set(sess.uuid, -(result.imported.sessions + 1));
2448
+ }
2449
+ result.imported.sessions++;
2450
+ }
2451
+ }
2452
+ // 3. Import observations
2453
+ if (data.observations && data.observations.length > 0) {
2454
+ for (const obs of data.observations) {
2455
+ if (!obs.title || !obs.content || !obs.type) {
2456
+ result.errors.push(`Invalid observation: missing required fields`);
2457
+ result.failed++;
2458
+ continue;
2459
+ }
2460
+ if (!validTypes.includes(obs.type)) {
2461
+ result.errors.push(`Invalid type: "${obs.type}"`);
2462
+ result.failed++;
2463
+ continue;
2464
+ }
2465
+ // Check for duplicate by UUID
2466
+ const existing = obs.uuid
2467
+ ? this.db.prepare('SELECT id FROM observations WHERE uuid = ?').get(obs.uuid)
2468
+ : undefined;
2469
+ if (existing) {
2470
+ if (conflictStrategy === 'skip') {
2471
+ result.skipped.observations++;
2472
+ continue;
2473
+ }
2474
+ if (conflictStrategy === 'overwrite' && !dryRun) {
2475
+ this.db.prepare(`UPDATE observations SET title = ?, content = ?, type = ?, topic_key = ?, metadata = ?, scope = ?, pinned = ?, read_only = ?, revision_count = revision_count + 1 WHERE id = ?`).run(obs.title, obs.content, obs.type, obs.topicKey ?? null, this.serialize(obs.metadata || {}), obs.scope || 'project', obs.pinned ? 1 : 0, obs.readOnly ? 1 : 0, existing.id);
2476
+ // FTS5 sync
2477
+ this.db.prepare('DELETE FROM observations_fts WHERE rowid = ?').run(existing.id);
2478
+ const row = this.db
2479
+ .prepare('SELECT title, content, topic_key, project_id FROM observations WHERE id = ?')
2480
+ .get(existing.id);
2481
+ if (row) {
2482
+ this.db.prepare('INSERT INTO observations_fts(rowid, title, content, topic_key, project_id) VALUES (?, ?, ?, ?, ?)').run(existing.id, row.title, row.content, row.topic_key, row.project_id);
2483
+ }
2484
+ }
2485
+ result.overwritten.observations++;
2486
+ continue;
2487
+ }
2488
+ if (!dryRun) {
2489
+ const uuid = obs.uuid || crypto.randomUUID();
2490
+ const createdAt = obs.createdAt ? new Date(obs.createdAt).getTime() : Date.now();
2491
+ const sessionId = importSession.id;
2492
+ const metadata = this.serialize(obs.metadata || {});
2493
+ const insertResult = this.db.prepare(`INSERT INTO observations (uuid, session_id, title, content, type, topic_key, project_id, created_at, deleted_at, metadata, scope, pinned, read_only, revision_count)
2494
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(uuid, sessionId, obs.title, obs.content, obs.type, obs.topicKey ?? null, targetProject, createdAt, obs.deletedAt ? new Date(obs.deletedAt).getTime() : null, metadata, obs.scope || 'project', obs.pinned ? 1 : 0, obs.readOnly ? 1 : 0, obs.revisionCount ?? 0);
2495
+ const insertId = typeof insertResult.lastInsertRowid === 'bigint'
2496
+ ? Number(insertResult.lastInsertRowid)
2497
+ : insertResult.lastInsertRowid;
2498
+ // FTS5 sync
2499
+ this.db.prepare('INSERT INTO observations_fts(rowid, title, content, topic_key, project_id) VALUES (?, ?, ?, ?, ?)').run(insertId, obs.title, obs.content, obs.topicKey ?? '', targetProject);
2500
+ }
2501
+ result.imported.observations++;
2502
+ }
2503
+ }
2504
+ // 4. Import prompts
2505
+ if (data.prompts && data.prompts.length > 0) {
2506
+ for (const prompt of data.prompts) {
2507
+ // Skip if UUID already exists
2508
+ if (prompt.uuid) {
2509
+ const existingPrompt = this.db
2510
+ .prepare('SELECT id FROM prompts WHERE uuid = ?')
2511
+ .get(prompt.uuid);
2512
+ if (existingPrompt)
2513
+ continue;
2514
+ }
2515
+ if (!dryRun) {
2516
+ const uuid = prompt.uuid || crypto.randomUUID();
2517
+ const createdAt = prompt.createdAt ? new Date(prompt.createdAt).getTime() : Date.now();
2518
+ this.db.prepare('INSERT INTO prompts (uuid, session_id, content, project_id, created_at, metadata) VALUES (?, ?, ?, ?, ?, ?)').run(uuid, importSession.id, prompt.content, targetProject, createdAt, this.serialize(prompt.metadata || {}));
2519
+ }
2520
+ result.imported.prompts++;
2521
+ }
2522
+ }
2523
+ // 5. Import journal entries (with tag support and supersedes UUID mapping)
2524
+ if (data.journal && data.journal.length > 0) {
2525
+ // Two-pass: first insert all entries, then resolve supersedes
2526
+ const journalUuidToId = new Map();
2527
+ for (const entry of data.journal) {
2528
+ // Check if UUID already exists
2529
+ const existingEntry = entry.uuid
2530
+ ? this.db.prepare('SELECT id FROM journal WHERE uuid = ?').get(entry.uuid)
2531
+ : undefined;
2532
+ if (existingEntry) {
2533
+ journalUuidToId.set(entry.uuid, existingEntry.id);
2534
+ result.skipped.journalEntries++;
2535
+ continue;
2536
+ }
2537
+ if (!dryRun) {
2538
+ const uuid = entry.uuid || crypto.randomUUID();
2539
+ const createdAt = entry.createdAt ? new Date(entry.createdAt).getTime() : Date.now();
2540
+ // Find session by UUID if available
2541
+ let sessionId = null;
2542
+ // We don't have session UUID reference in journal import, use import session
2543
+ const insertResult = this.db.prepare(`INSERT INTO journal (uuid, project_id, session_id, title, body, model, provider, agent, superseded_by, invalidated_at, metadata, created_at)
2544
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?, ?)`).run(uuid, targetProject, sessionId, entry.title, entry.body, entry.model ?? null, entry.provider ?? null, entry.agent ?? null, entry.invalidatedAt ? new Date(entry.invalidatedAt).getTime() : null, this.serialize(entry.metadata || {}), createdAt);
2545
+ const newId = typeof insertResult.lastInsertRowid === 'bigint'
2546
+ ? Number(insertResult.lastInsertRowid)
2547
+ : insertResult.lastInsertRowid;
2548
+ journalUuidToId.set(uuid, newId);
2549
+ // Insert tags
2550
+ if (entry.tags && entry.tags.length > 0) {
2551
+ for (const tag of entry.tags) {
2552
+ this.db.prepare('INSERT INTO journal_tags (journal_id, tag) VALUES (?, ?)').run(newId, tag);
2553
+ }
2554
+ }
2555
+ }
2556
+ result.imported.journalEntries++;
2557
+ }
2558
+ // Second pass: resolve supersedes references
2559
+ if (!dryRun) {
2560
+ for (const entry of data.journal) {
2561
+ if (entry.supersededByUuid && entry.uuid) {
2562
+ const entryId = journalUuidToId.get(entry.uuid);
2563
+ const supersededById = journalUuidToId.get(entry.supersededByUuid);
2564
+ if (entryId && supersededById) {
2565
+ this.db.prepare('UPDATE journal SET superseded_by = ? WHERE id = ?').run(supersededById, entryId);
2566
+ }
2567
+ }
2568
+ }
2569
+ }
2570
+ }
2571
+ });
2572
+ try {
2573
+ doImport();
2574
+ }
2575
+ catch (error) {
2576
+ if (error instanceof Error) {
2577
+ result.errors.push(`Transaction failed: ${error.message}`);
2578
+ }
2579
+ result.failed++;
2580
+ }
2581
+ return result;
2582
+ }
368
2583
  close() {
369
2584
  this.db.close();
370
2585
  }