@slorenzot/memento-core 0.7.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.es.md +446 -0
- package/README.md +117 -115
- package/dist/ConfigManager.d.ts +90 -0
- package/dist/ConfigManager.d.ts.map +1 -1
- package/dist/ConfigManager.js +263 -7
- package/dist/ConfigManager.js.map +1 -1
- package/dist/EmbeddingService.d.ts +54 -0
- package/dist/EmbeddingService.d.ts.map +1 -0
- package/dist/EmbeddingService.js +135 -0
- package/dist/EmbeddingService.js.map +1 -0
- package/dist/MemoryEngine.d.ts +238 -8
- package/dist/MemoryEngine.d.ts.map +1 -1
- package/dist/MemoryEngine.js +2267 -52
- package/dist/MemoryEngine.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/types.d.ts +290 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +9 -4
- package/dist/db/schema.d.ts +0 -353
- package/dist/db/schema.d.ts.map +0 -1
- package/dist/db/schema.js +0 -40
- package/dist/db/schema.js.map +0 -1
- package/drizzle.config.ts +0 -10
- package/src/ConfigManager.test.ts +0 -248
- package/src/ConfigManager.ts +0 -131
- package/src/MemoryEngine.test.ts +0 -456
- package/src/MemoryEngine.ts +0 -450
- package/src/db/schema.js +0 -39
- package/src/db/schema.ts +0 -40
- package/src/index.ts +0 -3
- package/src/types.ts +0 -52
- package/tsconfig.json +0 -18
- package/tsconfig.json.bak +0 -30
package/dist/MemoryEngine.js
CHANGED
|
@@ -35,41 +35,36 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
36
|
exports.MemoryEngine = void 0;
|
|
37
37
|
const crypto = __importStar(require("crypto"));
|
|
38
|
+
const EmbeddingService_js_1 = require("./EmbeddingService.js");
|
|
38
39
|
const bun_sqlite_1 = require("bun:sqlite");
|
|
39
40
|
const fs_1 = require("fs");
|
|
40
41
|
const path_1 = require("path");
|
|
41
42
|
class MemoryEngine {
|
|
43
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Bun Database type is incompatible with mock pattern
|
|
42
44
|
db;
|
|
43
45
|
dbPath;
|
|
44
46
|
initError = null;
|
|
47
|
+
embeddingService;
|
|
45
48
|
constructor(dbPath = './data/memento.db') {
|
|
46
|
-
this.dbPath = dbPath;
|
|
49
|
+
this.dbPath = (0, path_1.resolve)(dbPath);
|
|
50
|
+
this.embeddingService = new EmbeddingService_js_1.EmbeddingService();
|
|
47
51
|
try {
|
|
48
|
-
const dbDir = (0, path_1.dirname)(dbPath);
|
|
49
|
-
// Create directory structure if it doesn't exist
|
|
52
|
+
const dbDir = (0, path_1.dirname)(this.dbPath);
|
|
50
53
|
(0, fs_1.mkdirSync)(dbDir, { recursive: true });
|
|
51
|
-
|
|
52
|
-
this.db = new bun_sqlite_1.Database(dbPath, { create: true });
|
|
54
|
+
this.db = new bun_sqlite_1.Database(this.dbPath, { create: true });
|
|
53
55
|
this.initializeDatabase();
|
|
54
|
-
console.error(`✓ Database initialized successfully at: ${dbPath}`);
|
|
55
56
|
}
|
|
56
57
|
catch (error) {
|
|
57
|
-
this.initError = error;
|
|
58
|
-
console.error(`✗ Failed to initialize database at ${dbPath}:`, error.message);
|
|
59
|
-
console.error(' The server will start but database operations will fail until this is resolved.');
|
|
60
|
-
// Create a mock database object to prevent null reference errors
|
|
58
|
+
this.initError = error instanceof Error ? error : new Error(String(error));
|
|
61
59
|
this.db = this.createMockDatabase();
|
|
62
60
|
}
|
|
63
61
|
}
|
|
62
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Mock fallback for failed DB init
|
|
64
63
|
createMockDatabase() {
|
|
65
64
|
const throwError = () => {
|
|
66
65
|
throw new Error(`Database not initialized: ${this.initError?.message || 'Unknown error'}`);
|
|
67
66
|
};
|
|
68
|
-
return {
|
|
69
|
-
prepare: throwError,
|
|
70
|
-
exec: throwError,
|
|
71
|
-
close: () => { },
|
|
72
|
-
};
|
|
67
|
+
return { prepare: throwError, exec: throwError, close: () => { }, transaction: throwError };
|
|
73
68
|
}
|
|
74
69
|
isHealthy() {
|
|
75
70
|
return this.initError === null;
|
|
@@ -83,6 +78,7 @@ class MemoryEngine {
|
|
|
83
78
|
initializeDatabase() {
|
|
84
79
|
this.db.exec('PRAGMA journal_mode = WAL');
|
|
85
80
|
this.db.exec('PRAGMA foreign_keys = ON');
|
|
81
|
+
this.db.exec('PRAGMA busy_timeout = 5000');
|
|
86
82
|
this.db.exec(`
|
|
87
83
|
CREATE TABLE IF NOT EXISTS sessions (
|
|
88
84
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
@@ -103,6 +99,7 @@ class MemoryEngine {
|
|
|
103
99
|
topic_key TEXT,
|
|
104
100
|
project_id TEXT NOT NULL,
|
|
105
101
|
created_at INTEGER NOT NULL,
|
|
102
|
+
deleted_at INTEGER DEFAULT NULL,
|
|
106
103
|
metadata TEXT,
|
|
107
104
|
FOREIGN KEY (session_id) REFERENCES sessions(id)
|
|
108
105
|
);
|
|
@@ -124,27 +121,186 @@ class MemoryEngine {
|
|
|
124
121
|
created_at INTEGER NOT NULL,
|
|
125
122
|
metadata TEXT
|
|
126
123
|
);
|
|
127
|
-
|
|
124
|
+
`);
|
|
125
|
+
// Migrate: add working_dir and aliases columns to projects table (Issue #177)
|
|
126
|
+
try {
|
|
127
|
+
this.db.exec('SELECT working_dir FROM projects LIMIT 0');
|
|
128
|
+
}
|
|
129
|
+
catch {
|
|
130
|
+
try {
|
|
131
|
+
this.db.exec('ALTER TABLE projects ADD COLUMN working_dir TEXT');
|
|
132
|
+
console.error('✓ Migration: added working_dir column to projects');
|
|
133
|
+
}
|
|
134
|
+
catch {
|
|
135
|
+
// Column may already exist
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
try {
|
|
139
|
+
this.db.exec('SELECT aliases FROM projects LIMIT 0');
|
|
140
|
+
}
|
|
141
|
+
catch {
|
|
142
|
+
try {
|
|
143
|
+
this.db.exec("ALTER TABLE projects ADD COLUMN aliases TEXT DEFAULT '[]'");
|
|
144
|
+
console.error('✓ Migration: added aliases column to projects');
|
|
145
|
+
}
|
|
146
|
+
catch {
|
|
147
|
+
// Column may already exist
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
// Migrate: add deleted_at column if missing
|
|
151
|
+
try {
|
|
152
|
+
this.db.exec('SELECT deleted_at FROM observations LIMIT 0');
|
|
153
|
+
}
|
|
154
|
+
catch {
|
|
155
|
+
try {
|
|
156
|
+
this.db.exec('ALTER TABLE observations ADD COLUMN deleted_at INTEGER DEFAULT NULL');
|
|
157
|
+
console.error('✓ Migration: added deleted_at column to observations');
|
|
158
|
+
}
|
|
159
|
+
catch {
|
|
160
|
+
// Column may already exist in a concurrent scenario
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
// Create index for deleted_at if not exists
|
|
164
|
+
this.db.exec('CREATE INDEX IF NOT EXISTS idx_observations_deleted ON observations(deleted_at)');
|
|
165
|
+
this.db.exec('CREATE INDEX IF NOT EXISTS idx_observations_created ON observations(created_at)');
|
|
166
|
+
this.db.exec('CREATE INDEX IF NOT EXISTS idx_observations_type ON observations(type)');
|
|
167
|
+
this.db.exec('CREATE INDEX IF NOT EXISTS idx_observations_project ON observations(project_id)');
|
|
168
|
+
this.db.exec('CREATE INDEX IF NOT EXISTS idx_observations_topic ON observations(topic_key)');
|
|
169
|
+
// Migrate: add scope column if missing
|
|
170
|
+
try {
|
|
171
|
+
this.db.exec('SELECT scope FROM observations LIMIT 0');
|
|
172
|
+
}
|
|
173
|
+
catch {
|
|
174
|
+
try {
|
|
175
|
+
this.db.exec("ALTER TABLE observations ADD COLUMN scope TEXT NOT NULL DEFAULT 'project'");
|
|
176
|
+
console.error('✓ Migration: added scope column to observations');
|
|
177
|
+
}
|
|
178
|
+
catch {
|
|
179
|
+
// Column may already exist in a concurrent scenario
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
this.db.exec('CREATE INDEX IF NOT EXISTS idx_observations_scope ON observations(scope)');
|
|
183
|
+
// Migrate: add revision_count column if missing
|
|
184
|
+
try {
|
|
185
|
+
this.db.exec('SELECT revision_count FROM observations LIMIT 0');
|
|
186
|
+
}
|
|
187
|
+
catch {
|
|
188
|
+
try {
|
|
189
|
+
this.db.exec('ALTER TABLE observations ADD COLUMN revision_count INTEGER NOT NULL DEFAULT 0');
|
|
190
|
+
console.error('✓ Migration: added revision_count column to observations');
|
|
191
|
+
}
|
|
192
|
+
catch {
|
|
193
|
+
// Column may already exist in a concurrent scenario
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
// Migrate: add pinned column if missing (Issue #50)
|
|
197
|
+
try {
|
|
198
|
+
this.db.exec('SELECT pinned FROM observations LIMIT 0');
|
|
199
|
+
}
|
|
200
|
+
catch {
|
|
201
|
+
try {
|
|
202
|
+
this.db.exec('ALTER TABLE observations ADD COLUMN pinned INTEGER NOT NULL DEFAULT 0');
|
|
203
|
+
console.error('✓ Migration: added pinned column to observations');
|
|
204
|
+
}
|
|
205
|
+
catch {
|
|
206
|
+
// Column may already exist in a concurrent scenario
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
this.db.exec('CREATE INDEX IF NOT EXISTS idx_observations_pinned ON observations(pinned) WHERE pinned = 1');
|
|
210
|
+
// Migrate: add read_only column if missing (Issue #54)
|
|
211
|
+
try {
|
|
212
|
+
this.db.exec('SELECT read_only FROM observations LIMIT 0');
|
|
213
|
+
}
|
|
214
|
+
catch {
|
|
215
|
+
try {
|
|
216
|
+
this.db.exec('ALTER TABLE observations ADD COLUMN read_only INTEGER NOT NULL DEFAULT 0');
|
|
217
|
+
console.error('✓ Migration: added read_only column to observations');
|
|
218
|
+
}
|
|
219
|
+
catch {
|
|
220
|
+
// Column may already exist in a concurrent scenario
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
// Migrate: clean up empty string topic_key → NULL (Issue #69)
|
|
224
|
+
try {
|
|
225
|
+
const result = this.db.prepare("UPDATE observations SET topic_key = NULL WHERE topic_key = ''").run();
|
|
226
|
+
if (result.changes > 0) {
|
|
227
|
+
console.error(`✓ Migration: cleaned up ${result.changes} empty topic_key value(s)`);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
catch {
|
|
231
|
+
// Table may not have topic_key column yet
|
|
232
|
+
}
|
|
233
|
+
// FTS5 — standalone mode (no content= parameter).
|
|
234
|
+
// FTS5 owns its own data copy, supports full CRUD (INSERT, DELETE, UPDATE),
|
|
235
|
+
// and avoids SQLITE_CORRUPT_VTAB that occurred with content='observations' mode
|
|
236
|
+
// under bun:sqlite + WAL + concurrent access. See: GitHub Issue #68.
|
|
237
|
+
this.db.exec(`
|
|
128
238
|
CREATE VIRTUAL TABLE IF NOT EXISTS observations_fts USING fts5(
|
|
129
239
|
title, content, topic_key, project_id,
|
|
130
|
-
|
|
240
|
+
tokenize='porter unicode61'
|
|
241
|
+
);
|
|
242
|
+
`);
|
|
243
|
+
// FTS5 sync is managed at application-level in each CRUD method.
|
|
244
|
+
// Previous trigger-based approach caused SQLITE_CORRUPT_VTAB under
|
|
245
|
+
// concurrent access (DELETE+INSERT on FTS5 virtual table within a trigger).
|
|
246
|
+
// See: GitHub Issue #68
|
|
247
|
+
// ─── Journal tables (append-only evidence) ────────────────
|
|
248
|
+
this.db.exec(`
|
|
249
|
+
CREATE TABLE IF NOT EXISTS journal (
|
|
250
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
251
|
+
uuid TEXT UNIQUE NOT NULL,
|
|
252
|
+
project_id TEXT NOT NULL,
|
|
253
|
+
session_id INTEGER REFERENCES sessions(id),
|
|
254
|
+
title TEXT NOT NULL,
|
|
255
|
+
body TEXT NOT NULL,
|
|
256
|
+
model TEXT,
|
|
257
|
+
provider TEXT,
|
|
258
|
+
agent TEXT,
|
|
259
|
+
superseded_by INTEGER REFERENCES journal(id),
|
|
260
|
+
invalidated_at INTEGER,
|
|
261
|
+
metadata TEXT,
|
|
262
|
+
created_at INTEGER NOT NULL
|
|
131
263
|
);
|
|
132
264
|
|
|
133
|
-
CREATE
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
265
|
+
CREATE TABLE IF NOT EXISTS journal_tags (
|
|
266
|
+
journal_id INTEGER NOT NULL REFERENCES journal(id) ON DELETE CASCADE,
|
|
267
|
+
tag TEXT NOT NULL,
|
|
268
|
+
PRIMARY KEY (journal_id, tag)
|
|
269
|
+
);
|
|
270
|
+
`);
|
|
271
|
+
// Journal indexes
|
|
272
|
+
this.db.exec('CREATE INDEX IF NOT EXISTS idx_journal_project ON journal(project_id)');
|
|
273
|
+
this.db.exec('CREATE INDEX IF NOT EXISTS idx_journal_session ON journal(session_id)');
|
|
274
|
+
this.db.exec('CREATE INDEX IF NOT EXISTS idx_journal_created ON journal(created_at)');
|
|
275
|
+
this.db.exec('CREATE INDEX IF NOT EXISTS idx_journal_superseded ON journal(superseded_by)');
|
|
276
|
+
this.db.exec('CREATE INDEX IF NOT EXISTS idx_journal_tags_tag ON journal_tags(tag)');
|
|
277
|
+
// Journal FTS5
|
|
278
|
+
this.db.exec(`
|
|
279
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS journal_fts USING fts5(
|
|
280
|
+
title, body, project_id,
|
|
281
|
+
content='journal',
|
|
282
|
+
tokenize='porter unicode61'
|
|
283
|
+
);
|
|
284
|
+
`);
|
|
285
|
+
// Journal FTS trigger — insert only (append-only: no update/delete triggers)
|
|
286
|
+
this.db.exec('DROP TRIGGER IF EXISTS journal_ai');
|
|
287
|
+
this.db.exec(`
|
|
288
|
+
CREATE TRIGGER journal_ai AFTER INSERT ON journal BEGIN
|
|
289
|
+
INSERT INTO journal_fts(rowid, title, body, project_id)
|
|
290
|
+
VALUES (new.id, new.title, new.body, new.project_id);
|
|
146
291
|
END;
|
|
147
292
|
`);
|
|
293
|
+
// ─── Embeddings table (semantic search) ──────────────────────
|
|
294
|
+
this.db.exec(`
|
|
295
|
+
CREATE TABLE IF NOT EXISTS embeddings (
|
|
296
|
+
observation_id INTEGER PRIMARY KEY REFERENCES observations(id) ON DELETE CASCADE,
|
|
297
|
+
embedding BLOB NOT NULL,
|
|
298
|
+
model TEXT NOT NULL DEFAULT 'all-MiniLM-L6-v2',
|
|
299
|
+
dimensions INTEGER NOT NULL DEFAULT 384,
|
|
300
|
+
generated_at INTEGER NOT NULL
|
|
301
|
+
);
|
|
302
|
+
`);
|
|
303
|
+
this.db.exec('CREATE INDEX IF NOT EXISTS idx_embeddings_obs ON embeddings(observation_id)');
|
|
148
304
|
}
|
|
149
305
|
serialize(value) {
|
|
150
306
|
return JSON.stringify(value);
|
|
@@ -164,26 +320,70 @@ class MemoryEngine {
|
|
|
164
320
|
throw this.initError || new Error('Database not initialized');
|
|
165
321
|
}
|
|
166
322
|
}
|
|
323
|
+
/**
|
|
324
|
+
* Retry wrapper with exponential backoff for SQLITE_BUSY errors.
|
|
325
|
+
* Dual defense: PRAGMA busy_timeout (5s) catches most contention,
|
|
326
|
+
* this catches the rest with retries.
|
|
327
|
+
*/
|
|
328
|
+
async withRetry(operation, maxRetries = 3, baseDelay = 100) {
|
|
329
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
330
|
+
try {
|
|
331
|
+
return operation();
|
|
332
|
+
}
|
|
333
|
+
catch (error) {
|
|
334
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
335
|
+
const isBusy = message.includes('SQLITE_BUSY') || message.includes('database is locked');
|
|
336
|
+
if (!isBusy || attempt === maxRetries) {
|
|
337
|
+
throw error;
|
|
338
|
+
}
|
|
339
|
+
const delay = baseDelay * Math.pow(2, attempt);
|
|
340
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
throw new Error('Max retries exceeded');
|
|
344
|
+
}
|
|
345
|
+
// ─── Observations CRUD ─────────────────────────────────────
|
|
167
346
|
async createObservation(data) {
|
|
168
347
|
this.checkHealth();
|
|
169
348
|
const uuid = crypto.randomUUID();
|
|
170
349
|
const createdAt = new Date();
|
|
171
350
|
const metadata = this.serialize(data.metadata);
|
|
172
|
-
const
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
const id =
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
351
|
+
const scope = data.scope || 'project';
|
|
352
|
+
const pinned = data.pinned ? 1 : 0;
|
|
353
|
+
const readOnly = data.readOnly ? 1 : 0;
|
|
354
|
+
const id = await this.withRetry(() => {
|
|
355
|
+
const result = this.db
|
|
356
|
+
.prepare(`INSERT INTO observations (uuid, session_id, title, content, type, topic_key, project_id, created_at, deleted_at, metadata, scope, pinned, read_only) VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?, ?, ?)`)
|
|
357
|
+
.run(uuid, data.sessionId, data.title, data.content, data.type, data.topicKey ?? null, data.projectId, createdAt.getTime(), metadata, scope, pinned, readOnly);
|
|
358
|
+
const insertId = typeof result.lastInsertRowid === 'bigint'
|
|
359
|
+
? Number(result.lastInsertRowid)
|
|
360
|
+
: result.lastInsertRowid;
|
|
361
|
+
// Application-level FTS5 sync (was trigger observations_ai)
|
|
362
|
+
this.db.prepare('INSERT INTO observations_fts(rowid, title, content, topic_key, project_id) VALUES (?, ?, ?, ?, ?)').run(insertId, data.title, data.content, data.topicKey ?? '', data.projectId);
|
|
363
|
+
return insertId;
|
|
364
|
+
});
|
|
365
|
+
const observation = await this.getObservationById(id, true);
|
|
179
366
|
if (!observation)
|
|
180
367
|
throw new Error('Failed to retrieve created observation');
|
|
368
|
+
// Generate embedding asynchronously (non-blocking, fire-and-forget)
|
|
369
|
+
this.generateEmbedding(id).catch(() => {
|
|
370
|
+
// Silently ignore — embedding generation failure doesn't affect save
|
|
371
|
+
});
|
|
181
372
|
return observation;
|
|
182
373
|
}
|
|
183
374
|
async updateObservation(id, updates) {
|
|
184
375
|
const current = await this.getObservationById(id);
|
|
185
376
|
if (!current)
|
|
186
377
|
throw new Error('Observation not found');
|
|
378
|
+
if (current.deletedAt)
|
|
379
|
+
throw new Error('Cannot update a soft-deleted observation');
|
|
380
|
+
// Read-only protection: only allow changing readOnly itself
|
|
381
|
+
if (current.readOnly) {
|
|
382
|
+
const nonReadOnlyFields = Object.keys(updates).filter(k => k !== 'readOnly');
|
|
383
|
+
if (nonReadOnlyFields.length > 0) {
|
|
384
|
+
throw new Error(`Observation #${id} is read-only. Only readOnly flag can be changed.`);
|
|
385
|
+
}
|
|
386
|
+
}
|
|
187
387
|
const fields = [];
|
|
188
388
|
const values = [];
|
|
189
389
|
if (updates.title !== undefined) {
|
|
@@ -200,35 +400,501 @@ class MemoryEngine {
|
|
|
200
400
|
}
|
|
201
401
|
if (updates.topicKey !== undefined) {
|
|
202
402
|
fields.push('topic_key = ?');
|
|
203
|
-
values.push(updates.topicKey
|
|
403
|
+
values.push(updates.topicKey ?? null);
|
|
204
404
|
}
|
|
205
405
|
if (updates.metadata !== undefined) {
|
|
206
406
|
fields.push('metadata = ?');
|
|
207
407
|
values.push(this.serialize(updates.metadata));
|
|
208
408
|
}
|
|
409
|
+
if (updates.pinned !== undefined) {
|
|
410
|
+
fields.push('pinned = ?');
|
|
411
|
+
values.push(updates.pinned ? 1 : 0);
|
|
412
|
+
}
|
|
413
|
+
if (updates.readOnly !== undefined) {
|
|
414
|
+
fields.push('read_only = ?');
|
|
415
|
+
values.push(updates.readOnly ? 1 : 0);
|
|
416
|
+
}
|
|
209
417
|
if (fields.length === 0)
|
|
210
418
|
return current;
|
|
419
|
+
fields.push('revision_count = revision_count + 1');
|
|
211
420
|
values.push(id);
|
|
212
|
-
|
|
213
|
-
const
|
|
421
|
+
// Check if any FTS5-indexed fields changed
|
|
422
|
+
const hasFtsUpdate = updates.title !== undefined ||
|
|
423
|
+
updates.content !== undefined ||
|
|
424
|
+
updates.topicKey !== undefined;
|
|
425
|
+
if (hasFtsUpdate) {
|
|
426
|
+
// Atomic transaction: UPDATE + FTS5 rebuild
|
|
427
|
+
// Replaces trigger observations_au that caused SQLITE_CORRUPT_VTAB (#68)
|
|
428
|
+
await this.withRetry(() => {
|
|
429
|
+
this.db.transaction(() => {
|
|
430
|
+
this.db.prepare(`UPDATE observations SET ${fields.join(', ')} WHERE id = ?`).run(...values);
|
|
431
|
+
// Re-read updated row for FTS5 sync
|
|
432
|
+
const row = this.db
|
|
433
|
+
.prepare('SELECT title, content, topic_key, project_id FROM observations WHERE id = ?')
|
|
434
|
+
.get(id);
|
|
435
|
+
if (row) {
|
|
436
|
+
this.db.prepare('DELETE FROM observations_fts WHERE rowid = ?').run(id);
|
|
437
|
+
this.db.prepare('INSERT INTO observations_fts(rowid, title, content, topic_key, project_id) VALUES (?, ?, ?, ?, ?)').run(id, row.title, row.content, row.topic_key, row.project_id);
|
|
438
|
+
}
|
|
439
|
+
})();
|
|
440
|
+
});
|
|
441
|
+
}
|
|
442
|
+
else {
|
|
443
|
+
// No FTS5 fields changed — just UPDATE
|
|
444
|
+
await this.withRetry(() => this.db.prepare(`UPDATE observations SET ${fields.join(', ')} WHERE id = ?`).run(...values));
|
|
445
|
+
}
|
|
446
|
+
const updated = await this.getObservationById(id, true);
|
|
214
447
|
if (!updated)
|
|
215
448
|
throw new Error('Failed to update observation');
|
|
449
|
+
// Regenerate embedding if content-affecting fields changed
|
|
450
|
+
if (hasFtsUpdate) {
|
|
451
|
+
this.generateEmbedding(id).catch(() => {
|
|
452
|
+
// Silently ignore — embedding regeneration failure doesn't affect update
|
|
453
|
+
});
|
|
454
|
+
}
|
|
216
455
|
return updated;
|
|
217
456
|
}
|
|
218
|
-
|
|
219
|
-
|
|
457
|
+
// ─── Soft Delete / Restore / Purge ─────────────────────────
|
|
458
|
+
async deleteObservation(id, reason) {
|
|
459
|
+
this.checkHealth();
|
|
460
|
+
const obs = await this.getObservationById(id, true);
|
|
461
|
+
if (!obs)
|
|
462
|
+
throw new Error('Observation not found');
|
|
463
|
+
if (obs.deletedAt)
|
|
464
|
+
throw new Error('Observation already deleted');
|
|
465
|
+
if (obs.readOnly)
|
|
466
|
+
throw new Error(`Observation #${id} is read-only. Cannot delete.`);
|
|
467
|
+
const now = Date.now();
|
|
468
|
+
// If there's a reason, store it in metadata
|
|
469
|
+
if (reason) {
|
|
470
|
+
const meta = { ...obs.metadata, deleteReason: reason };
|
|
471
|
+
await this.withRetry(() => {
|
|
472
|
+
this.db
|
|
473
|
+
.prepare('UPDATE observations SET deleted_at = ?, metadata = ? WHERE id = ?')
|
|
474
|
+
.run(now, this.serialize(meta), id);
|
|
475
|
+
// Application-level FTS5 sync (was trigger observations_soft_delete)
|
|
476
|
+
this.db.prepare('DELETE FROM observations_fts WHERE rowid = ?').run(id);
|
|
477
|
+
});
|
|
478
|
+
}
|
|
479
|
+
else {
|
|
480
|
+
await this.withRetry(() => {
|
|
481
|
+
this.db.prepare('UPDATE observations SET deleted_at = ? WHERE id = ?').run(now, id);
|
|
482
|
+
// Application-level FTS5 sync (was trigger observations_soft_delete)
|
|
483
|
+
this.db.prepare('DELETE FROM observations_fts WHERE rowid = ?').run(id);
|
|
484
|
+
});
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
async restoreObservation(id) {
|
|
488
|
+
this.checkHealth();
|
|
489
|
+
const obs = await this.getObservationById(id, true);
|
|
490
|
+
if (!obs)
|
|
491
|
+
throw new Error('Observation not found');
|
|
492
|
+
if (!obs.deletedAt)
|
|
493
|
+
throw new Error('Observation is not deleted');
|
|
494
|
+
await this.withRetry(() => {
|
|
495
|
+
this.db.prepare('UPDATE observations SET deleted_at = NULL WHERE id = ?').run(id);
|
|
496
|
+
// Application-level FTS5 sync (was trigger observations_undelete)
|
|
497
|
+
const row = this.db
|
|
498
|
+
.prepare('SELECT title, content, topic_key, project_id FROM observations WHERE id = ?')
|
|
499
|
+
.get(id);
|
|
500
|
+
if (row) {
|
|
501
|
+
this.db.prepare('INSERT INTO observations_fts(rowid, title, content, topic_key, project_id) VALUES (?, ?, ?, ?, ?)').run(id, row.title, row.content, row.topic_key, row.project_id);
|
|
502
|
+
}
|
|
503
|
+
});
|
|
504
|
+
const restored = await this.getObservationById(id, true);
|
|
505
|
+
if (!restored)
|
|
506
|
+
throw new Error('Failed to restore observation');
|
|
507
|
+
return restored;
|
|
508
|
+
}
|
|
509
|
+
async purgeObservations(params) {
|
|
510
|
+
this.checkHealth();
|
|
511
|
+
let sql = 'SELECT id FROM observations WHERE deleted_at IS NOT NULL';
|
|
512
|
+
const values = [];
|
|
513
|
+
if (params.observationIds && params.observationIds.length > 0) {
|
|
514
|
+
const placeholders = params.observationIds.map(() => '?').join(',');
|
|
515
|
+
sql += ` AND id IN (${placeholders})`;
|
|
516
|
+
values.push(...params.observationIds);
|
|
517
|
+
}
|
|
518
|
+
if (params.projectId) {
|
|
519
|
+
sql += ' AND project_id = ?';
|
|
520
|
+
values.push(params.projectId);
|
|
521
|
+
}
|
|
522
|
+
const rows = this.db.prepare(sql).all(...values);
|
|
523
|
+
const ids = rows.map((r) => r.id);
|
|
524
|
+
if (ids.length === 0)
|
|
525
|
+
return { purgedCount: 0, purgedIds: [] };
|
|
526
|
+
const placeholders = ids.map(() => '?').join(',');
|
|
527
|
+
// Application-level FTS5 sync (was trigger observations_ad)
|
|
528
|
+
this.db.prepare(`DELETE FROM observations_fts WHERE rowid IN (${placeholders})`).run(...ids);
|
|
529
|
+
this.db.prepare(`DELETE FROM observations WHERE id IN (${placeholders})`).run(...ids);
|
|
530
|
+
return { purgedCount: ids.length, purgedIds: ids };
|
|
531
|
+
}
|
|
532
|
+
async listDeleted(params) {
|
|
533
|
+
this.checkHealth();
|
|
534
|
+
const { projectId, limit = 20 } = params;
|
|
535
|
+
let countSql = 'SELECT COUNT(*) as count FROM observations WHERE deleted_at IS NOT NULL';
|
|
536
|
+
let sql = 'SELECT * FROM observations WHERE deleted_at IS NOT NULL';
|
|
537
|
+
const values = [];
|
|
538
|
+
if (projectId) {
|
|
539
|
+
countSql += ' AND project_id = ?';
|
|
540
|
+
sql += ' AND project_id = ?';
|
|
541
|
+
values.push(projectId);
|
|
542
|
+
}
|
|
543
|
+
const countResult = this.db.prepare(countSql).get(...values);
|
|
544
|
+
const total = countResult ? countResult.count : 0;
|
|
545
|
+
sql += ' ORDER BY deleted_at DESC, id DESC LIMIT ?';
|
|
546
|
+
const rows = this.db.prepare(sql).all(...values, limit);
|
|
547
|
+
const observations = rows.map((row) => this.mapObservation(row));
|
|
548
|
+
return { observations, total };
|
|
220
549
|
}
|
|
221
|
-
|
|
222
|
-
|
|
550
|
+
// ─── Merge ─────────────────────────────────────────────────
|
|
551
|
+
async findMergeCandidates(params) {
|
|
552
|
+
this.checkHealth();
|
|
553
|
+
const { projectId, strategy, similarityThreshold = 0.85 } = params;
|
|
554
|
+
const groups = [];
|
|
555
|
+
if (strategy === 'by_topic') {
|
|
556
|
+
const rows = this.db
|
|
557
|
+
.prepare(`SELECT topic_key, COUNT(*) as cnt FROM observations
|
|
558
|
+
WHERE project_id = ? AND deleted_at IS NULL AND topic_key IS NOT NULL AND topic_key != ''
|
|
559
|
+
GROUP BY topic_key HAVING cnt >= 2
|
|
560
|
+
ORDER BY cnt DESC`)
|
|
561
|
+
.all(projectId);
|
|
562
|
+
for (const row of rows) {
|
|
563
|
+
const obs = this.db
|
|
564
|
+
.prepare('SELECT * FROM observations WHERE project_id = ? AND topic_key = ? AND deleted_at IS NULL ORDER BY created_at ASC, id ASC')
|
|
565
|
+
.all(projectId, row.topic_key);
|
|
566
|
+
groups.push({
|
|
567
|
+
reason: `topic_key: ${row.topic_key}`,
|
|
568
|
+
observations: obs.map((o) => this.mapObservation(o)),
|
|
569
|
+
estimatedReduction: obs.length - 1,
|
|
570
|
+
});
|
|
571
|
+
}
|
|
572
|
+
}
|
|
573
|
+
else {
|
|
574
|
+
// by_similarity — compare recent observations pairwise
|
|
575
|
+
const allObs = this.db
|
|
576
|
+
.prepare('SELECT * FROM observations WHERE project_id = ? AND deleted_at IS NULL ORDER BY created_at DESC, id DESC LIMIT 200')
|
|
577
|
+
.all(projectId);
|
|
578
|
+
const mapped = allObs.map((o) => this.mapObservation(o));
|
|
579
|
+
const visited = new Set();
|
|
580
|
+
for (let i = 0; i < mapped.length; i++) {
|
|
581
|
+
if (visited.has(mapped[i].id))
|
|
582
|
+
continue;
|
|
583
|
+
const group = [mapped[i]];
|
|
584
|
+
for (let j = i + 1; j < mapped.length; j++) {
|
|
585
|
+
if (visited.has(mapped[j].id))
|
|
586
|
+
continue;
|
|
587
|
+
const sim = this.jaccardSimilarity(mapped[i].content, mapped[j].content);
|
|
588
|
+
if (sim >= similarityThreshold) {
|
|
589
|
+
group.push(mapped[j]);
|
|
590
|
+
visited.add(mapped[j].id);
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
if (group.length >= 2) {
|
|
594
|
+
visited.add(mapped[i].id);
|
|
595
|
+
groups.push({
|
|
596
|
+
reason: `similarity >= ${similarityThreshold}`,
|
|
597
|
+
observations: group,
|
|
598
|
+
estimatedReduction: group.length - 1,
|
|
599
|
+
});
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
const totalCandidates = groups.reduce((acc, g) => acc + g.observations.length, 0);
|
|
604
|
+
const estimatedReduction = groups.reduce((acc, g) => acc + g.estimatedReduction, 0);
|
|
605
|
+
return { groups, totalCandidates, estimatedReduction };
|
|
223
606
|
}
|
|
224
|
-
async
|
|
225
|
-
|
|
607
|
+
async mergeObservations(params) {
|
|
608
|
+
this.checkHealth();
|
|
609
|
+
const { projectId, topicKey, observationIds, strategy, dryRun = false } = params;
|
|
610
|
+
let groups = [];
|
|
611
|
+
if (strategy === 'by_ids' && observationIds && observationIds.length >= 2) {
|
|
612
|
+
const obs = [];
|
|
613
|
+
for (const id of observationIds) {
|
|
614
|
+
const o = await this.getObservationById(id);
|
|
615
|
+
if (!o)
|
|
616
|
+
throw new Error(`Observation ${id} not found`);
|
|
617
|
+
if (o.deletedAt)
|
|
618
|
+
throw new Error(`Observation ${id} is soft-deleted`);
|
|
619
|
+
if (o.projectId !== projectId) {
|
|
620
|
+
throw new Error(`Observation ${id} belongs to project '${o.projectId}', not '${projectId}'`);
|
|
621
|
+
}
|
|
622
|
+
obs.push(o);
|
|
623
|
+
}
|
|
624
|
+
groups = [{ observations: obs }];
|
|
625
|
+
}
|
|
626
|
+
else if (strategy === 'by_topic') {
|
|
627
|
+
const candidates = await this.findMergeCandidates({
|
|
628
|
+
projectId,
|
|
629
|
+
strategy: 'by_topic',
|
|
630
|
+
});
|
|
631
|
+
if (topicKey) {
|
|
632
|
+
groups = candidates.groups.filter((g) => g.reason === `topic_key: ${topicKey}`);
|
|
633
|
+
}
|
|
634
|
+
else {
|
|
635
|
+
groups = candidates.groups;
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
else {
|
|
639
|
+
const candidates = await this.findMergeCandidates({
|
|
640
|
+
projectId,
|
|
641
|
+
strategy: 'by_similarity',
|
|
642
|
+
});
|
|
643
|
+
groups = candidates.groups;
|
|
644
|
+
}
|
|
645
|
+
if (groups.length === 0)
|
|
646
|
+
return [];
|
|
647
|
+
// Read-only protection: reject merge if any source observation is read-only
|
|
648
|
+
for (const group of groups) {
|
|
649
|
+
for (const obs of group.observations) {
|
|
650
|
+
if (obs.readOnly) {
|
|
651
|
+
throw new Error(`Cannot merge: observation #${obs.id} is read-only.`);
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
if (dryRun) {
|
|
656
|
+
return groups.map((g) => ({
|
|
657
|
+
mergedObservation: g.observations[0], // placeholder
|
|
658
|
+
deletedIds: g.observations.map((o) => o.id),
|
|
659
|
+
originalCount: g.observations.length,
|
|
660
|
+
strategy,
|
|
661
|
+
}));
|
|
662
|
+
}
|
|
663
|
+
const results = [];
|
|
664
|
+
const doMerge = this.db.transaction(() => {
|
|
665
|
+
for (const group of groups) {
|
|
666
|
+
const obs = group.observations.sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime());
|
|
667
|
+
// Synthesize content
|
|
668
|
+
const synthTitle = obs[obs.length - 1].title; // most recent title
|
|
669
|
+
const synthContent = obs
|
|
670
|
+
.map((o) => `--- [${o.createdAt.toISOString()}] ${o.title} ---\n${o.content}`)
|
|
671
|
+
.join('\n\n');
|
|
672
|
+
// Most frequent type
|
|
673
|
+
const typeCounts = {};
|
|
674
|
+
for (const o of obs) {
|
|
675
|
+
typeCounts[o.type] = (typeCounts[o.type] || 0) + 1;
|
|
676
|
+
}
|
|
677
|
+
const synthType = Object.entries(typeCounts).sort((a, b) => b[1] - a[1])[0][0];
|
|
678
|
+
const synthTopicKey = obs.find((o) => o.topicKey)?.topicKey ?? null;
|
|
679
|
+
const sourceIds = obs.map((o) => o.id);
|
|
680
|
+
const synthMetadata = this.serialize({
|
|
681
|
+
merged: true,
|
|
682
|
+
sourceIds,
|
|
683
|
+
mergedAt: new Date().toISOString(),
|
|
684
|
+
originalCount: obs.length,
|
|
685
|
+
});
|
|
686
|
+
const uuid = crypto.randomUUID();
|
|
687
|
+
const createdAt = Date.now();
|
|
688
|
+
const sessionId = obs[obs.length - 1].sessionId;
|
|
689
|
+
const insertResult = this.db
|
|
690
|
+
.prepare(`INSERT INTO observations (uuid, session_id, title, content, type, topic_key, project_id, created_at, deleted_at, metadata)
|
|
691
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL, ?)`)
|
|
692
|
+
.run(uuid, sessionId, synthTitle, synthContent, synthType, synthTopicKey, projectId, createdAt, synthMetadata);
|
|
693
|
+
const newId = typeof insertResult.lastInsertRowid === 'bigint'
|
|
694
|
+
? Number(insertResult.lastInsertRowid)
|
|
695
|
+
: insertResult.lastInsertRowid;
|
|
696
|
+
// Application-level FTS5 sync: insert merged, delete originals
|
|
697
|
+
this.db.prepare('INSERT INTO observations_fts(rowid, title, content, topic_key, project_id) VALUES (?, ?, ?, ?, ?)').run(newId, synthTitle, synthContent, synthTopicKey ?? '', projectId);
|
|
698
|
+
// Delete originals (from both observations and observations_fts)
|
|
699
|
+
const placeholders = sourceIds.map(() => '?').join(',');
|
|
700
|
+
this.db.prepare(`DELETE FROM observations_fts WHERE rowid IN (${placeholders})`).run(...sourceIds);
|
|
701
|
+
this.db.prepare(`DELETE FROM observations WHERE id IN (${placeholders})`).run(...sourceIds);
|
|
702
|
+
// Fetch the newly created merged observation
|
|
703
|
+
const merged = this.db.prepare('SELECT * FROM observations WHERE id = ?').get(newId);
|
|
704
|
+
results.push({
|
|
705
|
+
mergedObservation: this.mapObservation(merged),
|
|
706
|
+
deletedIds: sourceIds,
|
|
707
|
+
originalCount: obs.length,
|
|
708
|
+
strategy,
|
|
709
|
+
});
|
|
710
|
+
}
|
|
711
|
+
});
|
|
712
|
+
doMerge();
|
|
713
|
+
return results;
|
|
714
|
+
}
|
|
715
|
+
jaccardSimilarity(text1, text2) {
|
|
716
|
+
const tokenize = (t) => new Set(t
|
|
717
|
+
.toLowerCase()
|
|
718
|
+
.split(/\s+/)
|
|
719
|
+
.filter((w) => w.length > 2));
|
|
720
|
+
const set1 = tokenize(text1);
|
|
721
|
+
const set2 = tokenize(text2);
|
|
722
|
+
const intersection = [...set1].filter((x) => set2.has(x)).length;
|
|
723
|
+
const union = new Set([...set1, ...set2]).size;
|
|
724
|
+
return union === 0 ? 0 : intersection / union;
|
|
725
|
+
}
|
|
726
|
+
// ─── Export ────────────────────────────────────────────────
|
|
727
|
+
async exportObservations(params) {
|
|
728
|
+
this.checkHealth();
|
|
729
|
+
const { format, projectId, type, topicKey, dateFrom, dateTo, includeDeleted = false } = params;
|
|
226
730
|
let sql = 'SELECT * FROM observations WHERE 1=1';
|
|
227
731
|
const values = [];
|
|
228
|
-
if (
|
|
732
|
+
if (!includeDeleted) {
|
|
733
|
+
sql += ' AND deleted_at IS NULL';
|
|
734
|
+
}
|
|
735
|
+
if (projectId) {
|
|
736
|
+
sql += ' AND project_id = ?';
|
|
737
|
+
values.push(projectId);
|
|
738
|
+
}
|
|
739
|
+
if (type) {
|
|
740
|
+
sql += ' AND type = ?';
|
|
741
|
+
values.push(type);
|
|
742
|
+
}
|
|
743
|
+
if (topicKey) {
|
|
744
|
+
sql += ' AND topic_key = ?';
|
|
745
|
+
values.push(topicKey);
|
|
746
|
+
}
|
|
747
|
+
if (dateFrom) {
|
|
748
|
+
sql += ' AND created_at >= ?';
|
|
749
|
+
values.push(dateFrom.getTime());
|
|
750
|
+
}
|
|
751
|
+
if (dateTo) {
|
|
752
|
+
sql += ' AND created_at <= ?';
|
|
753
|
+
values.push(dateTo.getTime());
|
|
754
|
+
}
|
|
755
|
+
sql += ' ORDER BY created_at ASC, id ASC';
|
|
756
|
+
const rows = this.db.prepare(sql).all(...values);
|
|
757
|
+
const observations = rows.map((row) => this.mapObservation(row));
|
|
758
|
+
const exportedAt = new Date();
|
|
759
|
+
let content;
|
|
760
|
+
switch (format) {
|
|
761
|
+
case 'json':
|
|
762
|
+
content = this.exportToJSON(observations, exportedAt, params);
|
|
763
|
+
break;
|
|
764
|
+
case 'xml':
|
|
765
|
+
content = this.exportToXML(observations, exportedAt, params);
|
|
766
|
+
break;
|
|
767
|
+
case 'txt':
|
|
768
|
+
content = this.exportToTXT(observations, exportedAt, params);
|
|
769
|
+
break;
|
|
770
|
+
default:
|
|
771
|
+
throw new Error(`Unsupported export format: ${format}`);
|
|
772
|
+
}
|
|
773
|
+
return {
|
|
774
|
+
content,
|
|
775
|
+
format,
|
|
776
|
+
recordCount: observations.length,
|
|
777
|
+
exportedAt,
|
|
778
|
+
};
|
|
779
|
+
}
|
|
780
|
+
exportToJSON(observations, exportedAt, params) {
|
|
781
|
+
const data = {
|
|
782
|
+
exportedAt: exportedAt.toISOString(),
|
|
783
|
+
version: '1.0',
|
|
784
|
+
project: params.projectId || 'all',
|
|
785
|
+
filters: {
|
|
786
|
+
...(params.type && { type: params.type }),
|
|
787
|
+
...(params.topicKey && { topicKey: params.topicKey }),
|
|
788
|
+
...(params.dateFrom && { dateFrom: params.dateFrom.toISOString() }),
|
|
789
|
+
...(params.dateTo && { dateTo: params.dateTo.toISOString() }),
|
|
790
|
+
...(params.includeDeleted && { includeDeleted: true }),
|
|
791
|
+
},
|
|
792
|
+
totalRecords: observations.length,
|
|
793
|
+
observations: observations.map((o) => ({
|
|
794
|
+
id: o.id,
|
|
795
|
+
uuid: o.uuid,
|
|
796
|
+
title: o.title,
|
|
797
|
+
content: o.content,
|
|
798
|
+
type: o.type,
|
|
799
|
+
topicKey: o.topicKey,
|
|
800
|
+
projectId: o.projectId,
|
|
801
|
+
createdAt: o.createdAt.toISOString(),
|
|
802
|
+
...(o.deletedAt && { deletedAt: o.deletedAt.toISOString() }),
|
|
803
|
+
metadata: o.metadata,
|
|
804
|
+
})),
|
|
805
|
+
};
|
|
806
|
+
return JSON.stringify(data, null, 2);
|
|
807
|
+
}
|
|
808
|
+
exportToXML(observations, exportedAt, params) {
|
|
809
|
+
const escapeXml = (s) => s.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"');
|
|
810
|
+
let xml = `<?xml version="1.0" encoding="UTF-8"?>\n`;
|
|
811
|
+
xml += `<memento-export version="1.0" exportedAt="${exportedAt.toISOString()}">\n`;
|
|
812
|
+
xml += ` <project>${escapeXml(params.projectId || 'all')}</project>\n`;
|
|
813
|
+
xml += ` <observations count="${observations.length}">\n`;
|
|
814
|
+
for (const o of observations) {
|
|
815
|
+
xml += ` <observation id="${o.id}" uuid="${escapeXml(o.uuid)}">\n`;
|
|
816
|
+
xml += ` <title>${escapeXml(o.title)}</title>\n`;
|
|
817
|
+
xml += ` <content><![CDATA[${o.content}]]></content>\n`;
|
|
818
|
+
xml += ` <type>${escapeXml(o.type)}</type>\n`;
|
|
819
|
+
if (o.topicKey)
|
|
820
|
+
xml += ` <topicKey>${escapeXml(o.topicKey)}</topicKey>\n`;
|
|
821
|
+
xml += ` <projectId>${escapeXml(o.projectId)}</projectId>\n`;
|
|
822
|
+
xml += ` <createdAt>${o.createdAt.toISOString()}</createdAt>\n`;
|
|
823
|
+
if (o.deletedAt)
|
|
824
|
+
xml += ` <deletedAt>${o.deletedAt.toISOString()}</deletedAt>\n`;
|
|
825
|
+
xml += ` </observation>\n`;
|
|
826
|
+
}
|
|
827
|
+
xml += ` </observations>\n`;
|
|
828
|
+
xml += `</memento-export>\n`;
|
|
829
|
+
return xml;
|
|
830
|
+
}
|
|
831
|
+
exportToTXT(observations, exportedAt, params) {
|
|
832
|
+
const sep = '═'.repeat(50);
|
|
833
|
+
const thinSep = '─'.repeat(50);
|
|
834
|
+
const dateStr = exportedAt.toISOString().split('T')[0];
|
|
835
|
+
let txt = `${sep}\n`;
|
|
836
|
+
txt += `MEMENTO EXPORT — Project: ${params.projectId || 'all'}\n`;
|
|
837
|
+
txt += `Exported: ${dateStr} | Records: ${observations.length}\n`;
|
|
838
|
+
txt += `${sep}\n\n`;
|
|
839
|
+
for (const o of observations) {
|
|
840
|
+
txt += `[#${o.id}] ${o.title}\n`;
|
|
841
|
+
txt += `Type: ${o.type}`;
|
|
842
|
+
if (o.topicKey)
|
|
843
|
+
txt += ` | Topic: ${o.topicKey}`;
|
|
844
|
+
txt += `\nDate: ${o.createdAt.toISOString().split('T')[0]}`;
|
|
845
|
+
if (o.deletedAt)
|
|
846
|
+
txt += ` | DELETED: ${o.deletedAt.toISOString().split('T')[0]}`;
|
|
847
|
+
txt += `\n\n ${o.content}\n\n${thinSep}\n\n`;
|
|
848
|
+
}
|
|
849
|
+
return txt;
|
|
850
|
+
}
|
|
851
|
+
// ─── Search ────────────────────────────────────────────────
|
|
852
|
+
/**
|
|
853
|
+
* Sanitize a user query for safe use in FTS5 MATCH.
|
|
854
|
+
* Strips FTS5 operators and special characters that would cause syntax errors.
|
|
855
|
+
* Returns empty string if nothing survives sanitization.
|
|
856
|
+
*/
|
|
857
|
+
sanitizeFTS5Query(input) {
|
|
858
|
+
return input
|
|
859
|
+
.replace(/[^a-zA-Z0-9\s]/g, ' ') // Keep only alphanumeric and whitespace
|
|
860
|
+
.replace(/\s+/g, ' ') // Collapse whitespace
|
|
861
|
+
.trim();
|
|
862
|
+
}
|
|
863
|
+
async getObservation(id, includeDeleted = false) {
|
|
864
|
+
return await this.getObservationById(id, includeDeleted);
|
|
865
|
+
}
|
|
866
|
+
async search(params) {
|
|
867
|
+
const mode = params.mode || 'keyword';
|
|
868
|
+
// Semantic and hybrid modes require a query string
|
|
869
|
+
if ((mode === 'semantic' || mode === 'hybrid') && !params.query) {
|
|
870
|
+
// Fall back to keyword mode if no query provided
|
|
871
|
+
return this.searchKeyword(params);
|
|
872
|
+
}
|
|
873
|
+
switch (mode) {
|
|
874
|
+
case 'semantic':
|
|
875
|
+
return this.searchSemantic(params);
|
|
876
|
+
case 'hybrid':
|
|
877
|
+
return this.searchHybrid(params);
|
|
878
|
+
case 'keyword':
|
|
879
|
+
default:
|
|
880
|
+
return this.searchKeyword(params);
|
|
881
|
+
}
|
|
882
|
+
}
|
|
883
|
+
/**
|
|
884
|
+
* Keyword search using existing FTS5 (unchanged behavior).
|
|
885
|
+
*/
|
|
886
|
+
async searchKeyword(params) {
|
|
887
|
+
const { query, type, projectId, topicKey, limit = 100, offset = 0, includeDeleted = false, scope, sessionId, } = params;
|
|
888
|
+
let sql;
|
|
889
|
+
const values = [];
|
|
890
|
+
const sanitizedQuery = query ? this.sanitizeFTS5Query(query) : '';
|
|
891
|
+
if (sanitizedQuery) {
|
|
229
892
|
sql =
|
|
230
893
|
'SELECT observations.* FROM observations JOIN observations_fts ON observations.id = observations_fts.rowid WHERE observations_fts MATCH ?';
|
|
231
|
-
values.push(
|
|
894
|
+
values.push(sanitizedQuery);
|
|
895
|
+
if (!includeDeleted) {
|
|
896
|
+
sql += ' AND observations.deleted_at IS NULL';
|
|
897
|
+
}
|
|
232
898
|
if (type) {
|
|
233
899
|
sql += ' AND observations.type = ?';
|
|
234
900
|
values.push(type);
|
|
@@ -241,8 +907,20 @@ class MemoryEngine {
|
|
|
241
907
|
sql += ' AND observations.topic_key = ?';
|
|
242
908
|
values.push(topicKey);
|
|
243
909
|
}
|
|
910
|
+
if (scope) {
|
|
911
|
+
sql += ' AND observations.scope = ?';
|
|
912
|
+
values.push(scope);
|
|
913
|
+
}
|
|
914
|
+
if (sessionId) {
|
|
915
|
+
sql += ' AND observations.session_id = ?';
|
|
916
|
+
values.push(sessionId);
|
|
917
|
+
}
|
|
244
918
|
}
|
|
245
919
|
else {
|
|
920
|
+
sql = 'SELECT * FROM observations WHERE 1=1';
|
|
921
|
+
if (!includeDeleted) {
|
|
922
|
+
sql += ' AND deleted_at IS NULL';
|
|
923
|
+
}
|
|
246
924
|
if (type) {
|
|
247
925
|
sql += ' AND type = ?';
|
|
248
926
|
values.push(type);
|
|
@@ -255,16 +933,196 @@ class MemoryEngine {
|
|
|
255
933
|
sql += ' AND topic_key = ?';
|
|
256
934
|
values.push(topicKey);
|
|
257
935
|
}
|
|
936
|
+
if (scope) {
|
|
937
|
+
sql += ' AND scope = ?';
|
|
938
|
+
values.push(scope);
|
|
939
|
+
}
|
|
940
|
+
if (sessionId) {
|
|
941
|
+
sql += ' AND session_id = ?';
|
|
942
|
+
values.push(sessionId);
|
|
943
|
+
}
|
|
258
944
|
}
|
|
259
945
|
const countSql = sql.replace(/SELECT.*?FROM/, 'SELECT COUNT(*) as count FROM');
|
|
260
946
|
const countResult = this.db.prepare(countSql).get(...values);
|
|
261
947
|
const total = countResult ? countResult.count : 0;
|
|
262
|
-
sql += ' ORDER BY created_at DESC LIMIT ? OFFSET ?';
|
|
948
|
+
sql += ' ORDER BY created_at DESC, id DESC LIMIT ? OFFSET ?';
|
|
263
949
|
values.push(limit, offset);
|
|
264
950
|
const rows = this.db.prepare(sql).all(...values);
|
|
265
951
|
const observations = rows.map((row) => this.mapObservation(row));
|
|
266
952
|
return { observations, total };
|
|
267
953
|
}
|
|
954
|
+
/**
|
|
955
|
+
* Semantic search using cosine similarity on embeddings.
|
|
956
|
+
* Falls back to keyword search if embeddings are unavailable.
|
|
957
|
+
*/
|
|
958
|
+
async searchSemantic(params) {
|
|
959
|
+
this.checkHealth();
|
|
960
|
+
const { query, type, projectId, topicKey, limit = 10, scope } = params;
|
|
961
|
+
if (!query)
|
|
962
|
+
return { observations: [], total: 0 };
|
|
963
|
+
// Generate query embedding
|
|
964
|
+
const queryEmbedding = await this.embeddingService.generate(query);
|
|
965
|
+
if (!queryEmbedding) {
|
|
966
|
+
// Fall back to keyword search if embedding service unavailable
|
|
967
|
+
return this.searchKeyword(params);
|
|
968
|
+
}
|
|
969
|
+
// Build candidate filter
|
|
970
|
+
let candidateSql = 'SELECT o.* FROM observations o WHERE o.deleted_at IS NULL';
|
|
971
|
+
const values = [];
|
|
972
|
+
if (type) {
|
|
973
|
+
candidateSql += ' AND o.type = ?';
|
|
974
|
+
values.push(type);
|
|
975
|
+
}
|
|
976
|
+
if (projectId) {
|
|
977
|
+
candidateSql += ' AND o.project_id = ?';
|
|
978
|
+
values.push(projectId);
|
|
979
|
+
}
|
|
980
|
+
if (topicKey) {
|
|
981
|
+
candidateSql += ' AND o.topic_key = ?';
|
|
982
|
+
values.push(topicKey);
|
|
983
|
+
}
|
|
984
|
+
if (scope) {
|
|
985
|
+
candidateSql += ' AND o.scope = ?';
|
|
986
|
+
values.push(scope);
|
|
987
|
+
}
|
|
988
|
+
if (params.sessionId) {
|
|
989
|
+
candidateSql += ' AND o.session_id = ?';
|
|
990
|
+
values.push(params.sessionId);
|
|
991
|
+
}
|
|
992
|
+
// Only observations that have embeddings
|
|
993
|
+
candidateSql += ' AND EXISTS (SELECT 1 FROM embeddings e WHERE e.observation_id = o.id)';
|
|
994
|
+
const rows = this.db.prepare(candidateSql).all(...values);
|
|
995
|
+
const candidates = rows.map((row) => this.mapObservation(row));
|
|
996
|
+
// Score each candidate by cosine similarity
|
|
997
|
+
const scored = [];
|
|
998
|
+
for (const obs of candidates) {
|
|
999
|
+
const embeddingRow = this.db
|
|
1000
|
+
.prepare('SELECT embedding, dimensions FROM embeddings WHERE observation_id = ?')
|
|
1001
|
+
.get(obs.id);
|
|
1002
|
+
if (!embeddingRow)
|
|
1003
|
+
continue;
|
|
1004
|
+
const storedEmbedding = EmbeddingService_js_1.EmbeddingService.deserializeEmbedding(embeddingRow.embedding, embeddingRow.dimensions);
|
|
1005
|
+
const score = EmbeddingService_js_1.EmbeddingService.cosineSimilarity(queryEmbedding.embedding, storedEmbedding);
|
|
1006
|
+
scored.push({ observation: obs, score });
|
|
1007
|
+
}
|
|
1008
|
+
// Sort by score descending
|
|
1009
|
+
scored.sort((a, b) => b.score - a.score);
|
|
1010
|
+
const total = scored.length;
|
|
1011
|
+
const results = scored.slice(0, limit);
|
|
1012
|
+
const scores = new Map(results.map((r) => [r.observation.id, r.score]));
|
|
1013
|
+
const observations = results.map((r) => r.observation);
|
|
1014
|
+
return { observations, total, scores };
|
|
1015
|
+
}
|
|
1016
|
+
/**
|
|
1017
|
+
* Hybrid search combining FTS5 BM25 + cosine similarity.
|
|
1018
|
+
* Score = 0.4 * bm25_normalized + 0.6 * cosine_similarity
|
|
1019
|
+
*/
|
|
1020
|
+
async searchHybrid(params) {
|
|
1021
|
+
this.checkHealth();
|
|
1022
|
+
const { query, limit = 10 } = params;
|
|
1023
|
+
if (!query)
|
|
1024
|
+
return { observations: [], total: 0 };
|
|
1025
|
+
// Run both searches in parallel
|
|
1026
|
+
const [keywordResult, semanticResult] = await Promise.all([
|
|
1027
|
+
this.searchKeyword({ ...params, limit: limit * 3 }), // Get more candidates for re-ranking
|
|
1028
|
+
this.searchSemantic({ ...params, limit: limit * 3 }),
|
|
1029
|
+
]);
|
|
1030
|
+
// Normalize BM25 scores (use position-based ranking as proxy)
|
|
1031
|
+
const maxBm25Rank = keywordResult.observations.length;
|
|
1032
|
+
const bm25Scores = new Map();
|
|
1033
|
+
keywordResult.observations.forEach((obs, index) => {
|
|
1034
|
+
// Higher rank (earlier) = higher score
|
|
1035
|
+
bm25Scores.set(obs.id, maxBm25Rank > 0 ? 1 - index / maxBm25Rank : 0);
|
|
1036
|
+
});
|
|
1037
|
+
// Get semantic scores
|
|
1038
|
+
const semanticScores = semanticResult.scores || new Map();
|
|
1039
|
+
// Combine all unique observation IDs
|
|
1040
|
+
const allIds = new Set([...bm25Scores.keys(), ...semanticScores.keys()]);
|
|
1041
|
+
// Calculate hybrid scores
|
|
1042
|
+
const hybridWeight = { bm25: 0.4, cosine: 0.6 };
|
|
1043
|
+
const scored = [];
|
|
1044
|
+
for (const id of allIds) {
|
|
1045
|
+
const bm25 = bm25Scores.get(id) ?? 0;
|
|
1046
|
+
const cosine = semanticScores.get(id) ?? 0;
|
|
1047
|
+
const hybridScore = hybridWeight.bm25 * bm25 + hybridWeight.cosine * cosine;
|
|
1048
|
+
scored.push({ observationId: id, score: hybridScore });
|
|
1049
|
+
}
|
|
1050
|
+
scored.sort((a, b) => b.score - a.score);
|
|
1051
|
+
const topResults = scored.slice(0, limit);
|
|
1052
|
+
// Fetch full observations
|
|
1053
|
+
const observations = [];
|
|
1054
|
+
const scores = new Map();
|
|
1055
|
+
for (const result of topResults) {
|
|
1056
|
+
const obs = await this.getObservation(result.observationId);
|
|
1057
|
+
if (obs) {
|
|
1058
|
+
observations.push(obs);
|
|
1059
|
+
scores.set(obs.id, result.score);
|
|
1060
|
+
}
|
|
1061
|
+
}
|
|
1062
|
+
return { observations, total: scored.length, scores };
|
|
1063
|
+
}
|
|
1064
|
+
// ─── Embedding Management ──────────────────────────────────
|
|
1065
|
+
/**
|
|
1066
|
+
* Generate and store embedding for an observation.
|
|
1067
|
+
* Non-blocking: does not throw on failure.
|
|
1068
|
+
*/
|
|
1069
|
+
async generateEmbedding(observationId) {
|
|
1070
|
+
this.checkHealth();
|
|
1071
|
+
const obs = await this.getObservationById(observationId);
|
|
1072
|
+
if (!obs)
|
|
1073
|
+
return false;
|
|
1074
|
+
const text = `${obs.title} ${obs.content}`;
|
|
1075
|
+
const result = await this.embeddingService.generate(text);
|
|
1076
|
+
if (!result)
|
|
1077
|
+
return false;
|
|
1078
|
+
const blob = EmbeddingService_js_1.EmbeddingService.serializeEmbedding(result.embedding);
|
|
1079
|
+
const now = Date.now();
|
|
1080
|
+
await this.withRetry(() => {
|
|
1081
|
+
this.db.prepare(`INSERT OR REPLACE INTO embeddings (observation_id, embedding, model, dimensions, generated_at)
|
|
1082
|
+
VALUES (?, ?, ?, ?, ?)`).run(observationId, blob, result.model, result.dimensions, now);
|
|
1083
|
+
});
|
|
1084
|
+
return true;
|
|
1085
|
+
}
|
|
1086
|
+
/**
|
|
1087
|
+
* Delete embedding for an observation.
|
|
1088
|
+
*/
|
|
1089
|
+
async deleteEmbedding(observationId) {
|
|
1090
|
+
this.checkHealth();
|
|
1091
|
+
this.db.prepare('DELETE FROM embeddings WHERE observation_id = ?').run(observationId);
|
|
1092
|
+
}
|
|
1093
|
+
/**
|
|
1094
|
+
* Get embedding status (is the service available?).
|
|
1095
|
+
*/
|
|
1096
|
+
async getEmbeddingStatus() {
|
|
1097
|
+
return this.embeddingService.status;
|
|
1098
|
+
}
|
|
1099
|
+
/**
|
|
1100
|
+
* Backfill embeddings for observations that don't have them.
|
|
1101
|
+
* Processes in batches. Returns count of embeddings generated.
|
|
1102
|
+
*/
|
|
1103
|
+
async backfillEmbeddings(batchSize = 50) {
|
|
1104
|
+
this.checkHealth();
|
|
1105
|
+
const rows = this.db.prepare(`SELECT o.id FROM observations o
|
|
1106
|
+
WHERE o.deleted_at IS NULL
|
|
1107
|
+
AND NOT EXISTS (SELECT 1 FROM embeddings e WHERE e.observation_id = o.id)
|
|
1108
|
+
ORDER BY o.id ASC`).all();
|
|
1109
|
+
let processed = 0;
|
|
1110
|
+
let failed = 0;
|
|
1111
|
+
for (let i = 0; i < rows.length; i += batchSize) {
|
|
1112
|
+
const batch = rows.slice(i, i + batchSize);
|
|
1113
|
+
for (const row of batch) {
|
|
1114
|
+
const success = await this.generateEmbedding(row.id);
|
|
1115
|
+
if (success) {
|
|
1116
|
+
processed++;
|
|
1117
|
+
}
|
|
1118
|
+
else {
|
|
1119
|
+
failed++;
|
|
1120
|
+
}
|
|
1121
|
+
}
|
|
1122
|
+
}
|
|
1123
|
+
return { processed, failed };
|
|
1124
|
+
}
|
|
1125
|
+
// ─── Sessions ──────────────────────────────────────────────
|
|
268
1126
|
async createSession(data) {
|
|
269
1127
|
const uuid = crypto.randomUUID();
|
|
270
1128
|
const startedAt = new Date();
|
|
@@ -278,8 +1136,62 @@ class MemoryEngine {
|
|
|
278
1136
|
const session = await this.getSessionById(id);
|
|
279
1137
|
if (!session)
|
|
280
1138
|
throw new Error('Failed to retrieve created session');
|
|
1139
|
+
// Seed default observations if requested and this is the first session for the project
|
|
1140
|
+
if (data.seedIfEmpty) {
|
|
1141
|
+
await this.seedIfEmpty(data.projectId, id);
|
|
1142
|
+
}
|
|
281
1143
|
return session;
|
|
282
1144
|
}
|
|
1145
|
+
/**
|
|
1146
|
+
* Seed default observations if project has no observations yet.
|
|
1147
|
+
* Creates 3 seeds: persona (personal scope), human (personal), project (project scope).
|
|
1148
|
+
* Public so that `memento init` can call it directly.
|
|
1149
|
+
*/
|
|
1150
|
+
async seedIfEmpty(projectId, sessionId) {
|
|
1151
|
+
// Check if project already has observations
|
|
1152
|
+
const existing = this.db
|
|
1153
|
+
.prepare('SELECT COUNT(*) as count FROM observations WHERE project_id = ? AND deleted_at IS NULL')
|
|
1154
|
+
.get(projectId);
|
|
1155
|
+
if (existing && existing.count > 0)
|
|
1156
|
+
return;
|
|
1157
|
+
// Seed personal-scope observations (only if no personal observations exist)
|
|
1158
|
+
const personalExisting = this.db
|
|
1159
|
+
.prepare("SELECT COUNT(*) as count FROM observations WHERE scope = 'personal' AND deleted_at IS NULL")
|
|
1160
|
+
.get();
|
|
1161
|
+
if (!personalExisting || personalExisting.count === 0) {
|
|
1162
|
+
await this.createObservation({
|
|
1163
|
+
sessionId,
|
|
1164
|
+
title: 'Agent Persona',
|
|
1165
|
+
content: 'Instructions for how the agent should behave and respond. Fill this in with your preferred communication style, tone, and approach.',
|
|
1166
|
+
type: 'preference',
|
|
1167
|
+
topicKey: 'persona',
|
|
1168
|
+
projectId,
|
|
1169
|
+
metadata: { seed: true },
|
|
1170
|
+
scope: 'personal',
|
|
1171
|
+
});
|
|
1172
|
+
await this.createObservation({
|
|
1173
|
+
sessionId,
|
|
1174
|
+
title: 'User Preferences',
|
|
1175
|
+
content: 'Key details about the user: preferred language, frameworks, communication style, constraints, and habits.',
|
|
1176
|
+
type: 'preference',
|
|
1177
|
+
topicKey: 'human',
|
|
1178
|
+
projectId,
|
|
1179
|
+
metadata: { seed: true },
|
|
1180
|
+
scope: 'personal',
|
|
1181
|
+
});
|
|
1182
|
+
}
|
|
1183
|
+
// Seed project-scope observation
|
|
1184
|
+
await this.createObservation({
|
|
1185
|
+
sessionId,
|
|
1186
|
+
title: 'Project Knowledge',
|
|
1187
|
+
content: 'Durable, high-signal information about this codebase: commands, architecture notes, conventions, and gotchas.',
|
|
1188
|
+
type: 'note',
|
|
1189
|
+
topicKey: 'project',
|
|
1190
|
+
projectId,
|
|
1191
|
+
metadata: { seed: true },
|
|
1192
|
+
scope: 'project',
|
|
1193
|
+
});
|
|
1194
|
+
}
|
|
283
1195
|
async endSession(id) {
|
|
284
1196
|
const session = await this.getSessionById(id);
|
|
285
1197
|
if (!session)
|
|
@@ -291,9 +1203,69 @@ class MemoryEngine {
|
|
|
291
1203
|
throw new Error('Failed to update session');
|
|
292
1204
|
return updated;
|
|
293
1205
|
}
|
|
1206
|
+
/**
|
|
1207
|
+
* Close stale (orphaned) sessions for a specific project.
|
|
1208
|
+
* Sessions are considered stale if they have no ended_at AND started_at is older than maxAgeMs.
|
|
1209
|
+
* Auto-closed sessions receive metadata: { auto_closed: true, reason: 'stale', closed_at: <timestamp> }.
|
|
1210
|
+
* Returns the count of closed sessions.
|
|
1211
|
+
*/
|
|
1212
|
+
closeStaleSessionsForProject(projectId, maxAgeMs) {
|
|
1213
|
+
this.checkHealth();
|
|
1214
|
+
const maxAge = maxAgeMs ?? 24 * 60 * 60 * 1000; // default 24h
|
|
1215
|
+
const cutoff = Date.now() - maxAge;
|
|
1216
|
+
const now = Date.now();
|
|
1217
|
+
// Get sessions to close (need to update metadata individually)
|
|
1218
|
+
const staleRows = this.db.prepare('SELECT id, metadata FROM sessions WHERE ended_at IS NULL AND project_id = ? AND started_at < ?').all(projectId, cutoff);
|
|
1219
|
+
if (staleRows.length === 0)
|
|
1220
|
+
return { closed: 0 };
|
|
1221
|
+
const updateStmt = this.db.prepare('UPDATE sessions SET ended_at = ?, metadata = ? WHERE id = ?');
|
|
1222
|
+
let closed = 0;
|
|
1223
|
+
for (const row of staleRows) {
|
|
1224
|
+
const existingMeta = this.deserialize(row.metadata);
|
|
1225
|
+
const mergedMeta = {
|
|
1226
|
+
...existingMeta,
|
|
1227
|
+
auto_closed: true,
|
|
1228
|
+
reason: 'stale',
|
|
1229
|
+
closed_at: now,
|
|
1230
|
+
};
|
|
1231
|
+
updateStmt.run(now, this.serialize(mergedMeta), row.id);
|
|
1232
|
+
closed++;
|
|
1233
|
+
}
|
|
1234
|
+
return { closed };
|
|
1235
|
+
}
|
|
1236
|
+
/**
|
|
1237
|
+
* Close ALL stale (orphaned) sessions across all projects.
|
|
1238
|
+
* Sessions are considered stale if they have no ended_at AND started_at is older than maxAgeMs.
|
|
1239
|
+
* Returns the count of closed sessions.
|
|
1240
|
+
*/
|
|
1241
|
+
closeStaleSessions(maxAgeMs) {
|
|
1242
|
+
this.checkHealth();
|
|
1243
|
+
const maxAge = maxAgeMs ?? 24 * 60 * 60 * 1000; // default 24h
|
|
1244
|
+
const cutoff = Date.now() - maxAge;
|
|
1245
|
+
const now = Date.now();
|
|
1246
|
+
// Get sessions to close
|
|
1247
|
+
const staleRows = this.db.prepare('SELECT id, metadata FROM sessions WHERE ended_at IS NULL AND started_at < ?').all(cutoff);
|
|
1248
|
+
if (staleRows.length === 0)
|
|
1249
|
+
return { closed: 0 };
|
|
1250
|
+
const updateStmt = this.db.prepare('UPDATE sessions SET ended_at = ?, metadata = ? WHERE id = ?');
|
|
1251
|
+
let closed = 0;
|
|
1252
|
+
for (const row of staleRows) {
|
|
1253
|
+
const existingMeta = this.deserialize(row.metadata);
|
|
1254
|
+
const mergedMeta = {
|
|
1255
|
+
...existingMeta,
|
|
1256
|
+
auto_closed: true,
|
|
1257
|
+
reason: 'stale',
|
|
1258
|
+
closed_at: now,
|
|
1259
|
+
};
|
|
1260
|
+
updateStmt.run(now, this.serialize(mergedMeta), row.id);
|
|
1261
|
+
closed++;
|
|
1262
|
+
}
|
|
1263
|
+
return { closed };
|
|
1264
|
+
}
|
|
294
1265
|
async getSession(id) {
|
|
295
1266
|
return await this.getSessionById(id);
|
|
296
1267
|
}
|
|
1268
|
+
// ─── Prompts ───────────────────────────────────────────────
|
|
297
1269
|
async savePrompt(data) {
|
|
298
1270
|
const uuid = crypto.randomUUID();
|
|
299
1271
|
const createdAt = new Date();
|
|
@@ -309,11 +1281,140 @@ class MemoryEngine {
|
|
|
309
1281
|
throw new Error('Failed to retrieve saved prompt');
|
|
310
1282
|
return prompt;
|
|
311
1283
|
}
|
|
312
|
-
|
|
313
|
-
|
|
1284
|
+
// ─── Prompt Injection (OpenCode Plugin) ──────────────────────
|
|
1285
|
+
/**
|
|
1286
|
+
* Select observations for prompt injection based on config.
|
|
1287
|
+
* Returns observations sorted deterministically for prompt caching:
|
|
1288
|
+
* 1. Pinned observations (by ID ascending)
|
|
1289
|
+
* 2. Non-pinned observations (by ID ascending)
|
|
1290
|
+
*/
|
|
1291
|
+
selectForPrompt(config) {
|
|
1292
|
+
this.checkHealth();
|
|
1293
|
+
const types = config.types.map((t) => `'${t}'`).join(',');
|
|
1294
|
+
const values = [];
|
|
1295
|
+
let sql = `SELECT * FROM observations WHERE deleted_at IS NULL AND type IN (${types})`;
|
|
1296
|
+
if (config.projectId) {
|
|
1297
|
+
sql += ' AND project_id = ?';
|
|
1298
|
+
values.push(config.projectId);
|
|
1299
|
+
}
|
|
1300
|
+
if (config.strategy === 'pinned-only') {
|
|
1301
|
+
sql += ' AND pinned = 1';
|
|
1302
|
+
}
|
|
1303
|
+
// Deterministic sort: pinned first (by ID ASC), then non-pinned (by ID ASC)
|
|
1304
|
+
sql += ' ORDER BY pinned DESC, id ASC';
|
|
1305
|
+
// Fetch more than needed to allow for token budget trimming
|
|
1306
|
+
const rows = this.db.prepare(sql).all(...values);
|
|
1307
|
+
const observations = rows.map((row) => this.mapObservation(row));
|
|
1308
|
+
// Apply maxObservations limit
|
|
1309
|
+
const limited = observations.slice(0, config.maxObservations);
|
|
1310
|
+
// Apply token budget
|
|
1311
|
+
const maxChars = config.maxTokens * 4; // approximate: chars / 4 = tokens
|
|
1312
|
+
let totalChars = 0;
|
|
1313
|
+
const result = [];
|
|
1314
|
+
for (const obs of limited) {
|
|
1315
|
+
const obsChars = this.estimateObservationChars(obs);
|
|
1316
|
+
if (totalChars + obsChars > maxChars && result.length > 0) {
|
|
1317
|
+
break;
|
|
1318
|
+
}
|
|
1319
|
+
totalChars += obsChars;
|
|
1320
|
+
result.push(obs);
|
|
1321
|
+
}
|
|
1322
|
+
// Re-sort: pinned first by ID ASC, then non-pinned by ID ASC
|
|
1323
|
+
// (ensures deterministic order even after budget trim)
|
|
1324
|
+
result.sort((a, b) => {
|
|
1325
|
+
if (a.pinned !== b.pinned)
|
|
1326
|
+
return a.pinned ? -1 : 1;
|
|
1327
|
+
return a.id - b.id;
|
|
1328
|
+
});
|
|
1329
|
+
return result;
|
|
1330
|
+
}
|
|
1331
|
+
/**
|
|
1332
|
+
* Render observations as compact XML for system prompt injection.
|
|
1333
|
+
*/
|
|
1334
|
+
renderPromptContext(observations, config) {
|
|
1335
|
+
if (observations.length === 0) {
|
|
1336
|
+
return { xml: '', observationCount: 0, tokenCount: 0, budgetExceeded: false };
|
|
1337
|
+
}
|
|
1338
|
+
const parts = ['<memento_context>'];
|
|
1339
|
+
let totalChars = 0;
|
|
1340
|
+
const maxChars = config.maxTokens * 4;
|
|
1341
|
+
let budgetExceeded = false;
|
|
1342
|
+
for (const obs of observations) {
|
|
1343
|
+
const xmlNode = this.renderObservationXml(obs);
|
|
1344
|
+
const nodeChars = xmlNode.length;
|
|
1345
|
+
if (totalChars + nodeChars > maxChars && parts.length > 1) {
|
|
1346
|
+
budgetExceeded = true;
|
|
1347
|
+
break;
|
|
1348
|
+
}
|
|
1349
|
+
parts.push(xmlNode);
|
|
1350
|
+
totalChars += nodeChars;
|
|
1351
|
+
}
|
|
1352
|
+
parts.push('</memento_context>');
|
|
1353
|
+
const xml = parts.join('\n');
|
|
1354
|
+
return {
|
|
1355
|
+
xml,
|
|
1356
|
+
observationCount: parts.length - 2, // subtract opening and closing tags
|
|
1357
|
+
tokenCount: Math.ceil(xml.length / 4),
|
|
1358
|
+
budgetExceeded,
|
|
1359
|
+
};
|
|
1360
|
+
}
|
|
1361
|
+
renderObservationXml(obs) {
|
|
1362
|
+
const pinnedAttr = obs.pinned ? ' pinned="true"' : '';
|
|
1363
|
+
const projectAttr = obs.projectId ? ` project="${obs.projectId}"` : '';
|
|
1364
|
+
// Compact: title as attribute, content as text (trimmed to first 500 chars for budget)
|
|
1365
|
+
const truncatedContent = obs.content.length > 500
|
|
1366
|
+
? obs.content.slice(0, 500) + '...'
|
|
1367
|
+
: obs.content;
|
|
1368
|
+
return `<observation id="${obs.id}" type="${obs.type}"${projectAttr}${pinnedAttr}>\n${obs.title}\n${truncatedContent}\n</observation>`;
|
|
1369
|
+
}
|
|
1370
|
+
estimateObservationChars(obs) {
|
|
1371
|
+
// Estimate XML overhead: ~100 chars for tags + attributes
|
|
1372
|
+
// Plus title + first 500 chars of content
|
|
1373
|
+
const contentLen = Math.min(obs.content.length, 500);
|
|
1374
|
+
return 100 + obs.title.length + contentLen;
|
|
1375
|
+
}
|
|
1376
|
+
/**
|
|
1377
|
+
* Pin an observation (always include in prompt injection).
|
|
1378
|
+
*/
|
|
1379
|
+
async pinObservation(id) {
|
|
1380
|
+
return this.updateObservation(id, { pinned: true });
|
|
1381
|
+
}
|
|
1382
|
+
/**
|
|
1383
|
+
* Unpin an observation.
|
|
1384
|
+
*/
|
|
1385
|
+
async unpinObservation(id) {
|
|
1386
|
+
return this.updateObservation(id, { pinned: false });
|
|
1387
|
+
}
|
|
1388
|
+
/**
|
|
1389
|
+
* Lock an observation (set read-only). Prevents all modifications.
|
|
1390
|
+
*/
|
|
1391
|
+
async lockObservation(id) {
|
|
1392
|
+
return this.updateObservation(id, { readOnly: true });
|
|
1393
|
+
}
|
|
1394
|
+
/**
|
|
1395
|
+
* Unlock an observation (remove read-only protection).
|
|
1396
|
+
*/
|
|
1397
|
+
async unlockObservation(id) {
|
|
1398
|
+
return this.updateObservation(id, { readOnly: false });
|
|
1399
|
+
}
|
|
1400
|
+
// ─── Private helpers ───────────────────────────────────────
|
|
1401
|
+
async getObservationById(id, includeDeleted = false) {
|
|
1402
|
+
let sql = 'SELECT * FROM observations WHERE id = ?';
|
|
1403
|
+
if (!includeDeleted) {
|
|
1404
|
+
sql += ' AND deleted_at IS NULL';
|
|
1405
|
+
}
|
|
1406
|
+
const row = this.db.prepare(sql).get(id);
|
|
314
1407
|
if (!row)
|
|
315
1408
|
return null;
|
|
316
|
-
|
|
1409
|
+
const observation = this.mapObservation(row);
|
|
1410
|
+
// Calculate duplicatesCount when topicKey is present
|
|
1411
|
+
if (observation.topicKey) {
|
|
1412
|
+
const dupResult = this.db
|
|
1413
|
+
.prepare('SELECT COUNT(*) as count FROM observations WHERE topic_key = ? AND deleted_at IS NULL')
|
|
1414
|
+
.get(observation.topicKey);
|
|
1415
|
+
observation.duplicatesCount = dupResult?.count ?? 0;
|
|
1416
|
+
}
|
|
1417
|
+
return observation;
|
|
317
1418
|
}
|
|
318
1419
|
async getSessionById(id) {
|
|
319
1420
|
const row = this.db.prepare('SELECT * FROM sessions WHERE id = ?').get(id);
|
|
@@ -339,7 +1440,12 @@ class MemoryEngine {
|
|
|
339
1440
|
topicKey: r.topic_key,
|
|
340
1441
|
projectId: r.project_id,
|
|
341
1442
|
createdAt: new Date(r.created_at),
|
|
1443
|
+
deletedAt: r.deleted_at ? new Date(r.deleted_at) : null,
|
|
342
1444
|
metadata: this.deserialize(r.metadata),
|
|
1445
|
+
scope: r.scope || 'project',
|
|
1446
|
+
pinned: Boolean(r.pinned),
|
|
1447
|
+
readOnly: Boolean(r.read_only),
|
|
1448
|
+
revisionCount: r.revision_count ?? 0,
|
|
343
1449
|
};
|
|
344
1450
|
}
|
|
345
1451
|
mapSession(row) {
|
|
@@ -351,6 +1457,7 @@ class MemoryEngine {
|
|
|
351
1457
|
startedAt: new Date(r.started_at),
|
|
352
1458
|
endedAt: r.ended_at ? new Date(r.ended_at) : null,
|
|
353
1459
|
metadata: this.deserialize(r.metadata),
|
|
1460
|
+
observationCount: r.observation_count ?? 0,
|
|
354
1461
|
};
|
|
355
1462
|
}
|
|
356
1463
|
mapPrompt(row) {
|
|
@@ -365,6 +1472,1114 @@ class MemoryEngine {
|
|
|
365
1472
|
metadata: this.deserialize(r.metadata),
|
|
366
1473
|
};
|
|
367
1474
|
}
|
|
1475
|
+
// ─── Timeline & Context ──────────────────────────────────────
|
|
1476
|
+
async getTimeline(params) {
|
|
1477
|
+
this.checkHealth();
|
|
1478
|
+
const { projectId, scope, limit = 50, offset = 0 } = params;
|
|
1479
|
+
let countSql = 'SELECT COUNT(*) as count FROM observations WHERE deleted_at IS NULL';
|
|
1480
|
+
let sql = 'SELECT * FROM observations WHERE deleted_at IS NULL';
|
|
1481
|
+
const values = [];
|
|
1482
|
+
if (projectId) {
|
|
1483
|
+
countSql += ' AND project_id = ?';
|
|
1484
|
+
sql += ' AND project_id = ?';
|
|
1485
|
+
values.push(projectId);
|
|
1486
|
+
}
|
|
1487
|
+
if (scope) {
|
|
1488
|
+
countSql += ' AND scope = ?';
|
|
1489
|
+
sql += ' AND scope = ?';
|
|
1490
|
+
values.push(scope);
|
|
1491
|
+
}
|
|
1492
|
+
const countResult = this.db.prepare(countSql).get(...values);
|
|
1493
|
+
const total = countResult?.count ?? 0;
|
|
1494
|
+
sql += ' ORDER BY created_at DESC, id DESC LIMIT ? OFFSET ?';
|
|
1495
|
+
const rows = this.db.prepare(sql).all(...values, limit, offset);
|
|
1496
|
+
const observations = rows.map((row) => this.mapObservation(row));
|
|
1497
|
+
return { observations, total };
|
|
1498
|
+
}
|
|
1499
|
+
async getRecentContext(params) {
|
|
1500
|
+
this.checkHealth();
|
|
1501
|
+
const { projectId, limit = 20, scope } = params;
|
|
1502
|
+
let sql = 'SELECT * FROM observations WHERE deleted_at IS NULL';
|
|
1503
|
+
const values = [];
|
|
1504
|
+
if (projectId) {
|
|
1505
|
+
sql += ' AND project_id = ?';
|
|
1506
|
+
values.push(projectId);
|
|
1507
|
+
}
|
|
1508
|
+
if (scope) {
|
|
1509
|
+
sql += ' AND scope = ?';
|
|
1510
|
+
values.push(scope);
|
|
1511
|
+
}
|
|
1512
|
+
sql += ' ORDER BY created_at DESC, id DESC LIMIT ?';
|
|
1513
|
+
const rows = this.db.prepare(sql).all(...values, limit);
|
|
1514
|
+
const observations = rows.map((row) => this.mapObservation(row));
|
|
1515
|
+
// Get total for context
|
|
1516
|
+
let countSql = 'SELECT COUNT(*) as count FROM observations WHERE deleted_at IS NULL';
|
|
1517
|
+
const countValues = [];
|
|
1518
|
+
if (projectId) {
|
|
1519
|
+
countSql += ' AND project_id = ?';
|
|
1520
|
+
countValues.push(projectId);
|
|
1521
|
+
}
|
|
1522
|
+
const countResult = this.db.prepare(countSql).get(...countValues);
|
|
1523
|
+
const total = countResult?.count ?? 0;
|
|
1524
|
+
return { observations, total };
|
|
1525
|
+
}
|
|
1526
|
+
// ─── TUI Explorer API ──────────────────────────────────────
|
|
1527
|
+
async listSessions(params) {
|
|
1528
|
+
this.checkHealth();
|
|
1529
|
+
const { projectId, activeOnly = false, limit = 50, offset = 0 } = params;
|
|
1530
|
+
// Build WHERE conditions once, reuse for count and fetch
|
|
1531
|
+
let whereClause = ' WHERE 1=1';
|
|
1532
|
+
const values = [];
|
|
1533
|
+
if (projectId) {
|
|
1534
|
+
whereClause += ' AND project_id = ?';
|
|
1535
|
+
values.push(projectId);
|
|
1536
|
+
}
|
|
1537
|
+
if (activeOnly) {
|
|
1538
|
+
whereClause += ' AND ended_at IS NULL';
|
|
1539
|
+
}
|
|
1540
|
+
// Count (no subquery needed)
|
|
1541
|
+
const countResult = this.db.prepare(`SELECT COUNT(*) as count FROM sessions${whereClause}`).get(...values);
|
|
1542
|
+
const total = countResult?.count ?? 0;
|
|
1543
|
+
// Fetch with observation count subquery
|
|
1544
|
+
const sql = `SELECT s.*,
|
|
1545
|
+
(SELECT COUNT(*) FROM observations o WHERE o.session_id = s.id AND o.deleted_at IS NULL) as observation_count
|
|
1546
|
+
FROM sessions s${whereClause}
|
|
1547
|
+
ORDER BY s.started_at DESC, s.id DESC LIMIT ? OFFSET ?`;
|
|
1548
|
+
values.push(limit, offset);
|
|
1549
|
+
const rows = this.db.prepare(sql).all(...values);
|
|
1550
|
+
const sessions = rows.map((row) => this.mapSession(row));
|
|
1551
|
+
return { sessions, total };
|
|
1552
|
+
}
|
|
1553
|
+
async listProjects() {
|
|
1554
|
+
this.checkHealth();
|
|
1555
|
+
// Get all distinct project_ids with their stats
|
|
1556
|
+
const rows = this.db
|
|
1557
|
+
.prepare(`
|
|
1558
|
+
SELECT
|
|
1559
|
+
project_id,
|
|
1560
|
+
SUM(CASE WHEN deleted_at IS NULL THEN 1 ELSE 0 END) as active_count,
|
|
1561
|
+
SUM(CASE WHEN deleted_at IS NOT NULL THEN 1 ELSE 0 END) as deleted_count,
|
|
1562
|
+
MAX(created_at) as last_activity,
|
|
1563
|
+
MAX(id) as max_id
|
|
1564
|
+
FROM observations
|
|
1565
|
+
GROUP BY project_id
|
|
1566
|
+
ORDER BY last_activity DESC, max_id DESC
|
|
1567
|
+
`)
|
|
1568
|
+
.all();
|
|
1569
|
+
const projects = rows.map((row) => {
|
|
1570
|
+
const projectId = row.project_id;
|
|
1571
|
+
// Get type distribution for this project
|
|
1572
|
+
const typeRows = this.db
|
|
1573
|
+
.prepare(`
|
|
1574
|
+
SELECT type, COUNT(*) as count
|
|
1575
|
+
FROM observations
|
|
1576
|
+
WHERE project_id = ?
|
|
1577
|
+
GROUP BY type
|
|
1578
|
+
`)
|
|
1579
|
+
.all(projectId);
|
|
1580
|
+
const byType = {
|
|
1581
|
+
decision: 0,
|
|
1582
|
+
bug: 0,
|
|
1583
|
+
discovery: 0,
|
|
1584
|
+
note: 0,
|
|
1585
|
+
summary: 0,
|
|
1586
|
+
learning: 0,
|
|
1587
|
+
pattern: 0,
|
|
1588
|
+
architecture: 0,
|
|
1589
|
+
config: 0,
|
|
1590
|
+
preference: 0,
|
|
1591
|
+
};
|
|
1592
|
+
for (const tr of typeRows) {
|
|
1593
|
+
byType[tr.type] = tr.count;
|
|
1594
|
+
}
|
|
1595
|
+
return {
|
|
1596
|
+
name: projectId,
|
|
1597
|
+
activeCount: row.active_count,
|
|
1598
|
+
deletedCount: row.deleted_count,
|
|
1599
|
+
lastActivity: row.last_activity ? new Date(row.last_activity) : null,
|
|
1600
|
+
byType,
|
|
1601
|
+
};
|
|
1602
|
+
});
|
|
1603
|
+
return projects;
|
|
1604
|
+
}
|
|
1605
|
+
// ─── Project Registry (Issue #177) ────────────────────────
|
|
1606
|
+
/**
|
|
1607
|
+
* Register a project in the projects table.
|
|
1608
|
+
* Uses INSERT OR IGNORE to be idempotent.
|
|
1609
|
+
* Returns the registered project name (normalized).
|
|
1610
|
+
*/
|
|
1611
|
+
registerProject(name, workingDir) {
|
|
1612
|
+
this.checkHealth();
|
|
1613
|
+
const { normalizeProjectId } = require('./ConfigManager');
|
|
1614
|
+
const normalizedName = normalizeProjectId(name);
|
|
1615
|
+
const now = Date.now();
|
|
1616
|
+
this.db
|
|
1617
|
+
.prepare(`INSERT OR IGNORE INTO projects (name, created_at, metadata, working_dir, aliases) VALUES (?, ?, ?, ?, ?)`)
|
|
1618
|
+
.run(normalizedName, now, '{}', workingDir || null, '[]');
|
|
1619
|
+
return normalizedName;
|
|
1620
|
+
}
|
|
1621
|
+
/**
|
|
1622
|
+
* Get all registered project names from the projects table.
|
|
1623
|
+
*/
|
|
1624
|
+
listRegisteredProjects() {
|
|
1625
|
+
this.checkHealth();
|
|
1626
|
+
const rows = this.db
|
|
1627
|
+
.prepare('SELECT name, working_dir, aliases FROM projects ORDER BY name')
|
|
1628
|
+
.all();
|
|
1629
|
+
return rows.map((row) => ({
|
|
1630
|
+
name: row.name,
|
|
1631
|
+
workingDir: row.working_dir,
|
|
1632
|
+
aliases: row.aliases ? JSON.parse(row.aliases) : [],
|
|
1633
|
+
}));
|
|
1634
|
+
}
|
|
1635
|
+
/**
|
|
1636
|
+
* Merge all observations and sessions from sourceProject into targetProject.
|
|
1637
|
+
* Also updates FTS index. Returns count of affected records.
|
|
1638
|
+
*
|
|
1639
|
+
* Tries both the raw source name and the normalized version to handle
|
|
1640
|
+
* un-normalized data in the database.
|
|
1641
|
+
*/
|
|
1642
|
+
mergeProject(sourceProjectId, targetProjectId) {
|
|
1643
|
+
this.checkHealth();
|
|
1644
|
+
const { normalizeProjectId } = require('./ConfigManager');
|
|
1645
|
+
const normalizedSource = normalizeProjectId(sourceProjectId);
|
|
1646
|
+
const target = normalizeProjectId(targetProjectId);
|
|
1647
|
+
// Check same-name BEFORE searching for data
|
|
1648
|
+
if (normalizedSource === target) {
|
|
1649
|
+
throw new Error(`Source and target are the same after normalization: "${normalizedSource}"`);
|
|
1650
|
+
}
|
|
1651
|
+
// Resolve source: try raw name first, then normalized
|
|
1652
|
+
const candidates = [sourceProjectId, normalizedSource];
|
|
1653
|
+
let source = null;
|
|
1654
|
+
for (const candidate of candidates) {
|
|
1655
|
+
if (candidate === target)
|
|
1656
|
+
continue; // skip if same as target
|
|
1657
|
+
const count = this.db
|
|
1658
|
+
.prepare('SELECT COUNT(*) as count FROM observations WHERE project_id = ?')
|
|
1659
|
+
.get(candidate);
|
|
1660
|
+
if (count.count > 0) {
|
|
1661
|
+
source = candidate;
|
|
1662
|
+
break;
|
|
1663
|
+
}
|
|
1664
|
+
// Also check sessions and journal
|
|
1665
|
+
const sessionCount = this.db
|
|
1666
|
+
.prepare('SELECT COUNT(*) as count FROM sessions WHERE project_id = ?')
|
|
1667
|
+
.get(candidate);
|
|
1668
|
+
const journalCount = this.db
|
|
1669
|
+
.prepare('SELECT COUNT(*) as count FROM journal WHERE project_id = ?')
|
|
1670
|
+
.get(candidate);
|
|
1671
|
+
if (sessionCount.count > 0 || journalCount.count > 0) {
|
|
1672
|
+
source = candidate;
|
|
1673
|
+
break;
|
|
1674
|
+
}
|
|
1675
|
+
}
|
|
1676
|
+
if (!source) {
|
|
1677
|
+
throw new Error(`No data found for source project: "${sourceProjectId}" (also tried "${normalizedSource}")`);
|
|
1678
|
+
}
|
|
1679
|
+
// Move observations
|
|
1680
|
+
const obsResult = this.db
|
|
1681
|
+
.prepare('UPDATE observations SET project_id = ? WHERE project_id = ?')
|
|
1682
|
+
.run(target, source);
|
|
1683
|
+
// Move sessions
|
|
1684
|
+
const sessionResult = this.db
|
|
1685
|
+
.prepare('UPDATE sessions SET project_id = ? WHERE project_id = ?')
|
|
1686
|
+
.run(target, source);
|
|
1687
|
+
// Move journal entries
|
|
1688
|
+
const journalResult = this.db
|
|
1689
|
+
.prepare('UPDATE journal SET project_id = ? WHERE project_id = ?')
|
|
1690
|
+
.run(target, source);
|
|
1691
|
+
// Move prompts
|
|
1692
|
+
const promptsResult = this.db
|
|
1693
|
+
.prepare('UPDATE prompts SET project_id = ? WHERE project_id = ?')
|
|
1694
|
+
.run(target, source);
|
|
1695
|
+
// Re-index FTS for moved observations (delete old + insert new)
|
|
1696
|
+
this.db.prepare('DELETE FROM observations_fts WHERE project_id = ?').run(source);
|
|
1697
|
+
const movedObs = this.db
|
|
1698
|
+
.prepare('SELECT id, title, content, topic_key, project_id FROM observations WHERE project_id = ?')
|
|
1699
|
+
.all(target);
|
|
1700
|
+
for (const obs of movedObs) {
|
|
1701
|
+
this.db
|
|
1702
|
+
.prepare('INSERT OR REPLACE INTO observations_fts(rowid, title, content, topic_key, project_id) VALUES (?, ?, ?, ?, ?)')
|
|
1703
|
+
.run(obs.id, obs.title, obs.content, obs.topic_key ?? '', obs.project_id);
|
|
1704
|
+
}
|
|
1705
|
+
// Add source as alias of target in projects table
|
|
1706
|
+
try {
|
|
1707
|
+
const existing = this.db
|
|
1708
|
+
.prepare('SELECT aliases FROM projects WHERE name = ?')
|
|
1709
|
+
.get(target);
|
|
1710
|
+
if (existing) {
|
|
1711
|
+
const aliases = existing.aliases ? JSON.parse(existing.aliases) : [];
|
|
1712
|
+
if (!aliases.includes(source)) {
|
|
1713
|
+
aliases.push(source);
|
|
1714
|
+
this.db.prepare('UPDATE projects SET aliases = ? WHERE name = ?').run(JSON.stringify(aliases), target);
|
|
1715
|
+
}
|
|
1716
|
+
}
|
|
1717
|
+
}
|
|
1718
|
+
catch {
|
|
1719
|
+
// Non-critical — alias tracking is best-effort
|
|
1720
|
+
}
|
|
1721
|
+
return {
|
|
1722
|
+
observationsMoved: obsResult.changes,
|
|
1723
|
+
sessionsMoved: sessionResult.changes,
|
|
1724
|
+
journalMoved: journalResult.changes,
|
|
1725
|
+
promptsMoved: promptsResult.changes,
|
|
1726
|
+
};
|
|
1727
|
+
}
|
|
1728
|
+
/**
|
|
1729
|
+
* Normalize all project_id values in the database.
|
|
1730
|
+
* After normalization, merges projects that became identical.
|
|
1731
|
+
* Returns details of what was changed.
|
|
1732
|
+
*/
|
|
1733
|
+
normalizeAllProjectIds() {
|
|
1734
|
+
this.checkHealth();
|
|
1735
|
+
const { normalizeProjectId } = require('./ConfigManager');
|
|
1736
|
+
// Get all distinct project_ids
|
|
1737
|
+
const obsProjects = this.db
|
|
1738
|
+
.prepare('SELECT DISTINCT project_id FROM observations')
|
|
1739
|
+
.all();
|
|
1740
|
+
const sessionProjects = this.db
|
|
1741
|
+
.prepare('SELECT DISTINCT project_id FROM sessions')
|
|
1742
|
+
.all();
|
|
1743
|
+
const allProjects = new Set([
|
|
1744
|
+
...obsProjects.map((r) => r.project_id),
|
|
1745
|
+
...sessionProjects.map((r) => r.project_id),
|
|
1746
|
+
]);
|
|
1747
|
+
// Build normalization map: original → normalized
|
|
1748
|
+
const normalizationMap = new Map();
|
|
1749
|
+
for (const original of allProjects) {
|
|
1750
|
+
const normalized = normalizeProjectId(original);
|
|
1751
|
+
if (normalized !== original) {
|
|
1752
|
+
normalizationMap.set(original, normalized);
|
|
1753
|
+
}
|
|
1754
|
+
}
|
|
1755
|
+
// Apply normalizations
|
|
1756
|
+
let normalizedCount = 0;
|
|
1757
|
+
for (const [original, normalized] of normalizationMap) {
|
|
1758
|
+
// Update observations
|
|
1759
|
+
const obsResult = this.db
|
|
1760
|
+
.prepare('UPDATE observations SET project_id = ? WHERE project_id = ?')
|
|
1761
|
+
.run(normalized, original);
|
|
1762
|
+
normalizedCount += obsResult.changes;
|
|
1763
|
+
// Update sessions
|
|
1764
|
+
const sessionResult = this.db
|
|
1765
|
+
.prepare('UPDATE sessions SET project_id = ? WHERE project_id = ?')
|
|
1766
|
+
.run(normalized, original);
|
|
1767
|
+
normalizedCount += sessionResult.changes;
|
|
1768
|
+
// Update journal
|
|
1769
|
+
const journalResult = this.db
|
|
1770
|
+
.prepare('UPDATE journal SET project_id = ? WHERE project_id = ?')
|
|
1771
|
+
.run(normalized, original);
|
|
1772
|
+
normalizedCount += journalResult.changes;
|
|
1773
|
+
// Update prompts
|
|
1774
|
+
const promptsResult = this.db
|
|
1775
|
+
.prepare('UPDATE prompts SET project_id = ? WHERE project_id = ?')
|
|
1776
|
+
.run(normalized, original);
|
|
1777
|
+
normalizedCount += promptsResult.changes;
|
|
1778
|
+
}
|
|
1779
|
+
// After normalization, find duplicates (same normalized name) and merge
|
|
1780
|
+
const currentProjects = this.db
|
|
1781
|
+
.prepare('SELECT DISTINCT project_id FROM observations')
|
|
1782
|
+
.all();
|
|
1783
|
+
// Group by normalized name to find duplicates — but since we already normalized,
|
|
1784
|
+
// just check for remaining duplicates (shouldn't be any after normalization above)
|
|
1785
|
+
// The main purpose of mergeProject is for MANUAL consolidation of differently-named projects
|
|
1786
|
+
// that happen to represent the same real-world project
|
|
1787
|
+
// Re-index FTS completely for affected projects
|
|
1788
|
+
if (normalizedCount > 0) {
|
|
1789
|
+
// Delete and rebuild FTS for all changed projects
|
|
1790
|
+
for (const normalized of new Set(normalizationMap.values())) {
|
|
1791
|
+
this.db.prepare('DELETE FROM observations_fts WHERE project_id = ?').run(normalized);
|
|
1792
|
+
const obs = this.db
|
|
1793
|
+
.prepare('SELECT id, title, content, topic_key, project_id FROM observations WHERE project_id = ?')
|
|
1794
|
+
.all(normalized);
|
|
1795
|
+
for (const o of obs) {
|
|
1796
|
+
this.db
|
|
1797
|
+
.prepare('INSERT OR REPLACE INTO observations_fts(rowid, title, content, topic_key, project_id) VALUES (?, ?, ?, ?, ?)')
|
|
1798
|
+
.run(o.id, o.title, o.content, o.topic_key ?? '', o.project_id);
|
|
1799
|
+
}
|
|
1800
|
+
}
|
|
1801
|
+
}
|
|
1802
|
+
return {
|
|
1803
|
+
normalized: normalizedCount,
|
|
1804
|
+
merged: [],
|
|
1805
|
+
};
|
|
1806
|
+
}
|
|
1807
|
+
async getDashboardStats() {
|
|
1808
|
+
this.checkHealth();
|
|
1809
|
+
// Total counts
|
|
1810
|
+
const countRow = this.db
|
|
1811
|
+
.prepare(`
|
|
1812
|
+
SELECT
|
|
1813
|
+
COUNT(*) as total,
|
|
1814
|
+
SUM(CASE WHEN deleted_at IS NULL THEN 1 ELSE 0 END) as active,
|
|
1815
|
+
SUM(CASE WHEN deleted_at IS NOT NULL THEN 1 ELSE 0 END) as deleted
|
|
1816
|
+
FROM observations
|
|
1817
|
+
`)
|
|
1818
|
+
.get();
|
|
1819
|
+
// By type
|
|
1820
|
+
const typeRows = this.db
|
|
1821
|
+
.prepare(`
|
|
1822
|
+
SELECT type, COUNT(*) as count
|
|
1823
|
+
FROM observations
|
|
1824
|
+
WHERE deleted_at IS NULL
|
|
1825
|
+
GROUP BY type
|
|
1826
|
+
`)
|
|
1827
|
+
.all();
|
|
1828
|
+
const byType = { decision: 0, bug: 0, discovery: 0, note: 0, summary: 0, learning: 0, pattern: 0, architecture: 0, config: 0, preference: 0 };
|
|
1829
|
+
for (const tr of typeRows) {
|
|
1830
|
+
byType[tr.type] = tr.count;
|
|
1831
|
+
}
|
|
1832
|
+
// By project (active only)
|
|
1833
|
+
const projectRows = this.db
|
|
1834
|
+
.prepare(`
|
|
1835
|
+
SELECT project_id, COUNT(*) as count
|
|
1836
|
+
FROM observations
|
|
1837
|
+
WHERE deleted_at IS NULL
|
|
1838
|
+
GROUP BY project_id
|
|
1839
|
+
ORDER BY count DESC
|
|
1840
|
+
`)
|
|
1841
|
+
.all();
|
|
1842
|
+
const byProject = {};
|
|
1843
|
+
for (const pr of projectRows) {
|
|
1844
|
+
byProject[pr.project_id] = pr.count;
|
|
1845
|
+
}
|
|
1846
|
+
// Active sessions
|
|
1847
|
+
const sessionRow = this.db
|
|
1848
|
+
.prepare('SELECT COUNT(*) as count FROM sessions WHERE ended_at IS NULL')
|
|
1849
|
+
.get();
|
|
1850
|
+
// Recent 5 observations (active only)
|
|
1851
|
+
const recentRows = this.db
|
|
1852
|
+
.prepare(`
|
|
1853
|
+
SELECT * FROM observations
|
|
1854
|
+
WHERE deleted_at IS NULL
|
|
1855
|
+
ORDER BY created_at DESC, id DESC
|
|
1856
|
+
LIMIT 5
|
|
1857
|
+
`)
|
|
1858
|
+
.all();
|
|
1859
|
+
const recentObservations = recentRows.map((row) => this.mapObservation(row));
|
|
1860
|
+
return {
|
|
1861
|
+
totalObservations: countRow.total ?? 0,
|
|
1862
|
+
activeObservations: countRow.active ?? 0,
|
|
1863
|
+
deletedObservations: countRow.deleted ?? 0,
|
|
1864
|
+
byType,
|
|
1865
|
+
byProject,
|
|
1866
|
+
activeSessions: sessionRow.count,
|
|
1867
|
+
recentObservations,
|
|
1868
|
+
};
|
|
1869
|
+
}
|
|
1870
|
+
// ─── Import/Export ──────────────────────────────────────────
|
|
1871
|
+
async exportToJson(options) {
|
|
1872
|
+
this.checkHealth();
|
|
1873
|
+
const { projectId, includeSessions = false } = options || {};
|
|
1874
|
+
const searchResult = await this.search({ projectId, limit: 100000 });
|
|
1875
|
+
const observations = searchResult.observations.map((obs) => ({
|
|
1876
|
+
uuid: obs.uuid,
|
|
1877
|
+
title: obs.title,
|
|
1878
|
+
content: obs.content,
|
|
1879
|
+
type: obs.type,
|
|
1880
|
+
topicKey: obs.topicKey,
|
|
1881
|
+
projectId: obs.projectId,
|
|
1882
|
+
metadata: obs.metadata,
|
|
1883
|
+
createdAt: obs.createdAt.toISOString(),
|
|
1884
|
+
}));
|
|
1885
|
+
let sessions;
|
|
1886
|
+
if (includeSessions) {
|
|
1887
|
+
const sessionIds = [...new Set(searchResult.observations.map((o) => o.sessionId))];
|
|
1888
|
+
const sessionResults = await Promise.all(sessionIds.map((id) => this.getSession(id)));
|
|
1889
|
+
sessions = sessionResults
|
|
1890
|
+
.filter((s) => s !== null)
|
|
1891
|
+
.map((s) => ({
|
|
1892
|
+
uuid: s.uuid,
|
|
1893
|
+
projectId: s.projectId,
|
|
1894
|
+
startedAt: s.startedAt.toISOString(),
|
|
1895
|
+
endedAt: s.endedAt?.toISOString() || null,
|
|
1896
|
+
metadata: s.metadata,
|
|
1897
|
+
}));
|
|
1898
|
+
}
|
|
1899
|
+
return {
|
|
1900
|
+
version: '1.0',
|
|
1901
|
+
exportedAt: new Date().toISOString(),
|
|
1902
|
+
project: projectId,
|
|
1903
|
+
observations,
|
|
1904
|
+
...(includeSessions && sessions && { sessions }),
|
|
1905
|
+
};
|
|
1906
|
+
}
|
|
1907
|
+
async importFromJson(data, options) {
|
|
1908
|
+
this.checkHealth();
|
|
1909
|
+
const { conflictStrategy = 'skip', dryRun = false, projectId } = options || {};
|
|
1910
|
+
// Validate schema
|
|
1911
|
+
if (!data.version || !Array.isArray(data.observations)) {
|
|
1912
|
+
throw new Error('Invalid import data: missing version or observations array');
|
|
1913
|
+
}
|
|
1914
|
+
const validTypes = ['decision', 'bug', 'discovery', 'note', 'summary', 'learning', 'pattern', 'architecture', 'config', 'preference'];
|
|
1915
|
+
const targetProject = projectId || data.project || 'import';
|
|
1916
|
+
const result = {
|
|
1917
|
+
imported: 0,
|
|
1918
|
+
skipped: 0,
|
|
1919
|
+
overwritten: 0,
|
|
1920
|
+
failed: 0,
|
|
1921
|
+
errors: [],
|
|
1922
|
+
observations: [],
|
|
1923
|
+
};
|
|
1924
|
+
if (data.observations.length === 0) {
|
|
1925
|
+
return result;
|
|
1926
|
+
}
|
|
1927
|
+
// Create import session
|
|
1928
|
+
const session = await this.createSession({
|
|
1929
|
+
projectId: targetProject,
|
|
1930
|
+
endedAt: null,
|
|
1931
|
+
metadata: { type: 'import', source: 'mem_import', timestamp: new Date().toISOString() },
|
|
1932
|
+
});
|
|
1933
|
+
// Use db.transaction() for safe atomic import (#70)
|
|
1934
|
+
// Replaces manual BEGIN/COMMIT which was fragile — if createObservation's
|
|
1935
|
+
// withRetry() hit SQLITE_BUSY inside a manual transaction, the retry would
|
|
1936
|
+
// fail because the transaction was already in error state.
|
|
1937
|
+
const doImport = this.db.transaction(() => {
|
|
1938
|
+
for (const obs of data.observations) {
|
|
1939
|
+
// Validate required fields
|
|
1940
|
+
if (!obs.title || !obs.content || !obs.type) {
|
|
1941
|
+
result.errors.push(`Invalid observation: missing required fields (title, content, type)`);
|
|
1942
|
+
if (conflictStrategy === 'fail') {
|
|
1943
|
+
result.failed++;
|
|
1944
|
+
throw new Error('IMPORT_ABORT');
|
|
1945
|
+
}
|
|
1946
|
+
result.failed++;
|
|
1947
|
+
continue;
|
|
1948
|
+
}
|
|
1949
|
+
// Validate type
|
|
1950
|
+
if (!validTypes.includes(obs.type)) {
|
|
1951
|
+
result.errors.push(`Invalid type: "${obs.type}". Valid types: ${validTypes.join(', ')}`);
|
|
1952
|
+
if (conflictStrategy === 'fail') {
|
|
1953
|
+
result.failed++;
|
|
1954
|
+
throw new Error('IMPORT_ABORT');
|
|
1955
|
+
}
|
|
1956
|
+
result.failed++;
|
|
1957
|
+
continue;
|
|
1958
|
+
}
|
|
1959
|
+
// Check for duplicates by uuid
|
|
1960
|
+
if (obs.uuid) {
|
|
1961
|
+
const existing = this.findObservationByUuid(obs.uuid);
|
|
1962
|
+
if (existing) {
|
|
1963
|
+
if (conflictStrategy === 'skip') {
|
|
1964
|
+
result.skipped++;
|
|
1965
|
+
continue;
|
|
1966
|
+
}
|
|
1967
|
+
if (conflictStrategy === 'overwrite') {
|
|
1968
|
+
if (!dryRun) {
|
|
1969
|
+
// Direct UPDATE + FTS5 sync (avoids withRetry inside transaction)
|
|
1970
|
+
const fields = [];
|
|
1971
|
+
const values = [];
|
|
1972
|
+
if (obs.title !== undefined) {
|
|
1973
|
+
fields.push('title = ?');
|
|
1974
|
+
values.push(obs.title);
|
|
1975
|
+
}
|
|
1976
|
+
if (obs.content !== undefined) {
|
|
1977
|
+
fields.push('content = ?');
|
|
1978
|
+
values.push(obs.content);
|
|
1979
|
+
}
|
|
1980
|
+
if (obs.type !== undefined) {
|
|
1981
|
+
fields.push('type = ?');
|
|
1982
|
+
values.push(obs.type);
|
|
1983
|
+
}
|
|
1984
|
+
if (obs.topicKey !== undefined) {
|
|
1985
|
+
fields.push('topic_key = ?');
|
|
1986
|
+
values.push(obs.topicKey ?? null);
|
|
1987
|
+
}
|
|
1988
|
+
if (obs.metadata !== undefined) {
|
|
1989
|
+
fields.push('metadata = ?');
|
|
1990
|
+
values.push(this.serialize(obs.metadata || {}));
|
|
1991
|
+
}
|
|
1992
|
+
fields.push('revision_count = revision_count + 1');
|
|
1993
|
+
values.push(existing.id);
|
|
1994
|
+
this.db.prepare(`UPDATE observations SET ${fields.join(', ')} WHERE id = ?`).run(...values);
|
|
1995
|
+
// FTS5 sync — delete + re-insert
|
|
1996
|
+
this.db.prepare('DELETE FROM observations_fts WHERE rowid = ?').run(existing.id);
|
|
1997
|
+
const row = this.db
|
|
1998
|
+
.prepare('SELECT title, content, topic_key, project_id FROM observations WHERE id = ?')
|
|
1999
|
+
.get(existing.id);
|
|
2000
|
+
if (row) {
|
|
2001
|
+
this.db.prepare('INSERT INTO observations_fts(rowid, title, content, topic_key, project_id) VALUES (?, ?, ?, ?, ?)').run(existing.id, row.title, row.content, row.topic_key, row.project_id);
|
|
2002
|
+
}
|
|
2003
|
+
}
|
|
2004
|
+
result.overwritten++;
|
|
2005
|
+
continue;
|
|
2006
|
+
}
|
|
2007
|
+
if (conflictStrategy === 'fail') {
|
|
2008
|
+
result.errors.push(`Duplicate uuid: ${obs.uuid}`);
|
|
2009
|
+
result.failed++;
|
|
2010
|
+
throw new Error('IMPORT_ABORT');
|
|
2011
|
+
}
|
|
2012
|
+
}
|
|
2013
|
+
}
|
|
2014
|
+
if (!dryRun) {
|
|
2015
|
+
// Direct INSERT + FTS5 sync (avoids withRetry inside transaction)
|
|
2016
|
+
const uuid = obs.uuid || crypto.randomUUID();
|
|
2017
|
+
const metadata = this.serialize(obs.metadata || {});
|
|
2018
|
+
const createdAt = Date.now();
|
|
2019
|
+
const insertResult = this.db.prepare(`INSERT INTO observations (uuid, session_id, title, content, type, topic_key, project_id, created_at, deleted_at, metadata, scope) VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, 'project')`).run(uuid, session.id, obs.title, obs.content, obs.type, obs.topicKey ?? null, targetProject, createdAt, metadata);
|
|
2020
|
+
const insertId = typeof insertResult.lastInsertRowid === 'bigint'
|
|
2021
|
+
? Number(insertResult.lastInsertRowid)
|
|
2022
|
+
: insertResult.lastInsertRowid;
|
|
2023
|
+
// FTS5 sync
|
|
2024
|
+
this.db.prepare('INSERT INTO observations_fts(rowid, title, content, topic_key, project_id) VALUES (?, ?, ?, ?, ?)').run(insertId, obs.title, obs.content, obs.topicKey ?? '', targetProject);
|
|
2025
|
+
// Fetch created observation for result
|
|
2026
|
+
const importedRow = this.db.prepare('SELECT * FROM observations WHERE id = ?').get(insertId);
|
|
2027
|
+
if (importedRow) {
|
|
2028
|
+
result.observations.push(this.mapObservation(importedRow));
|
|
2029
|
+
}
|
|
2030
|
+
}
|
|
2031
|
+
result.imported++;
|
|
2032
|
+
}
|
|
2033
|
+
});
|
|
2034
|
+
try {
|
|
2035
|
+
doImport();
|
|
2036
|
+
}
|
|
2037
|
+
catch (error) {
|
|
2038
|
+
if (error instanceof Error && error.message === 'IMPORT_ABORT') {
|
|
2039
|
+
// Transaction auto-rolled back; return partial result with errors
|
|
2040
|
+
return result;
|
|
2041
|
+
}
|
|
2042
|
+
throw error;
|
|
2043
|
+
}
|
|
2044
|
+
return result;
|
|
2045
|
+
}
|
|
2046
|
+
findObservationByUuid(uuid) {
|
|
2047
|
+
const row = this.db.prepare('SELECT * FROM observations WHERE uuid = ?').get(uuid);
|
|
2048
|
+
if (!row)
|
|
2049
|
+
return null;
|
|
2050
|
+
return this.mapObservation(row);
|
|
2051
|
+
}
|
|
2052
|
+
async resetFull() {
|
|
2053
|
+
this.checkHealth();
|
|
2054
|
+
// Count before reset
|
|
2055
|
+
const countResult = this.db.prepare('SELECT COUNT(*) as count FROM observations').get();
|
|
2056
|
+
const deleted = countResult?.count || 0;
|
|
2057
|
+
// Drop everything (no observation triggers — FTS5 sync is application-level)
|
|
2058
|
+
this.db.exec('DROP TABLE IF EXISTS observations_fts');
|
|
2059
|
+
this.db.exec('DROP TRIGGER IF EXISTS journal_ai');
|
|
2060
|
+
this.db.exec('DROP TABLE IF EXISTS journal_fts');
|
|
2061
|
+
this.db.exec('DROP TABLE IF EXISTS journal_tags');
|
|
2062
|
+
this.db.exec('DROP TABLE IF EXISTS journal');
|
|
2063
|
+
this.db.exec('DROP TABLE IF EXISTS observations');
|
|
2064
|
+
this.db.exec('DROP TABLE IF EXISTS prompts');
|
|
2065
|
+
this.db.exec('DROP TABLE IF EXISTS sessions');
|
|
2066
|
+
this.db.exec('DROP TABLE IF EXISTS projects');
|
|
2067
|
+
// Recreate schema
|
|
2068
|
+
this.initializeDatabase();
|
|
2069
|
+
return { deleted };
|
|
2070
|
+
}
|
|
2071
|
+
async resetProject(projectId) {
|
|
2072
|
+
this.checkHealth();
|
|
2073
|
+
// Count observations to delete
|
|
2074
|
+
const obsCount = this.db
|
|
2075
|
+
.prepare('SELECT COUNT(*) as count FROM observations WHERE project_id = ?')
|
|
2076
|
+
.get(projectId);
|
|
2077
|
+
const promptCount = this.db
|
|
2078
|
+
.prepare('SELECT COUNT(*) as count FROM prompts WHERE project_id = ?')
|
|
2079
|
+
.get(projectId);
|
|
2080
|
+
const deleted = obsCount.count + promptCount.count;
|
|
2081
|
+
// Delete observations (FTS5 first, then main table) and prompts for this project
|
|
2082
|
+
// Application-level FTS5 sync (was trigger observations_ad)
|
|
2083
|
+
this.db.prepare('DELETE FROM observations_fts WHERE rowid IN (SELECT id FROM observations WHERE project_id = ?)').run(projectId);
|
|
2084
|
+
this.db.prepare('DELETE FROM observations WHERE project_id = ?').run(projectId);
|
|
2085
|
+
this.db.prepare('DELETE FROM prompts WHERE project_id = ?').run(projectId);
|
|
2086
|
+
// Clean up orphan sessions (no observations AND no prompts)
|
|
2087
|
+
const orphanResult = this.db.prepare(`
|
|
2088
|
+
DELETE FROM sessions WHERE id NOT IN (
|
|
2089
|
+
SELECT DISTINCT session_id FROM observations
|
|
2090
|
+
UNION
|
|
2091
|
+
SELECT DISTINCT session_id FROM prompts
|
|
2092
|
+
)
|
|
2093
|
+
`).run();
|
|
2094
|
+
const orphanSessions = typeof orphanResult.changes === 'bigint'
|
|
2095
|
+
? Number(orphanResult.changes)
|
|
2096
|
+
: orphanResult.changes;
|
|
2097
|
+
return { deleted, orphanSessions };
|
|
2098
|
+
}
|
|
2099
|
+
async countByProject(projectId) {
|
|
2100
|
+
this.checkHealth();
|
|
2101
|
+
const obsCount = this.db
|
|
2102
|
+
.prepare('SELECT COUNT(*) as count FROM observations WHERE project_id = ?')
|
|
2103
|
+
.get(projectId);
|
|
2104
|
+
const promptCount = this.db
|
|
2105
|
+
.prepare('SELECT COUNT(*) as count FROM prompts WHERE project_id = ?')
|
|
2106
|
+
.get(projectId);
|
|
2107
|
+
const sessionCount = this.db
|
|
2108
|
+
.prepare('SELECT COUNT(DISTINCT session_id) as count FROM observations WHERE project_id = ?')
|
|
2109
|
+
.get(projectId);
|
|
2110
|
+
return {
|
|
2111
|
+
observations: obsCount.count,
|
|
2112
|
+
prompts: promptCount.count,
|
|
2113
|
+
sessions: sessionCount.count,
|
|
2114
|
+
};
|
|
2115
|
+
}
|
|
2116
|
+
// ─── Journal (append-only evidence) ──────────────────────────
|
|
2117
|
+
async writeJournal(data) {
|
|
2118
|
+
this.checkHealth();
|
|
2119
|
+
const uuid = crypto.randomUUID();
|
|
2120
|
+
const createdAt = Date.now();
|
|
2121
|
+
const metadata = this.serialize(data.metadata || {});
|
|
2122
|
+
// If supersedes is set, validate and invalidate the old entry
|
|
2123
|
+
if (data.supersedes) {
|
|
2124
|
+
const existing = await this.readJournal(data.supersedes);
|
|
2125
|
+
if (!existing)
|
|
2126
|
+
throw new Error(`Journal entry ${data.supersedes} not found`);
|
|
2127
|
+
if (existing.invalidatedAt)
|
|
2128
|
+
throw new Error(`Journal entry ${data.supersedes} is already invalidated`);
|
|
2129
|
+
}
|
|
2130
|
+
const id = await this.withRetry(() => {
|
|
2131
|
+
const result = this.db
|
|
2132
|
+
.prepare(`INSERT INTO journal (uuid, project_id, session_id, title, body, model, provider, agent, superseded_by, invalidated_at, metadata, created_at)
|
|
2133
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, ?, ?)`)
|
|
2134
|
+
.run(uuid, data.projectId, data.sessionId ?? null, data.title, data.body, data.model ?? null, data.provider ?? null, data.agent ?? null, metadata, createdAt);
|
|
2135
|
+
return typeof result.lastInsertRowid === 'bigint'
|
|
2136
|
+
? Number(result.lastInsertRowid)
|
|
2137
|
+
: result.lastInsertRowid;
|
|
2138
|
+
});
|
|
2139
|
+
// Insert tags
|
|
2140
|
+
if (data.tags && data.tags.length > 0) {
|
|
2141
|
+
const insertTag = this.db.prepare('INSERT OR IGNORE INTO journal_tags (journal_id, tag) VALUES (?, ?)');
|
|
2142
|
+
for (const tag of data.tags) {
|
|
2143
|
+
insertTag.run(id, tag);
|
|
2144
|
+
}
|
|
2145
|
+
}
|
|
2146
|
+
// If supersedes, mark the old entry
|
|
2147
|
+
if (data.supersedes) {
|
|
2148
|
+
await this.withRetry(() => this.db
|
|
2149
|
+
.prepare('UPDATE journal SET superseded_by = ?, invalidated_at = ? WHERE id = ?')
|
|
2150
|
+
.run(id, createdAt, data.supersedes));
|
|
2151
|
+
}
|
|
2152
|
+
const entry = await this.readJournal(id);
|
|
2153
|
+
if (!entry)
|
|
2154
|
+
throw new Error('Failed to retrieve created journal entry');
|
|
2155
|
+
return entry;
|
|
2156
|
+
}
|
|
2157
|
+
async readJournal(id) {
|
|
2158
|
+
this.checkHealth();
|
|
2159
|
+
const row = this.db.prepare('SELECT * FROM journal WHERE id = ?').get(id);
|
|
2160
|
+
if (!row)
|
|
2161
|
+
return null;
|
|
2162
|
+
// Fetch tags
|
|
2163
|
+
const tagRows = this.db
|
|
2164
|
+
.prepare('SELECT tag FROM journal_tags WHERE journal_id = ?')
|
|
2165
|
+
.all(id);
|
|
2166
|
+
const tags = tagRows.map((t) => t.tag);
|
|
2167
|
+
return this.mapJournalEntry(row, tags);
|
|
2168
|
+
}
|
|
2169
|
+
async searchJournal(params) {
|
|
2170
|
+
this.checkHealth();
|
|
2171
|
+
const { query, tags, projectId, sessionId, dateFrom, dateTo, activeOnly = false, limit = 50, offset = 0, } = params;
|
|
2172
|
+
let sql;
|
|
2173
|
+
const values = [];
|
|
2174
|
+
const sanitizedQuery = query ? this.sanitizeFTS5Query(query) : '';
|
|
2175
|
+
if (sanitizedQuery) {
|
|
2176
|
+
// FTS5 search — qualify column names to avoid ambiguity with journal_fts
|
|
2177
|
+
sql =
|
|
2178
|
+
'SELECT journal.* FROM journal JOIN journal_fts ON journal.id = journal_fts.rowid WHERE journal_fts MATCH ?';
|
|
2179
|
+
values.push(sanitizedQuery);
|
|
2180
|
+
}
|
|
2181
|
+
else {
|
|
2182
|
+
sql = 'SELECT * FROM journal WHERE 1=1';
|
|
2183
|
+
}
|
|
2184
|
+
if (activeOnly) {
|
|
2185
|
+
sql += ' AND journal.invalidated_at IS NULL';
|
|
2186
|
+
}
|
|
2187
|
+
if (projectId) {
|
|
2188
|
+
sql += ' AND journal.project_id = ?';
|
|
2189
|
+
values.push(projectId);
|
|
2190
|
+
}
|
|
2191
|
+
if (sessionId) {
|
|
2192
|
+
sql += ' AND journal.session_id = ?';
|
|
2193
|
+
values.push(sessionId);
|
|
2194
|
+
}
|
|
2195
|
+
if (dateFrom) {
|
|
2196
|
+
sql += ' AND journal.created_at >= ?';
|
|
2197
|
+
values.push(dateFrom.getTime());
|
|
2198
|
+
}
|
|
2199
|
+
if (dateTo) {
|
|
2200
|
+
sql += ' AND journal.created_at <= ?';
|
|
2201
|
+
values.push(dateTo.getTime());
|
|
2202
|
+
}
|
|
2203
|
+
// Tag filtering — if tags specified, intersect with journal_tags
|
|
2204
|
+
if (tags && tags.length > 0) {
|
|
2205
|
+
// Find journal entries that have ALL specified tags
|
|
2206
|
+
const tagPlaceholders = tags.map(() => '?').join(',');
|
|
2207
|
+
const tagSubquery = `
|
|
2208
|
+
AND journal.id IN (
|
|
2209
|
+
SELECT jt.journal_id FROM journal_tags jt
|
|
2210
|
+
WHERE jt.tag IN (${tagPlaceholders})
|
|
2211
|
+
GROUP BY jt.journal_id
|
|
2212
|
+
HAVING COUNT(DISTINCT jt.tag) = ${tags.length}
|
|
2213
|
+
)
|
|
2214
|
+
`;
|
|
2215
|
+
sql += tagSubquery;
|
|
2216
|
+
values.push(...tags);
|
|
2217
|
+
}
|
|
2218
|
+
// Count
|
|
2219
|
+
const countSql = sql.replace(/SELECT.*?FROM/, 'SELECT COUNT(*) as count FROM');
|
|
2220
|
+
const countResult = this.db.prepare(countSql).get(...values);
|
|
2221
|
+
const total = countResult?.count ?? 0;
|
|
2222
|
+
// Fetch
|
|
2223
|
+
sql += ' ORDER BY journal.created_at DESC, journal.id DESC LIMIT ? OFFSET ?';
|
|
2224
|
+
values.push(limit, offset);
|
|
2225
|
+
const rows = this.db.prepare(sql).all(...values);
|
|
2226
|
+
// Fetch tags for each entry
|
|
2227
|
+
const entries = [];
|
|
2228
|
+
for (const row of rows) {
|
|
2229
|
+
const entryId = row.id;
|
|
2230
|
+
const tagRows = this.db
|
|
2231
|
+
.prepare('SELECT tag FROM journal_tags WHERE journal_id = ?')
|
|
2232
|
+
.all(entryId);
|
|
2233
|
+
entries.push(this.mapJournalEntry(row, tagRows.map((t) => t.tag)));
|
|
2234
|
+
}
|
|
2235
|
+
return { entries, total };
|
|
2236
|
+
}
|
|
2237
|
+
async invalidateJournal(id, supersededById) {
|
|
2238
|
+
this.checkHealth();
|
|
2239
|
+
const entry = await this.readJournal(id);
|
|
2240
|
+
if (!entry)
|
|
2241
|
+
throw new Error(`Journal entry ${id} not found`);
|
|
2242
|
+
if (entry.invalidatedAt)
|
|
2243
|
+
throw new Error(`Journal entry ${id} is already invalidated`);
|
|
2244
|
+
// Validate the superseding entry exists
|
|
2245
|
+
const superseding = await this.readJournal(supersededById);
|
|
2246
|
+
if (!superseding)
|
|
2247
|
+
throw new Error(`Superseding journal entry ${supersededById} not found`);
|
|
2248
|
+
const now = Date.now();
|
|
2249
|
+
await this.withRetry(() => this.db
|
|
2250
|
+
.prepare('UPDATE journal SET superseded_by = ?, invalidated_at = ? WHERE id = ?')
|
|
2251
|
+
.run(supersededById, now, id));
|
|
2252
|
+
}
|
|
2253
|
+
mapJournalEntry(row, tags) {
|
|
2254
|
+
const r = row;
|
|
2255
|
+
return {
|
|
2256
|
+
id: r.id,
|
|
2257
|
+
uuid: r.uuid,
|
|
2258
|
+
projectId: r.project_id,
|
|
2259
|
+
sessionId: r.session_id,
|
|
2260
|
+
title: r.title,
|
|
2261
|
+
body: r.body,
|
|
2262
|
+
tags,
|
|
2263
|
+
model: r.model,
|
|
2264
|
+
provider: r.provider,
|
|
2265
|
+
agent: r.agent,
|
|
2266
|
+
supersededBy: r.superseded_by,
|
|
2267
|
+
invalidatedAt: r.invalidated_at ? new Date(r.invalidated_at) : null,
|
|
2268
|
+
metadata: this.deserialize(r.metadata),
|
|
2269
|
+
createdAt: new Date(r.created_at),
|
|
2270
|
+
};
|
|
2271
|
+
}
|
|
2272
|
+
// ─── Full Export/Import (v2.0) ────────────────────────────────
|
|
2273
|
+
async listPrompts(params) {
|
|
2274
|
+
this.checkHealth();
|
|
2275
|
+
const { projectId, sessionId, limit = 50, offset = 0 } = params || {};
|
|
2276
|
+
let whereClause = ' WHERE 1=1';
|
|
2277
|
+
const values = [];
|
|
2278
|
+
if (projectId) {
|
|
2279
|
+
whereClause += ' AND project_id = ?';
|
|
2280
|
+
values.push(projectId);
|
|
2281
|
+
}
|
|
2282
|
+
if (sessionId) {
|
|
2283
|
+
whereClause += ' AND session_id = ?';
|
|
2284
|
+
values.push(sessionId);
|
|
2285
|
+
}
|
|
2286
|
+
const countResult = this.db.prepare(`SELECT COUNT(*) as count FROM prompts${whereClause}`).get(...values);
|
|
2287
|
+
const total = countResult?.count ?? 0;
|
|
2288
|
+
const rows = this.db.prepare(`SELECT * FROM prompts${whereClause} ORDER BY created_at ASC LIMIT ? OFFSET ?`).all(...values, limit, offset);
|
|
2289
|
+
const prompts = rows.map((row) => this.mapPrompt(row));
|
|
2290
|
+
return { prompts, total };
|
|
2291
|
+
}
|
|
2292
|
+
async exportProject(projectId) {
|
|
2293
|
+
this.checkHealth();
|
|
2294
|
+
// Fetch all data in parallel using Promise.all
|
|
2295
|
+
const [observationsResult, sessionsResult, journalResult, promptsResult,] = await Promise.all([
|
|
2296
|
+
this.search({ projectId, limit: 100000, includeDeleted: true }),
|
|
2297
|
+
this.listSessions({ projectId, limit: 100000 }),
|
|
2298
|
+
this.searchJournal({ projectId, limit: 100000 }),
|
|
2299
|
+
this.listPrompts({ projectId, limit: 100000 }),
|
|
2300
|
+
]);
|
|
2301
|
+
// Fetch project data
|
|
2302
|
+
const projectRow = this.db
|
|
2303
|
+
.prepare('SELECT * FROM projects WHERE name = ?')
|
|
2304
|
+
.get(projectId);
|
|
2305
|
+
const projects = projectRow
|
|
2306
|
+
? [{
|
|
2307
|
+
name: projectRow.name,
|
|
2308
|
+
createdAt: new Date(projectRow.created_at).toISOString(),
|
|
2309
|
+
metadata: this.deserialize(projectRow.metadata),
|
|
2310
|
+
}]
|
|
2311
|
+
: [{
|
|
2312
|
+
name: projectId,
|
|
2313
|
+
createdAt: new Date().toISOString(),
|
|
2314
|
+
metadata: {},
|
|
2315
|
+
}];
|
|
2316
|
+
const sessions = sessionsResult.sessions.map((s) => ({
|
|
2317
|
+
uuid: s.uuid,
|
|
2318
|
+
projectId: s.projectId,
|
|
2319
|
+
startedAt: s.startedAt.toISOString(),
|
|
2320
|
+
endedAt: s.endedAt ? s.endedAt.toISOString() : null,
|
|
2321
|
+
metadata: s.metadata,
|
|
2322
|
+
}));
|
|
2323
|
+
const observations = observationsResult.observations.map((o) => ({
|
|
2324
|
+
uuid: o.uuid,
|
|
2325
|
+
title: o.title,
|
|
2326
|
+
content: o.content,
|
|
2327
|
+
type: o.type,
|
|
2328
|
+
topicKey: o.topicKey,
|
|
2329
|
+
projectId: o.projectId,
|
|
2330
|
+
scope: o.scope,
|
|
2331
|
+
pinned: o.pinned,
|
|
2332
|
+
readOnly: o.readOnly,
|
|
2333
|
+
revisionCount: o.revisionCount,
|
|
2334
|
+
createdAt: o.createdAt.toISOString(),
|
|
2335
|
+
deletedAt: o.deletedAt ? o.deletedAt.toISOString() : null,
|
|
2336
|
+
metadata: o.metadata,
|
|
2337
|
+
}));
|
|
2338
|
+
const prompts = promptsResult.prompts.map((p) => ({
|
|
2339
|
+
uuid: p.uuid,
|
|
2340
|
+
content: p.content,
|
|
2341
|
+
projectId: p.projectId,
|
|
2342
|
+
createdAt: p.createdAt.toISOString(),
|
|
2343
|
+
metadata: p.metadata,
|
|
2344
|
+
}));
|
|
2345
|
+
const journal = journalResult.entries.map((j) => ({
|
|
2346
|
+
uuid: j.uuid,
|
|
2347
|
+
title: j.title,
|
|
2348
|
+
body: j.body,
|
|
2349
|
+
tags: j.tags,
|
|
2350
|
+
projectId: j.projectId,
|
|
2351
|
+
model: j.model,
|
|
2352
|
+
provider: j.provider,
|
|
2353
|
+
agent: j.agent,
|
|
2354
|
+
supersededByUuid: null, // Resolved below
|
|
2355
|
+
invalidatedAt: j.invalidatedAt ? j.invalidatedAt.toISOString() : null,
|
|
2356
|
+
metadata: j.metadata,
|
|
2357
|
+
createdAt: j.createdAt.toISOString(),
|
|
2358
|
+
}));
|
|
2359
|
+
// Resolve journal supersedes references from ID to UUID
|
|
2360
|
+
const journalIdToUuid = new Map();
|
|
2361
|
+
for (const entry of journalResult.entries) {
|
|
2362
|
+
journalIdToUuid.set(entry.id, entry.uuid);
|
|
2363
|
+
}
|
|
2364
|
+
for (let i = 0; i < journal.length; i++) {
|
|
2365
|
+
const supersededById = journalResult.entries[i].supersededBy;
|
|
2366
|
+
if (supersededById !== null) {
|
|
2367
|
+
journal[i].supersededByUuid = journalIdToUuid.get(supersededById) ?? null;
|
|
2368
|
+
}
|
|
2369
|
+
}
|
|
2370
|
+
return {
|
|
2371
|
+
version: '2.0',
|
|
2372
|
+
exportedAt: new Date().toISOString(),
|
|
2373
|
+
source: {
|
|
2374
|
+
project: projectId,
|
|
2375
|
+
allProjects: false,
|
|
2376
|
+
},
|
|
2377
|
+
stats: {
|
|
2378
|
+
totalProjects: projects.length,
|
|
2379
|
+
totalSessions: sessions.length,
|
|
2380
|
+
totalObservations: observations.length,
|
|
2381
|
+
totalPrompts: prompts.length,
|
|
2382
|
+
totalJournalEntries: journal.length,
|
|
2383
|
+
},
|
|
2384
|
+
projects,
|
|
2385
|
+
sessions,
|
|
2386
|
+
observations,
|
|
2387
|
+
prompts,
|
|
2388
|
+
journal,
|
|
2389
|
+
};
|
|
2390
|
+
}
|
|
2391
|
+
async importProject(data, options) {
|
|
2392
|
+
this.checkHealth();
|
|
2393
|
+
const { conflictStrategy = 'skip', dryRun = false } = options || {};
|
|
2394
|
+
const result = {
|
|
2395
|
+
imported: { projects: 0, sessions: 0, observations: 0, prompts: 0, journalEntries: 0 },
|
|
2396
|
+
skipped: { observations: 0, sessions: 0, journalEntries: 0 },
|
|
2397
|
+
overwritten: { observations: 0 },
|
|
2398
|
+
failed: 0,
|
|
2399
|
+
errors: [],
|
|
2400
|
+
};
|
|
2401
|
+
if (!data.version) {
|
|
2402
|
+
throw new Error('Invalid import data: missing version field');
|
|
2403
|
+
}
|
|
2404
|
+
// Determine target project
|
|
2405
|
+
const targetProject = options?.projectId || data.source?.project || 'import';
|
|
2406
|
+
// Validate observations types
|
|
2407
|
+
const validTypes = ['decision', 'bug', 'discovery', 'note', 'summary', 'learning', 'pattern', 'architecture', 'config', 'preference'];
|
|
2408
|
+
// UUID → new ID mappings for referential integrity
|
|
2409
|
+
const sessionUuidToId = new Map();
|
|
2410
|
+
// Create import session
|
|
2411
|
+
const importSession = await this.createSession({
|
|
2412
|
+
projectId: targetProject,
|
|
2413
|
+
endedAt: null,
|
|
2414
|
+
metadata: { type: 'full-import', source: 'web-ui', timestamp: new Date().toISOString() },
|
|
2415
|
+
});
|
|
2416
|
+
const doImport = this.db.transaction(() => {
|
|
2417
|
+
// 1. Import projects (register if not exists)
|
|
2418
|
+
if (data.projects && data.projects.length > 0) {
|
|
2419
|
+
for (const proj of data.projects) {
|
|
2420
|
+
if (!dryRun) {
|
|
2421
|
+
this.db.prepare('INSERT OR IGNORE INTO projects (name, created_at, metadata) VALUES (?, ?, ?)').run(proj.name, Date.now(), this.serialize(proj.metadata || {}));
|
|
2422
|
+
}
|
|
2423
|
+
result.imported.projects++;
|
|
2424
|
+
}
|
|
2425
|
+
}
|
|
2426
|
+
// 2. Import sessions (create new, map UUID → new ID)
|
|
2427
|
+
if (data.sessions && data.sessions.length > 0) {
|
|
2428
|
+
for (const sess of data.sessions) {
|
|
2429
|
+
// Check if session with this UUID already exists
|
|
2430
|
+
const existing = this.db
|
|
2431
|
+
.prepare('SELECT id FROM sessions WHERE uuid = ?')
|
|
2432
|
+
.get(sess.uuid);
|
|
2433
|
+
if (existing) {
|
|
2434
|
+
sessionUuidToId.set(sess.uuid, existing.id);
|
|
2435
|
+
result.skipped.sessions++;
|
|
2436
|
+
continue;
|
|
2437
|
+
}
|
|
2438
|
+
if (!dryRun) {
|
|
2439
|
+
const insertResult = this.db.prepare('INSERT INTO sessions (uuid, project_id, started_at, ended_at, metadata) VALUES (?, ?, ?, ?, ?)').run(sess.uuid, targetProject, new Date(sess.startedAt).getTime(), sess.endedAt ? new Date(sess.endedAt).getTime() : null, this.serialize(sess.metadata || {}));
|
|
2440
|
+
const newId = typeof insertResult.lastInsertRowid === 'bigint'
|
|
2441
|
+
? Number(insertResult.lastInsertRowid)
|
|
2442
|
+
: insertResult.lastInsertRowid;
|
|
2443
|
+
sessionUuidToId.set(sess.uuid, newId);
|
|
2444
|
+
}
|
|
2445
|
+
else {
|
|
2446
|
+
// For dry run, assign placeholder IDs
|
|
2447
|
+
sessionUuidToId.set(sess.uuid, -(result.imported.sessions + 1));
|
|
2448
|
+
}
|
|
2449
|
+
result.imported.sessions++;
|
|
2450
|
+
}
|
|
2451
|
+
}
|
|
2452
|
+
// 3. Import observations
|
|
2453
|
+
if (data.observations && data.observations.length > 0) {
|
|
2454
|
+
for (const obs of data.observations) {
|
|
2455
|
+
if (!obs.title || !obs.content || !obs.type) {
|
|
2456
|
+
result.errors.push(`Invalid observation: missing required fields`);
|
|
2457
|
+
result.failed++;
|
|
2458
|
+
continue;
|
|
2459
|
+
}
|
|
2460
|
+
if (!validTypes.includes(obs.type)) {
|
|
2461
|
+
result.errors.push(`Invalid type: "${obs.type}"`);
|
|
2462
|
+
result.failed++;
|
|
2463
|
+
continue;
|
|
2464
|
+
}
|
|
2465
|
+
// Check for duplicate by UUID
|
|
2466
|
+
const existing = obs.uuid
|
|
2467
|
+
? this.db.prepare('SELECT id FROM observations WHERE uuid = ?').get(obs.uuid)
|
|
2468
|
+
: undefined;
|
|
2469
|
+
if (existing) {
|
|
2470
|
+
if (conflictStrategy === 'skip') {
|
|
2471
|
+
result.skipped.observations++;
|
|
2472
|
+
continue;
|
|
2473
|
+
}
|
|
2474
|
+
if (conflictStrategy === 'overwrite' && !dryRun) {
|
|
2475
|
+
this.db.prepare(`UPDATE observations SET title = ?, content = ?, type = ?, topic_key = ?, metadata = ?, scope = ?, pinned = ?, read_only = ?, revision_count = revision_count + 1 WHERE id = ?`).run(obs.title, obs.content, obs.type, obs.topicKey ?? null, this.serialize(obs.metadata || {}), obs.scope || 'project', obs.pinned ? 1 : 0, obs.readOnly ? 1 : 0, existing.id);
|
|
2476
|
+
// FTS5 sync
|
|
2477
|
+
this.db.prepare('DELETE FROM observations_fts WHERE rowid = ?').run(existing.id);
|
|
2478
|
+
const row = this.db
|
|
2479
|
+
.prepare('SELECT title, content, topic_key, project_id FROM observations WHERE id = ?')
|
|
2480
|
+
.get(existing.id);
|
|
2481
|
+
if (row) {
|
|
2482
|
+
this.db.prepare('INSERT INTO observations_fts(rowid, title, content, topic_key, project_id) VALUES (?, ?, ?, ?, ?)').run(existing.id, row.title, row.content, row.topic_key, row.project_id);
|
|
2483
|
+
}
|
|
2484
|
+
}
|
|
2485
|
+
result.overwritten.observations++;
|
|
2486
|
+
continue;
|
|
2487
|
+
}
|
|
2488
|
+
if (!dryRun) {
|
|
2489
|
+
const uuid = obs.uuid || crypto.randomUUID();
|
|
2490
|
+
const createdAt = obs.createdAt ? new Date(obs.createdAt).getTime() : Date.now();
|
|
2491
|
+
const sessionId = importSession.id;
|
|
2492
|
+
const metadata = this.serialize(obs.metadata || {});
|
|
2493
|
+
const insertResult = this.db.prepare(`INSERT INTO observations (uuid, session_id, title, content, type, topic_key, project_id, created_at, deleted_at, metadata, scope, pinned, read_only, revision_count)
|
|
2494
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(uuid, sessionId, obs.title, obs.content, obs.type, obs.topicKey ?? null, targetProject, createdAt, obs.deletedAt ? new Date(obs.deletedAt).getTime() : null, metadata, obs.scope || 'project', obs.pinned ? 1 : 0, obs.readOnly ? 1 : 0, obs.revisionCount ?? 0);
|
|
2495
|
+
const insertId = typeof insertResult.lastInsertRowid === 'bigint'
|
|
2496
|
+
? Number(insertResult.lastInsertRowid)
|
|
2497
|
+
: insertResult.lastInsertRowid;
|
|
2498
|
+
// FTS5 sync
|
|
2499
|
+
this.db.prepare('INSERT INTO observations_fts(rowid, title, content, topic_key, project_id) VALUES (?, ?, ?, ?, ?)').run(insertId, obs.title, obs.content, obs.topicKey ?? '', targetProject);
|
|
2500
|
+
}
|
|
2501
|
+
result.imported.observations++;
|
|
2502
|
+
}
|
|
2503
|
+
}
|
|
2504
|
+
// 4. Import prompts
|
|
2505
|
+
if (data.prompts && data.prompts.length > 0) {
|
|
2506
|
+
for (const prompt of data.prompts) {
|
|
2507
|
+
// Skip if UUID already exists
|
|
2508
|
+
if (prompt.uuid) {
|
|
2509
|
+
const existingPrompt = this.db
|
|
2510
|
+
.prepare('SELECT id FROM prompts WHERE uuid = ?')
|
|
2511
|
+
.get(prompt.uuid);
|
|
2512
|
+
if (existingPrompt)
|
|
2513
|
+
continue;
|
|
2514
|
+
}
|
|
2515
|
+
if (!dryRun) {
|
|
2516
|
+
const uuid = prompt.uuid || crypto.randomUUID();
|
|
2517
|
+
const createdAt = prompt.createdAt ? new Date(prompt.createdAt).getTime() : Date.now();
|
|
2518
|
+
this.db.prepare('INSERT INTO prompts (uuid, session_id, content, project_id, created_at, metadata) VALUES (?, ?, ?, ?, ?, ?)').run(uuid, importSession.id, prompt.content, targetProject, createdAt, this.serialize(prompt.metadata || {}));
|
|
2519
|
+
}
|
|
2520
|
+
result.imported.prompts++;
|
|
2521
|
+
}
|
|
2522
|
+
}
|
|
2523
|
+
// 5. Import journal entries (with tag support and supersedes UUID mapping)
|
|
2524
|
+
if (data.journal && data.journal.length > 0) {
|
|
2525
|
+
// Two-pass: first insert all entries, then resolve supersedes
|
|
2526
|
+
const journalUuidToId = new Map();
|
|
2527
|
+
for (const entry of data.journal) {
|
|
2528
|
+
// Check if UUID already exists
|
|
2529
|
+
const existingEntry = entry.uuid
|
|
2530
|
+
? this.db.prepare('SELECT id FROM journal WHERE uuid = ?').get(entry.uuid)
|
|
2531
|
+
: undefined;
|
|
2532
|
+
if (existingEntry) {
|
|
2533
|
+
journalUuidToId.set(entry.uuid, existingEntry.id);
|
|
2534
|
+
result.skipped.journalEntries++;
|
|
2535
|
+
continue;
|
|
2536
|
+
}
|
|
2537
|
+
if (!dryRun) {
|
|
2538
|
+
const uuid = entry.uuid || crypto.randomUUID();
|
|
2539
|
+
const createdAt = entry.createdAt ? new Date(entry.createdAt).getTime() : Date.now();
|
|
2540
|
+
// Find session by UUID if available
|
|
2541
|
+
let sessionId = null;
|
|
2542
|
+
// We don't have session UUID reference in journal import, use import session
|
|
2543
|
+
const insertResult = this.db.prepare(`INSERT INTO journal (uuid, project_id, session_id, title, body, model, provider, agent, superseded_by, invalidated_at, metadata, created_at)
|
|
2544
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?, ?)`).run(uuid, targetProject, sessionId, entry.title, entry.body, entry.model ?? null, entry.provider ?? null, entry.agent ?? null, entry.invalidatedAt ? new Date(entry.invalidatedAt).getTime() : null, this.serialize(entry.metadata || {}), createdAt);
|
|
2545
|
+
const newId = typeof insertResult.lastInsertRowid === 'bigint'
|
|
2546
|
+
? Number(insertResult.lastInsertRowid)
|
|
2547
|
+
: insertResult.lastInsertRowid;
|
|
2548
|
+
journalUuidToId.set(uuid, newId);
|
|
2549
|
+
// Insert tags
|
|
2550
|
+
if (entry.tags && entry.tags.length > 0) {
|
|
2551
|
+
for (const tag of entry.tags) {
|
|
2552
|
+
this.db.prepare('INSERT INTO journal_tags (journal_id, tag) VALUES (?, ?)').run(newId, tag);
|
|
2553
|
+
}
|
|
2554
|
+
}
|
|
2555
|
+
}
|
|
2556
|
+
result.imported.journalEntries++;
|
|
2557
|
+
}
|
|
2558
|
+
// Second pass: resolve supersedes references
|
|
2559
|
+
if (!dryRun) {
|
|
2560
|
+
for (const entry of data.journal) {
|
|
2561
|
+
if (entry.supersededByUuid && entry.uuid) {
|
|
2562
|
+
const entryId = journalUuidToId.get(entry.uuid);
|
|
2563
|
+
const supersededById = journalUuidToId.get(entry.supersededByUuid);
|
|
2564
|
+
if (entryId && supersededById) {
|
|
2565
|
+
this.db.prepare('UPDATE journal SET superseded_by = ? WHERE id = ?').run(supersededById, entryId);
|
|
2566
|
+
}
|
|
2567
|
+
}
|
|
2568
|
+
}
|
|
2569
|
+
}
|
|
2570
|
+
}
|
|
2571
|
+
});
|
|
2572
|
+
try {
|
|
2573
|
+
doImport();
|
|
2574
|
+
}
|
|
2575
|
+
catch (error) {
|
|
2576
|
+
if (error instanceof Error) {
|
|
2577
|
+
result.errors.push(`Transaction failed: ${error.message}`);
|
|
2578
|
+
}
|
|
2579
|
+
result.failed++;
|
|
2580
|
+
}
|
|
2581
|
+
return result;
|
|
2582
|
+
}
|
|
368
2583
|
close() {
|
|
369
2584
|
this.db.close();
|
|
370
2585
|
}
|