@equationalapplications/expo-llm-wiki 2.3.0 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -1,1385 +1,15 @@
1
- // src/db/schema.ts
2
- async function setupDatabase(db, prefix) {
3
- await db.execAsync(`
4
- CREATE TABLE IF NOT EXISTS ${prefix}entries (
5
- id TEXT PRIMARY KEY,
6
- entity_id TEXT NOT NULL,
7
- title TEXT NOT NULL,
8
- body TEXT NOT NULL,
9
- tags TEXT NOT NULL DEFAULT '[]',
10
- confidence TEXT NOT NULL DEFAULT 'inferred',
11
- source_type TEXT NOT NULL DEFAULT 'agent_inferred',
12
- source_hash TEXT,
13
- source_ref TEXT,
14
- created_at INTEGER NOT NULL,
15
- updated_at INTEGER NOT NULL,
16
- last_accessed_at INTEGER,
17
- access_count INTEGER NOT NULL DEFAULT 0,
18
- deleted_at INTEGER
19
- );
20
-
21
- CREATE INDEX IF NOT EXISTS ${prefix}entries_entity_idx ON ${prefix}entries(entity_id);
22
- CREATE INDEX IF NOT EXISTS ${prefix}entries_source_ref_idx ON ${prefix}entries(entity_id, source_ref);
23
- CREATE INDEX IF NOT EXISTS ${prefix}entries_source_hash_idx ON ${prefix}entries(entity_id, source_hash) WHERE source_hash IS NOT NULL;
24
- CREATE INDEX IF NOT EXISTS ${prefix}entries_updated_idx ON ${prefix}entries(updated_at DESC);
25
-
26
- -- FTS5 Virtual Table for full-text search
27
- CREATE VIRTUAL TABLE IF NOT EXISTS ${prefix}entries_fts USING fts5(
28
- title,
29
- body,
30
- tags,
31
- content='${prefix}entries',
32
- content_rowid='rowid',
33
- tokenize='porter unicode61'
34
- );
35
-
36
- -- Triggers to keep FTS5 in sync with entries
37
- CREATE TRIGGER IF NOT EXISTS ${prefix}entries_ai AFTER INSERT ON ${prefix}entries BEGIN
38
- INSERT INTO ${prefix}entries_fts(rowid, title, body, tags)
39
- VALUES (new.rowid, new.title, new.body, new.tags);
40
- END;
41
-
42
- CREATE TRIGGER IF NOT EXISTS ${prefix}entries_ad AFTER DELETE ON ${prefix}entries BEGIN
43
- INSERT INTO ${prefix}entries_fts(${prefix}entries_fts, rowid, title, body, tags)
44
- VALUES ('delete', old.rowid, old.title, old.body, old.tags);
45
- END;
46
-
47
- CREATE TRIGGER IF NOT EXISTS ${prefix}entries_au AFTER UPDATE ON ${prefix}entries BEGIN
48
- INSERT INTO ${prefix}entries_fts(${prefix}entries_fts, rowid, title, body, tags)
49
- VALUES ('delete', old.rowid, old.title, old.body, old.tags);
50
- INSERT INTO ${prefix}entries_fts(rowid, title, body, tags)
51
- VALUES (new.rowid, new.title, new.body, new.tags);
52
- END;
53
-
54
- CREATE TABLE IF NOT EXISTS ${prefix}tasks (
55
- id TEXT PRIMARY KEY,
56
- entity_id TEXT NOT NULL,
57
- description TEXT NOT NULL,
58
- status TEXT NOT NULL DEFAULT 'pending',
59
- priority INTEGER NOT NULL DEFAULT 0,
60
- created_at INTEGER NOT NULL,
61
- updated_at INTEGER NOT NULL,
62
- resolved_at INTEGER,
63
- deleted_at INTEGER
64
- );
65
-
66
- CREATE INDEX IF NOT EXISTS ${prefix}tasks_entity_idx ON ${prefix}tasks(entity_id, status);
67
-
68
- CREATE TABLE IF NOT EXISTS ${prefix}events (
69
- id TEXT PRIMARY KEY,
70
- entity_id TEXT NOT NULL,
71
- event_type TEXT NOT NULL,
72
- summary TEXT NOT NULL,
73
- related_entry_id TEXT,
74
- created_at INTEGER NOT NULL
75
- );
76
-
77
- CREATE INDEX IF NOT EXISTS ${prefix}events_entity_idx ON ${prefix}events(entity_id, created_at DESC);
78
-
79
- CREATE TABLE IF NOT EXISTS ${prefix}checkpoints (
80
- entity_id TEXT PRIMARY KEY,
81
- heal_checkpoint INTEGER NOT NULL DEFAULT 0,
82
- memory_checkpoint INTEGER NOT NULL DEFAULT 0
83
- );
84
-
85
- CREATE TABLE IF NOT EXISTS ${prefix}meta (
86
- key TEXT PRIMARY KEY,
87
- value TEXT NOT NULL
88
- );
89
- `);
90
- }
91
-
92
- // src/db/migrations.ts
93
- var MIGRATIONS = [
94
- {
95
- version: 1,
96
- description: "Rebuild FTS5 with porter unicode61 tokenizer",
97
- run: async (db, prefix) => {
98
- await db.withTransactionAsync(async () => {
99
- await db.execAsync(`
100
- DROP TRIGGER IF EXISTS ${prefix}entries_ai;
101
- DROP TRIGGER IF EXISTS ${prefix}entries_ad;
102
- DROP TRIGGER IF EXISTS ${prefix}entries_au;
103
- DROP TABLE IF EXISTS ${prefix}entries_fts;
104
- CREATE VIRTUAL TABLE ${prefix}entries_fts USING fts5(
105
- title,
106
- body,
107
- tags,
108
- content='${prefix}entries',
109
- content_rowid='rowid',
110
- tokenize='porter unicode61'
111
- );
112
- INSERT INTO ${prefix}entries_fts(rowid, title, body, tags)
113
- SELECT rowid, title, body, tags FROM ${prefix}entries;
114
- CREATE TRIGGER ${prefix}entries_ai AFTER INSERT ON ${prefix}entries BEGIN
115
- INSERT INTO ${prefix}entries_fts(rowid, title, body, tags)
116
- VALUES (new.rowid, new.title, new.body, new.tags);
117
- END;
118
- CREATE TRIGGER ${prefix}entries_ad AFTER DELETE ON ${prefix}entries BEGIN
119
- INSERT INTO ${prefix}entries_fts(${prefix}entries_fts, rowid, title, body, tags)
120
- VALUES ('delete', old.rowid, old.title, old.body, old.tags);
121
- END;
122
- CREATE TRIGGER ${prefix}entries_au AFTER UPDATE ON ${prefix}entries BEGIN
123
- INSERT INTO ${prefix}entries_fts(${prefix}entries_fts, rowid, title, body, tags)
124
- VALUES ('delete', old.rowid, old.title, old.body, old.tags);
125
- INSERT INTO ${prefix}entries_fts(rowid, title, body, tags)
126
- VALUES (new.rowid, new.title, new.body, new.tags);
127
- END;
128
- `);
129
- });
130
- }
131
- }
132
- ];
133
- for (let i = 1; i < MIGRATIONS.length; i++) {
134
- if (MIGRATIONS[i].version <= MIGRATIONS[i - 1].version) {
135
- throw new Error(
136
- `migrations.ts: MIGRATIONS must be in strictly ascending version order. Found version ${MIGRATIONS[i].version} after ${MIGRATIONS[i - 1].version} at index ${i}.`
137
- );
138
- }
139
- }
140
- var CURRENT_SCHEMA_VERSION = MIGRATIONS.length > 0 ? MIGRATIONS[MIGRATIONS.length - 1].version : 0;
141
-
142
- // src/types.ts
143
- var WikiBusyError = class extends Error {
144
- operation;
145
- entityId;
146
- constructor(operation, entityId) {
147
- super(`${operation} already running for entity ${entityId}`);
148
- this.name = "WikiBusyError";
149
- this.operation = operation;
150
- this.entityId = entityId;
151
- }
152
- };
153
-
154
- // src/prompts.ts
155
- var LIBRARIAN_SYSTEM_PROMPT = `You are a knowledge extraction agent. Your job is to analyze recent episodic events and extract stable facts and actionable tasks about the user or entity.
156
- Return ONLY a valid JSON object matching this schema:
157
- {
158
- "facts": [{ "title": "string (max 80 chars)", "body": "string (max 800 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative" }],
159
- "tasks": [{ "description": "string", "priority": "number (0-10)" }]
160
- }
161
- Keep facts concise. Do not return markdown, just raw JSON.`;
162
- var HEAL_SYSTEM_PROMPT = `You are a memory grooming agent. Your job is to review a full dump of facts and recent events to resolve contradictions, downgrade stale claims, and flag obsolete facts for deletion.
163
- Return ONLY a valid JSON object matching this schema:
164
- {
165
- "downgraded": ["string (fact IDs)"],
166
- "deleted": ["string (fact IDs)"],
167
- "newFacts": [{ "title": "string (max 80 chars)", "body": "string (max 800 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative" }]
168
- }
169
- Do not return markdown, just raw JSON.`;
170
- var INGEST_SYSTEM_PROMPT = `You are a document ingestion agent. Your job is to extract factual knowledge from the provided document chunk.
171
- Return ONLY a valid JSON object matching this schema:
172
- {
173
- "facts": [{ "title": "string (max 80 chars)", "body": "string (max 800 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative" }]
174
- }
175
- Extract verbatim factual content. Do not return markdown, just raw JSON.`;
176
-
177
- // src/WikiMemory.ts
178
- function parseJsonResponse(text) {
179
- const firstBrace = text.indexOf("{");
180
- const firstBracket = text.indexOf("[");
181
- let start;
182
- let openChar;
183
- let closeChar;
184
- if (firstBrace !== -1 && (firstBracket === -1 || firstBrace < firstBracket)) {
185
- start = firstBrace;
186
- openChar = "{";
187
- closeChar = "}";
188
- } else if (firstBracket !== -1) {
189
- start = firstBracket;
190
- openChar = "[";
191
- closeChar = "]";
192
- } else {
193
- throw new SyntaxError("No JSON object/array found in LLM response");
194
- }
195
- let depth = 0;
196
- let inString = false;
197
- let escape = false;
198
- let end = -1;
199
- for (let i = start; i < text.length; i++) {
200
- const ch = text[i];
201
- if (escape) {
202
- escape = false;
203
- continue;
204
- }
205
- if (ch === "\\" && inString) {
206
- escape = true;
207
- continue;
208
- }
209
- if (ch === '"') {
210
- inString = !inString;
211
- continue;
212
- }
213
- if (inString) continue;
214
- if (ch === openChar) {
215
- depth++;
216
- continue;
217
- }
218
- if (ch === closeChar) {
219
- depth--;
220
- if (depth === 0) {
221
- end = i;
222
- break;
223
- }
224
- }
225
- }
226
- if (end === -1) throw new SyntaxError("No JSON object/array found in LLM response");
227
- return JSON.parse(text.slice(start, end + 1));
228
- }
229
- function generateId(prefix = "") {
230
- return prefix + Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15);
231
- }
232
- function safeSlice(value, start, end) {
233
- const length = value.length;
234
- let safeStart = start < 0 ? Math.max(length + start, 0) : Math.min(start, length);
235
- let safeEnd = end === void 0 ? length : end < 0 ? Math.max(length + end, 0) : Math.min(end, length);
236
- if (safeStart > safeEnd) {
237
- [safeStart, safeEnd] = [safeEnd, safeStart];
238
- }
239
- if (safeStart > 0 && safeStart < length && value.charCodeAt(safeStart) >= 56320 && value.charCodeAt(safeStart) <= 57343 && value.charCodeAt(safeStart - 1) >= 55296 && value.charCodeAt(safeStart - 1) <= 56319) {
240
- safeStart--;
241
- }
242
- if (safeEnd > 0 && safeEnd < length && value.charCodeAt(safeEnd - 1) >= 55296 && value.charCodeAt(safeEnd - 1) <= 56319 && value.charCodeAt(safeEnd) >= 56320 && value.charCodeAt(safeEnd) <= 57343) {
243
- safeEnd--;
244
- }
245
- return value.slice(safeStart, safeEnd);
246
- }
247
- function chunkText(input, maxChunkLength, overlap) {
248
- const text = input.trim();
249
- if (text.length === 0) return { chunks: [], truncated: false };
250
- if (!Number.isInteger(maxChunkLength) || maxChunkLength < 2) {
251
- throw new Error("maxChunkLength must be an integer >= 2");
252
- }
253
- if (!Number.isInteger(overlap) || overlap < 0 || overlap >= maxChunkLength) {
254
- throw new Error("overlap must be a non-negative integer < maxChunkLength");
255
- }
256
- const chunks = [];
257
- let truncated = false;
258
- let cursor = 0;
259
- const halfMax = Math.floor(maxChunkLength / 2);
260
- while (cursor < text.length) {
261
- const remaining = text.length - cursor;
262
- if (remaining <= maxChunkLength) {
263
- chunks.push(safeSlice(text, cursor, text.length));
264
- break;
265
- }
266
- const windowEnd = cursor + maxChunkLength;
267
- const minSplit = cursor + halfMax;
268
- let splitPoint = -1;
269
- const paraIdx = text.lastIndexOf("\n\n", windowEnd);
270
- if (paraIdx >= minSplit && paraIdx + 2 <= windowEnd) {
271
- splitPoint = paraIdx + 2;
272
- }
273
- if (splitPoint === -1) {
274
- let lastTerm = -1;
275
- for (let i = minSplit; i < windowEnd - 1; i++) {
276
- const ch = text[i];
277
- if ((ch === "." || ch === "!" || ch === "?") && /\s/.test(text[i + 1])) {
278
- lastTerm = i + 2;
279
- }
280
- }
281
- if (lastTerm !== -1 && lastTerm <= windowEnd) splitPoint = lastTerm;
282
- }
283
- if (splitPoint === -1) {
284
- for (let i = windowEnd - 1; i >= minSplit; i--) {
285
- if (/\s/.test(text[i])) {
286
- splitPoint = i + 1;
287
- break;
288
- }
289
- }
290
- }
291
- if (splitPoint === -1) {
292
- truncated = true;
293
- splitPoint = windowEnd;
294
- }
295
- chunks.push(safeSlice(text, cursor, splitPoint));
296
- const next = Math.max(splitPoint - overlap, cursor + 1);
297
- cursor = next;
298
- }
299
- return { chunks, truncated };
300
- }
301
- async function withConcurrency(tasks, limit) {
302
- const results = new Array(tasks.length);
303
- let index = 0;
304
- let failed = false;
305
- let firstError;
306
- async function worker() {
307
- while (index < tasks.length && !failed) {
308
- const i = index++;
309
- try {
310
- results[i] = await tasks[i]();
311
- } catch (e) {
312
- if (!failed) {
313
- failed = true;
314
- firstError = e;
315
- }
316
- return;
317
- }
318
- }
319
- }
320
- const workerCount = tasks.length === 0 ? 0 : Math.min(Math.max(limit, 1), tasks.length);
321
- await Promise.allSettled(Array.from({ length: workerCount }, worker));
322
- if (failed) throw firstError;
323
- return results;
324
- }
325
- function clip(value, max) {
326
- if (typeof value !== "string") return "";
327
- const s = value.trim();
328
- return s.length <= max ? s : safeSlice(s, 0, max).trimEnd();
329
- }
330
- function validateTags(tags) {
331
- if (!Array.isArray(tags)) return [];
332
- return tags.filter((t) => typeof t === "string").map((t) => t.trim().toLowerCase()).filter((t) => t.length > 0 && t.length <= 40).slice(0, 6);
333
- }
334
- function validateFact(fact) {
335
- if (typeof fact?.title !== "string" || typeof fact?.body !== "string") return null;
336
- const title = clip(fact.title, 80);
337
- const body = clip(fact.body, 800);
338
- if (!title || !body) return null;
339
- let confidence = fact.confidence;
340
- if (confidence !== "certain" && confidence !== "tentative") confidence = "inferred";
341
- return {
342
- ...fact,
343
- title,
344
- body,
345
- confidence,
346
- tags: validateTags(fact.tags)
347
- };
348
- }
349
- function validateTask(task) {
350
- if (typeof task?.description !== "string") return null;
351
- const description = clip(task.description, 200);
352
- if (!description) return null;
353
- let priority = task.priority;
354
- if (typeof priority !== "number" || !isFinite(priority)) priority = 0;
355
- return {
356
- ...task,
357
- description,
358
- priority
359
- };
360
- }
361
- function normalizeSourceRef(value) {
362
- if (typeof value !== "string") return null;
363
- const cleaned = value.replace(/[^A-Za-z0-9._\- ]/g, "").trim().slice(0, 255);
364
- return cleaned.length > 0 ? cleaned : null;
365
- }
366
- function normalizeSourceHash(value) {
367
- if (typeof value !== "string") return null;
368
- return /^[0-9a-f]{64}$/i.test(value) ? value.toLowerCase() : null;
369
- }
370
- function titleTokens(title) {
371
- return new Set(title.toLowerCase().replace(/[^a-z0-9\s]/g, "").split(/\s+/).filter((t) => t.length >= 3));
372
- }
373
- function jaccardScore(a, b) {
374
- if (a.size === 0 && b.size === 0) return 0;
375
- const intersection = new Set([...a].filter((x) => b.has(x)));
376
- const union = /* @__PURE__ */ new Set([...a, ...b]);
377
- return intersection.size / union.size;
378
- }
379
- var FUZZY_THRESHOLD = 0.5;
380
- var MIN_TOKENS_TO_QUALIFY = 3;
381
- var WikiMemory = class {
382
- db;
383
- prefix;
384
- options;
385
- activeMaintenanceJobs = /* @__PURE__ */ new Set();
386
- activeIngestJobs = /* @__PURE__ */ new Set();
387
- _librarianKey(entityId) {
388
- return `${this.prefix}:${entityId}:librarian`;
389
- }
390
- _healKey(entityId) {
391
- return `${this.prefix}:${entityId}:heal`;
392
- }
393
- _warnCrossEntityCollision(type, id, existingEntityId, targetEntityId) {
394
- console.warn(`[WikiMemory] importDump: ${type} id "${id}" already belongs to entity "${existingEntityId}"; skipping for entity "${targetEntityId}"`);
395
- }
396
- constructor(db, options) {
397
- this.db = db;
398
- this.options = options;
399
- this.prefix = options.config?.tablePrefix || "llm_wiki_";
400
- }
401
- async setup() {
402
- const entriesExistedBeforeSetup = await this.db.getFirstAsync(
403
- `SELECT name FROM sqlite_master WHERE type='table' AND name=?`,
404
- [`${this.prefix}entries`]
405
- );
406
- await setupDatabase(this.db, this.prefix);
407
- let currentVersion;
408
- if (!entriesExistedBeforeSetup) {
409
- await this.db.runAsync(
410
- `INSERT OR REPLACE INTO ${this.prefix}meta (key, value) VALUES ('schema_version', ?)`,
411
- [String(CURRENT_SCHEMA_VERSION)]
412
- );
413
- currentVersion = CURRENT_SCHEMA_VERSION;
414
- } else {
415
- const metaRow = await this.db.getFirstAsync(
416
- `SELECT value FROM ${this.prefix}meta WHERE key = 'schema_version'`
417
- );
418
- if (metaRow) {
419
- currentVersion = parseInt(metaRow.value, 10);
420
- if (!Number.isFinite(currentVersion)) currentVersion = 0;
421
- } else {
422
- const ftsMeta = await this.db.getFirstAsync(
423
- `SELECT sql FROM sqlite_master WHERE type='table' AND name=?`,
424
- [`${this.prefix}entries_fts`]
425
- );
426
- const hasPorter = /tokenize\s*=\s*['"]porter\s+unicode61['"]/i.test(ftsMeta?.sql ?? "");
427
- currentVersion = hasPorter ? 1 : 0;
428
- }
429
- }
430
- for (const migration of MIGRATIONS) {
431
- if (migration.version > currentVersion) {
432
- await migration.run(this.db, this.prefix);
433
- await this.db.runAsync(
434
- `INSERT OR REPLACE INTO ${this.prefix}meta (key, value) VALUES ('schema_version', ?)`,
435
- [String(migration.version)]
436
- );
437
- currentVersion = migration.version;
438
- }
439
- }
440
- if (entriesExistedBeforeSetup) {
441
- const metaCheck = await this.db.getFirstAsync(
442
- `SELECT value FROM ${this.prefix}meta WHERE key = 'schema_version'`
443
- );
444
- if (!metaCheck) {
445
- await this.db.runAsync(
446
- `INSERT OR REPLACE INTO ${this.prefix}meta (key, value) VALUES ('schema_version', ?)`,
447
- [String(currentVersion)]
448
- );
449
- }
450
- }
451
- const rows = await this.db.getAllAsync(`
452
- SELECT rowid, source_ref FROM ${this.prefix}entries
453
- WHERE source_ref IS NOT NULL
454
- AND (
455
- TRIM(source_ref) != source_ref
456
- OR INSTR(source_ref, '/') > 0
457
- OR INSTR(source_ref, '\\') > 0
458
- OR INSTR(source_ref, CHAR(0)) > 0
459
- OR source_ref GLOB '*[^-A-Za-z0-9._ ]*'
460
- )
461
- `);
462
- await this.db.withTransactionAsync(async () => {
463
- for (const row of rows) {
464
- const normalized = normalizeSourceRef(row.source_ref);
465
- if (normalized !== row.source_ref) {
466
- await this.db.runAsync(
467
- `UPDATE ${this.prefix}entries SET source_ref = ? WHERE rowid = ?`,
468
- [normalized, row.rowid]
469
- );
470
- }
471
- }
472
- });
473
- }
474
- async hasChanged(entityId, sourceRef, sourceHash) {
475
- const normalizedRef = normalizeSourceRef(sourceRef);
476
- if (!normalizedRef) {
477
- throw new Error(`Invalid sourceRef: "${sourceRef}"`);
478
- }
479
- const normalizedHash = normalizeSourceHash(sourceHash);
480
- if (!normalizedHash) {
481
- throw new Error(`Invalid sourceHash: must be a 64-character hex string (normalized to lowercase)`);
482
- }
483
- const row = await this.db.getFirstAsync(
484
- `SELECT source_hash FROM ${this.prefix}entries
485
- WHERE entity_id = ? AND source_ref = ? AND deleted_at IS NULL
486
- ORDER BY updated_at DESC
487
- LIMIT 1`,
488
- [entityId, normalizedRef]
489
- );
490
- if (!row) return true;
491
- const normalizedStoredHash = row.source_hash ? normalizeSourceHash(row.source_hash) : null;
492
- return normalizedStoredHash !== normalizedHash;
493
- }
494
- _pruneKey(entityId) {
495
- return `${this.prefix}:${entityId}:prune`;
496
- }
497
- _validatePruneDuration(value, name) {
498
- if (value !== null && value !== void 0 && (typeof value !== "number" || !isFinite(value) || value < 0)) {
499
- throw new Error(`Invalid ${name}: must be a non-negative finite number or null`);
500
- }
501
- }
502
- async runPrune(entityId, options) {
503
- const pruneKey = this._pruneKey(entityId);
504
- const ingestPrefix = `${this.prefix}:${entityId}:`;
505
- let isIngestRunning = false;
506
- for (const k of this.activeIngestJobs) {
507
- if (k.startsWith(ingestPrefix)) {
508
- isIngestRunning = true;
509
- break;
510
- }
511
- }
512
- let blockingOperation = null;
513
- if (this.activeMaintenanceJobs.has(pruneKey)) {
514
- blockingOperation = "prune";
515
- } else if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) {
516
- blockingOperation = "librarian";
517
- } else if (this.activeMaintenanceJobs.has(this._healKey(entityId))) {
518
- blockingOperation = "heal";
519
- } else if (isIngestRunning) {
520
- blockingOperation = "ingest";
521
- }
522
- if (blockingOperation !== null) {
523
- throw new WikiBusyError(blockingOperation, entityId);
524
- }
525
- this.activeMaintenanceJobs.add(pruneKey);
526
- try {
527
- const retainSoftDeletedFor = options?.retainSoftDeletedFor !== void 0 ? options.retainSoftDeletedFor : this.options.config?.pruneRetainSoftDeletedFor ?? 7;
528
- const retainEventsFor = options?.retainEventsFor !== void 0 ? options.retainEventsFor : this.options.config?.pruneEventsAfter ?? 30;
529
- const vacuum = options?.vacuum ?? false;
530
- this._validatePruneDuration(retainSoftDeletedFor, "retainSoftDeletedFor");
531
- this._validatePruneDuration(retainEventsFor, "retainEventsFor");
532
- const now = Date.now();
533
- let deletedEntries = 0;
534
- let deletedTasks = 0;
535
- let deletedEvents = 0;
536
- if (retainSoftDeletedFor !== null) {
537
- const cutoff = now - retainSoftDeletedFor * 864e5;
538
- const entryResult = await this.db.runAsync(
539
- `DELETE FROM ${this.prefix}entries
540
- WHERE entity_id = ? AND deleted_at IS NOT NULL AND deleted_at < ?`,
541
- [entityId, cutoff]
542
- );
543
- deletedEntries = entryResult.changes;
544
- const taskResult = await this.db.runAsync(
545
- `DELETE FROM ${this.prefix}tasks
546
- WHERE entity_id = ? AND deleted_at IS NOT NULL AND deleted_at < ?`,
547
- [entityId, cutoff]
548
- );
549
- deletedTasks = taskResult.changes;
550
- }
551
- if (retainEventsFor !== null) {
552
- const cutoff = now - retainEventsFor * 864e5;
553
- const eventResult = await this.db.runAsync(
554
- `DELETE FROM ${this.prefix}events
555
- WHERE entity_id = ? AND created_at < ?`,
556
- [entityId, cutoff]
557
- );
558
- deletedEvents = eventResult.changes;
559
- }
560
- if (vacuum) {
561
- await this.db.execAsync(`PRAGMA wal_checkpoint(TRUNCATE)`);
562
- await this.db.execAsync(`VACUUM`);
563
- }
564
- return { entries: deletedEntries, tasks: deletedTasks, events: deletedEvents };
565
- } finally {
566
- this.activeMaintenanceJobs.delete(pruneKey);
567
- }
568
- }
569
- formatSearchQuery(query) {
570
- const normalizeTokens = (value) => value.toLowerCase().replace(/[^a-z0-9\s]/g, "").split(/\s+/).filter((t) => t.length >= 3);
571
- const baseTokens = normalizeTokens(query);
572
- if (baseTokens.length === 0) return "";
573
- const synonymMap = this.options.config?.synonymMap;
574
- const expanded = [];
575
- const seen = /* @__PURE__ */ new Set();
576
- const pushNormalized = (value) => {
577
- for (const token of normalizeTokens(value)) {
578
- if (expanded.length >= 12) return false;
579
- if (seen.has(token)) continue;
580
- seen.add(token);
581
- expanded.push(token);
582
- }
583
- return true;
584
- };
585
- for (const t of baseTokens) {
586
- if (!pushNormalized(t)) break;
587
- if (synonymMap) {
588
- const synonyms = synonymMap[t];
589
- if (Array.isArray(synonyms)) {
590
- for (const s of synonyms) {
591
- if (typeof s === "string") {
592
- if (!pushNormalized(s)) break;
593
- }
594
- }
595
- }
596
- }
597
- }
598
- return expanded.map((t) => `"${t}"*`).join(" OR ");
599
- }
600
- async read(entityId, query) {
601
- const ftsQuery = this.formatSearchQuery(query);
602
- const maxResults = this.options.config?.maxFtsResults || 10;
603
- let factsPromise;
604
- if (ftsQuery) {
605
- factsPromise = this.db.getAllAsync(`
606
- SELECT e.* FROM ${this.prefix}entries e
607
- JOIN ${this.prefix}entries_fts fts ON e.rowid = fts.rowid
608
- WHERE fts.${this.prefix}entries_fts MATCH ?
609
- AND e.entity_id = ?
610
- AND e.deleted_at IS NULL
611
- ORDER BY e.confidence DESC, e.access_count DESC, e.updated_at DESC
612
- LIMIT ?
613
- `, [ftsQuery, entityId, maxResults]);
614
- } else {
615
- factsPromise = this.db.getAllAsync(`
616
- SELECT * FROM ${this.prefix}entries
617
- WHERE entity_id = ? AND deleted_at IS NULL
618
- ORDER BY updated_at DESC
619
- LIMIT ?
620
- `, [entityId, maxResults]);
621
- }
622
- const tasksPromise = this.db.getAllAsync(`
623
- SELECT * FROM ${this.prefix}tasks
624
- WHERE entity_id = ? AND status IN ('pending', 'in_progress') AND deleted_at IS NULL
625
- ORDER BY priority DESC, created_at ASC
626
- `, [entityId]);
627
- const eventsPromise = this.db.getAllAsync(`
628
- SELECT * FROM ${this.prefix}events
629
- WHERE entity_id = ?
630
- ORDER BY created_at DESC
631
- LIMIT 10
632
- `, [entityId]);
633
- const [factsRaw, tasks, events] = await Promise.all([factsPromise, tasksPromise, eventsPromise]);
634
- if (ftsQuery && factsRaw.length > 0) {
635
- const ids = factsRaw.map((f) => f.id);
636
- const placeholders = ids.map(() => "?").join(",");
637
- const now = Date.now();
638
- await this.db.runAsync(`
639
- UPDATE ${this.prefix}entries
640
- SET access_count = access_count + 1, last_accessed_at = ?
641
- WHERE id IN (${placeholders})
642
- `, [now, ...ids]);
643
- }
644
- const facts = factsRaw.map((f) => ({
645
- ...f,
646
- tags: typeof f.tags === "string" ? JSON.parse(f.tags) : f.tags
647
- }));
648
- return { facts, tasks, events: events.reverse() };
649
- }
650
- async getMemoryBundle(entityId) {
651
- return this._getFullBundle(entityId, { maxEvents: 10 });
652
- }
653
- async write(entityId, event) {
654
- const id = generateId("evt_");
655
- const now = Date.now();
656
- let eventType = event.event_type;
657
- if (!["observation", "decision", "action", "outcome"].includes(eventType)) {
658
- eventType = "observation";
659
- }
660
- await this.db.runAsync(`
661
- INSERT INTO ${this.prefix}events (id, entity_id, event_type, summary, related_entry_id, created_at)
662
- VALUES (?, ?, ?, ?, ?, ?)
663
- `, [id, entityId, eventType, event.summary, event.related_entry_id || null, now]);
664
- const threshold = this.options.config?.autoLibrarianThreshold || 20;
665
- const [row, cp] = await Promise.all([
666
- this.db.getFirstAsync(`SELECT COUNT(*) as count FROM ${this.prefix}events WHERE entity_id = ?`, [entityId]),
667
- this.db.getFirstAsync(`SELECT * FROM ${this.prefix}checkpoints WHERE entity_id = ?`, [entityId])
668
- ]);
669
- const count = row?.count || 0;
670
- let memoryCheckpoint = cp?.memory_checkpoint || 0;
671
- if (memoryCheckpoint > count) memoryCheckpoint = 0;
672
- if (count - memoryCheckpoint >= threshold) {
673
- const jobKey = this._librarianKey(entityId);
674
- if (!this.activeMaintenanceJobs.has(jobKey) && !this.activeMaintenanceJobs.has(this._pruneKey(entityId))) {
675
- this.activeMaintenanceJobs.add(jobKey);
676
- this.runLibrarianThenMaybeHeal(entityId, count).catch(console.error).finally(() => this.activeMaintenanceJobs.delete(jobKey));
677
- }
678
- }
679
- }
680
- async runLibrarianThenMaybeHeal(entityId, currentEventCount) {
681
- await this._doRunLibrarian(entityId);
682
- await this.db.runAsync(`
683
- INSERT INTO ${this.prefix}checkpoints (entity_id, memory_checkpoint)
684
- VALUES (?, ?)
685
- ON CONFLICT(entity_id) DO UPDATE SET memory_checkpoint = ?
686
- `, [entityId, currentEventCount, currentEventCount]);
687
- const autoHealThreshold = this.options.config?.autoHealThreshold || 100;
688
- const cp = await this.db.getFirstAsync(`SELECT * FROM ${this.prefix}checkpoints WHERE entity_id = ?`, [entityId]);
689
- let healCheckpoint = cp?.heal_checkpoint || 0;
690
- if (healCheckpoint > currentEventCount) healCheckpoint = 0;
691
- if (currentEventCount - healCheckpoint >= autoHealThreshold) {
692
- const healKey = this._healKey(entityId);
693
- if (!this.activeMaintenanceJobs.has(healKey)) {
694
- this.activeMaintenanceJobs.add(healKey);
695
- try {
696
- await this._doRunHeal(entityId);
697
- await this.db.runAsync(`
698
- INSERT INTO ${this.prefix}checkpoints (entity_id, heal_checkpoint)
699
- VALUES (?, ?)
700
- ON CONFLICT(entity_id) DO UPDATE SET heal_checkpoint = ?
701
- `, [entityId, currentEventCount, currentEventCount]);
702
- } finally {
703
- this.activeMaintenanceJobs.delete(healKey);
704
- }
705
- }
706
- }
707
- }
708
- async _doRunLibrarian(entityId) {
709
- const events = await this.db.getAllAsync(`
710
- SELECT * FROM ${this.prefix}events
711
- WHERE entity_id = ?
712
- ORDER BY created_at DESC
713
- LIMIT 50
714
- `, [entityId]);
715
- const currentFactsRows = await this.db.getAllAsync(`
716
- SELECT * FROM ${this.prefix}entries
717
- WHERE entity_id = ? AND deleted_at IS NULL
718
- ORDER BY updated_at DESC
719
- LIMIT 100
720
- `, [entityId]);
721
- const currentFacts = currentFactsRows.map((f) => ({
722
- ...f,
723
- tags: typeof f.tags === "string" ? JSON.parse(f.tags) : f.tags
724
- }));
725
- const userPrompt = `Events:
726
- ${JSON.stringify(events.reverse(), null, 2)}
727
-
728
- Current Facts:
729
- ${JSON.stringify(currentFacts, null, 2)}`;
730
- const responseText = await this.options.llmProvider.generateText({
731
- systemPrompt: LIBRARIAN_SYSTEM_PROMPT,
732
- userPrompt
733
- });
734
- const result = parseJsonResponse(responseText);
735
- const facts = Array.isArray(result.facts) ? result.facts : [];
736
- const tasks = Array.isArray(result.tasks) ? result.tasks : [];
737
- const validFacts = facts.map(validateFact).filter((f) => f !== null);
738
- const validTasks = tasks.map(validateTask).filter((t) => t !== null);
739
- const now = Date.now();
740
- await this.db.withTransactionAsync(async () => {
741
- for (const fact of validFacts) {
742
- const newTokens = titleTokens(fact.title);
743
- let skip = false;
744
- if (newTokens.size >= MIN_TOKENS_TO_QUALIFY) {
745
- for (const existing of currentFactsRows) {
746
- if (existing.source_type !== "agent_inferred") continue;
747
- const existingTokens = titleTokens(existing.title);
748
- if (existingTokens.size >= MIN_TOKENS_TO_QUALIFY) {
749
- if (jaccardScore(newTokens, existingTokens) >= FUZZY_THRESHOLD) {
750
- skip = true;
751
- break;
752
- }
753
- }
754
- }
755
- }
756
- if (skip) continue;
757
- const id = generateId("fact_");
758
- await this.db.runAsync(`
759
- INSERT INTO ${this.prefix}entries (id, entity_id, title, body, tags, confidence, source_type, created_at, updated_at)
760
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
761
- `, [id, entityId, fact.title, fact.body, JSON.stringify(fact.tags), fact.confidence, "agent_inferred", now, now]);
762
- }
763
- for (const task of validTasks) {
764
- const id = generateId("task_");
765
- await this.db.runAsync(`
766
- INSERT INTO ${this.prefix}tasks (id, entity_id, description, status, priority, created_at, updated_at)
767
- VALUES (?, ?, ?, ?, ?, ?, ?)
768
- `, [id, entityId, task.description, "pending", task.priority, now, now]);
769
- }
770
- });
771
- }
772
- async _doRunHeal(entityId) {
773
- const now = Date.now();
774
- const orphanAfterDays = this.options.config?.orphanAfterDays !== void 0 ? this.options.config.orphanAfterDays : 30;
775
- const staleInferredAfterDays = this.options.config?.staleInferredAfterDays !== void 0 ? this.options.config.staleInferredAfterDays : 60;
776
- const MS_PER_DAY = 24 * 60 * 60 * 1e3;
777
- if (orphanAfterDays !== null && (typeof orphanAfterDays !== "number" || !Number.isFinite(orphanAfterDays) || orphanAfterDays < 0)) {
778
- throw new Error("Invalid orphanAfterDays: must be a finite number >= 0 or null");
779
- }
780
- if (staleInferredAfterDays !== null && (typeof staleInferredAfterDays !== "number" || !Number.isFinite(staleInferredAfterDays) || staleInferredAfterDays < 0)) {
781
- throw new Error("Invalid staleInferredAfterDays: must be a finite number >= 0 or null");
782
- }
783
- await this.db.withTransactionAsync(async () => {
784
- if (orphanAfterDays !== null) {
785
- const orphanThreshold = now - orphanAfterDays * MS_PER_DAY;
786
- await this.db.runAsync(`
787
- UPDATE ${this.prefix}entries
788
- SET deleted_at = ?, updated_at = ?
789
- WHERE entity_id = ? AND access_count = 0 AND created_at < ? AND source_type != 'user_document' AND deleted_at IS NULL
790
- `, [now, now, entityId, orphanThreshold]);
791
- }
792
- if (staleInferredAfterDays !== null) {
793
- const staleThreshold = now - staleInferredAfterDays * MS_PER_DAY;
794
- await this.db.runAsync(`
795
- UPDATE ${this.prefix}entries
796
- SET confidence = 'tentative', updated_at = ?
797
- WHERE entity_id = ? AND confidence = 'inferred' AND (last_accessed_at < ? OR (last_accessed_at IS NULL AND created_at < ?)) AND source_type != 'user_document' AND deleted_at IS NULL
798
- `, [now, entityId, staleThreshold, staleThreshold]);
799
- }
800
- });
801
- const allFactsRows = await this.db.getAllAsync(`SELECT * FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL`, [entityId]);
802
- const allTasks = await this.db.getAllAsync(`SELECT * FROM ${this.prefix}tasks WHERE entity_id = ? AND status IN ('pending', 'in_progress') AND deleted_at IS NULL`, [entityId]);
803
- const recentEvents = await this.db.getAllAsync(`SELECT * FROM ${this.prefix}events WHERE entity_id = ? ORDER BY created_at DESC LIMIT 20`, [entityId]);
804
- const healCandidates = allFactsRows.filter((f) => f.source_type !== "user_document");
805
- const documentAnchors = allFactsRows.filter((f) => f.source_type === "user_document").map(({ id, title, source_ref }) => ({ id, title, source_ref }));
806
- const userPrompt = `Heal Candidates:
807
- ${JSON.stringify(healCandidates.map((f) => ({ ...f, tags: typeof f.tags === "string" ? JSON.parse(f.tags) : f.tags })), null, 2)}
808
-
809
- Document Anchors (DO NOT MODIFY OR DELETE):
810
- ${JSON.stringify(documentAnchors, null, 2)}
811
-
812
- All Tasks:
813
- ${JSON.stringify(allTasks, null, 2)}
814
-
815
- Recent Events:
816
- ${JSON.stringify(recentEvents, null, 2)}
817
-
818
- The following document anchors are provided for contradiction detection only. Do not include them in \`downgraded\`, \`deleted\`, or \`newFacts\`.`;
819
- const responseText = await this.options.llmProvider.generateText({
820
- systemPrompt: HEAL_SYSTEM_PROMPT,
821
- userPrompt
822
- });
823
- const result = parseJsonResponse(responseText);
824
- const mutableIds = new Set(healCandidates.map((f) => f.id));
825
- const downgraded = Array.isArray(result.downgraded) ? result.downgraded : [];
826
- const deleted = Array.isArray(result.deleted) ? result.deleted : [];
827
- const newFacts = Array.isArray(result.newFacts) ? result.newFacts : [];
828
- const safeDowngraded = downgraded.filter((id) => mutableIds.has(id));
829
- const safeDeleted = deleted.filter((id) => mutableIds.has(id));
830
- const validNewFacts = newFacts.map(validateFact).filter((f) => f !== null);
831
- await this.db.withTransactionAsync(async () => {
832
- for (const id of safeDowngraded) {
833
- await this.db.runAsync(`UPDATE ${this.prefix}entries SET confidence = 'tentative', updated_at = ? WHERE id = ? AND entity_id = ?`, [now, id, entityId]);
834
- }
835
- for (const id of safeDeleted) {
836
- await this.db.runAsync(`UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE id = ? AND entity_id = ?`, [now, now, id, entityId]);
837
- }
838
- for (const fact of validNewFacts) {
839
- const id = generateId("fact_");
840
- await this.db.runAsync(`
841
- INSERT INTO ${this.prefix}entries (id, entity_id, title, body, tags, confidence, source_type, created_at, updated_at)
842
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
843
- `, [id, entityId, fact.title, fact.body, JSON.stringify(fact.tags), fact.confidence, "agent_inferred", now, now]);
844
- }
845
- });
846
- }
847
- async runLibrarian(entityId) {
848
- const jobKey = this._librarianKey(entityId);
849
- if (this.activeMaintenanceJobs.has(jobKey)) {
850
- throw new WikiBusyError("librarian", entityId);
851
- }
852
- if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) {
853
- throw new WikiBusyError("prune", entityId);
854
- }
855
- this.activeMaintenanceJobs.add(jobKey);
856
- try {
857
- await this._doRunLibrarian(entityId);
858
- } finally {
859
- this.activeMaintenanceJobs.delete(jobKey);
860
- }
861
- }
862
- async runHeal(entityId) {
863
- const jobKey = this._healKey(entityId);
864
- if (this.activeMaintenanceJobs.has(jobKey)) {
865
- throw new WikiBusyError("heal", entityId);
866
- }
867
- if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) {
868
- throw new WikiBusyError("prune", entityId);
869
- }
870
- this.activeMaintenanceJobs.add(jobKey);
871
- try {
872
- await this._doRunHeal(entityId);
873
- } finally {
874
- this.activeMaintenanceJobs.delete(jobKey);
875
- }
876
- }
877
- getEntityStatus(entityId) {
878
- const ingestPrefix = `${this.prefix}:${entityId}:`;
879
- let ingesting = false;
880
- for (const k of this.activeIngestJobs) {
881
- if (k.startsWith(ingestPrefix)) {
882
- ingesting = true;
883
- break;
884
- }
885
- }
886
- return {
887
- ingesting,
888
- librarian: this.activeMaintenanceJobs.has(this._librarianKey(entityId)),
889
- heal: this.activeMaintenanceJobs.has(this._healKey(entityId))
890
- };
891
- }
892
- async _getFullBundle(entityId, opts) {
893
- const maxEvents = opts?.maxEvents;
894
- const eventsQuery = maxEvents != null ? `SELECT * FROM ${this.prefix}events WHERE entity_id = ? ORDER BY created_at DESC LIMIT ?` : `SELECT * FROM ${this.prefix}events WHERE entity_id = ? ORDER BY created_at ASC`;
895
- const eventsParams = maxEvents != null ? [entityId, maxEvents] : [entityId];
896
- const [factsRaw, tasks, eventsRaw] = await Promise.all([
897
- this.db.getAllAsync(
898
- `SELECT * FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL ORDER BY updated_at DESC`,
899
- [entityId]
900
- ),
901
- this.db.getAllAsync(
902
- `SELECT * FROM ${this.prefix}tasks WHERE entity_id = ? AND deleted_at IS NULL ORDER BY priority DESC, created_at ASC`,
903
- [entityId]
904
- ),
905
- this.db.getAllAsync(eventsQuery, eventsParams)
906
- ]);
907
- const facts = factsRaw.map((f) => ({
908
- ...f,
909
- tags: typeof f.tags === "string" ? JSON.parse(f.tags) : f.tags
910
- }));
911
- const events = maxEvents != null ? eventsRaw.slice().reverse() : eventsRaw;
912
- return { facts, tasks, events };
913
- }
914
- async exportDump(entityIds) {
915
- let ids;
916
- if (entityIds && entityIds.length > 0) {
917
- ids = Array.from(new Set(entityIds));
918
- } else {
919
- const rows = await this.db.getAllAsync(`
920
- SELECT DISTINCT entity_id FROM (
921
- SELECT entity_id FROM ${this.prefix}entries WHERE deleted_at IS NULL
922
- UNION
923
- SELECT entity_id FROM ${this.prefix}tasks WHERE deleted_at IS NULL
924
- UNION
925
- SELECT entity_id FROM ${this.prefix}events
926
- ) ORDER BY entity_id
927
- `);
928
- ids = rows.map((r) => r.entity_id);
929
- }
930
- const entities = {};
931
- const BATCH = 3;
932
- for (let i = 0; i < ids.length; i += BATCH) {
933
- const batch = ids.slice(i, i + BATCH);
934
- const batchResults = await Promise.all(
935
- batch.map(async (id) => [id, await this._getFullBundle(id)])
936
- );
937
- for (const [id, bundle] of batchResults) {
938
- entities[id] = bundle;
939
- }
940
- }
941
- return { generatedAt: Date.now(), entities };
942
- }
943
- async importDump(dump, opts) {
944
- const merge = opts?.merge ?? false;
945
- for (const [entityId, bundle] of Object.entries(dump.entities)) {
946
- await this.db.withTransactionAsync(async () => {
947
- if (!merge) {
948
- const now = Date.now();
949
- await this.db.runAsync(
950
- `UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE entity_id = ? AND deleted_at IS NULL`,
951
- [now, now, entityId]
952
- );
953
- await this.db.runAsync(
954
- `UPDATE ${this.prefix}tasks SET deleted_at = ?, updated_at = ? WHERE entity_id = ? AND deleted_at IS NULL`,
955
- [now, now, entityId]
956
- );
957
- await this.db.runAsync(
958
- `DELETE FROM ${this.prefix}checkpoints WHERE entity_id = ?`,
959
- [entityId]
960
- );
961
- }
962
- const factIds = bundle.facts.map((fact) => fact.id);
963
- const existingFactsById = /* @__PURE__ */ new Map();
964
- const factLookupChunkSize = 500;
965
- for (let i = 0; i < factIds.length; i += factLookupChunkSize) {
966
- const factIdChunk = factIds.slice(i, i + factLookupChunkSize);
967
- if (factIdChunk.length === 0) continue;
968
- const placeholders = factIdChunk.map(() => "?").join(", ");
969
- const existingFacts = await this.db.getAllAsync(
970
- `SELECT id, entity_id, updated_at FROM ${this.prefix}entries WHERE id IN (${placeholders})`,
971
- factIdChunk
972
- );
973
- for (const existingFact of existingFacts) {
974
- existingFactsById.set(existingFact.id, existingFact);
975
- }
976
- }
977
- for (const fact of bundle.facts) {
978
- const tagsJson = JSON.stringify(Array.isArray(fact.tags) ? fact.tags : []);
979
- const safeUpdatedAt = Number.isFinite(fact.updated_at) ? fact.updated_at : 0;
980
- const existing = existingFactsById.get(fact.id);
981
- if (existing) {
982
- if (existing.entity_id !== entityId) {
983
- this._warnCrossEntityCollision("entry", fact.id, existing.entity_id, entityId);
984
- continue;
985
- }
986
- if (merge) {
987
- if (safeUpdatedAt <= existing.updated_at) continue;
988
- }
989
- await this.db.runAsync(
990
- `UPDATE ${this.prefix}entries SET entity_id = ?, title = ?, body = ?, tags = ?, confidence = ?, source_type = ?, source_hash = ?, source_ref = ?, created_at = ?, updated_at = ?, last_accessed_at = ?, access_count = ?, deleted_at = ? WHERE id = ?`,
991
- [entityId, fact.title, fact.body, tagsJson, fact.confidence, fact.source_type, fact.source_hash, fact.source_ref, fact.created_at, safeUpdatedAt, fact.last_accessed_at, fact.access_count, fact.deleted_at, fact.id]
992
- );
993
- existingFactsById.set(fact.id, { id: fact.id, entity_id: entityId, updated_at: safeUpdatedAt });
994
- } else {
995
- await this.db.runAsync(
996
- `INSERT INTO ${this.prefix}entries (id, entity_id, title, body, tags, confidence, source_type, source_hash, source_ref, created_at, updated_at, last_accessed_at, access_count, deleted_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
997
- [fact.id, entityId, fact.title, fact.body, tagsJson, fact.confidence, fact.source_type, fact.source_hash, fact.source_ref, fact.created_at, safeUpdatedAt, fact.last_accessed_at, fact.access_count, fact.deleted_at]
998
- );
999
- existingFactsById.set(fact.id, { id: fact.id, entity_id: entityId, updated_at: safeUpdatedAt });
1000
- }
1001
- }
1002
- const taskIds = bundle.tasks.map((task) => task.id);
1003
- const existingTasksById = /* @__PURE__ */ new Map();
1004
- const taskLookupChunkSize = 500;
1005
- for (let i = 0; i < taskIds.length; i += taskLookupChunkSize) {
1006
- const taskIdChunk = taskIds.slice(i, i + taskLookupChunkSize);
1007
- if (taskIdChunk.length === 0) continue;
1008
- const placeholders = taskIdChunk.map(() => "?").join(", ");
1009
- const existingTasks = await this.db.getAllAsync(
1010
- `SELECT id, entity_id, updated_at FROM ${this.prefix}tasks WHERE id IN (${placeholders})`,
1011
- taskIdChunk
1012
- );
1013
- for (const existingTask of existingTasks) {
1014
- existingTasksById.set(existingTask.id, existingTask);
1015
- }
1016
- }
1017
- for (const task of bundle.tasks) {
1018
- const safeUpdatedAt = Number.isFinite(task.updated_at) ? task.updated_at : 0;
1019
- const existing = existingTasksById.get(task.id);
1020
- if (existing) {
1021
- if (existing.entity_id !== entityId) {
1022
- this._warnCrossEntityCollision("task", task.id, existing.entity_id, entityId);
1023
- continue;
1024
- }
1025
- if (merge) {
1026
- if (safeUpdatedAt <= existing.updated_at) continue;
1027
- }
1028
- await this.db.runAsync(
1029
- `UPDATE ${this.prefix}tasks SET entity_id = ?, description = ?, status = ?, priority = ?, created_at = ?, updated_at = ?, resolved_at = ?, deleted_at = ? WHERE id = ?`,
1030
- [entityId, task.description, task.status, task.priority, task.created_at, safeUpdatedAt, task.resolved_at, task.deleted_at, task.id]
1031
- );
1032
- existingTasksById.set(task.id, { id: task.id, entity_id: entityId, updated_at: safeUpdatedAt });
1033
- } else {
1034
- await this.db.runAsync(
1035
- `INSERT INTO ${this.prefix}tasks (id, entity_id, description, status, priority, created_at, updated_at, resolved_at, deleted_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
1036
- [task.id, entityId, task.description, task.status, task.priority, task.created_at, safeUpdatedAt, task.resolved_at, task.deleted_at]
1037
- );
1038
- existingTasksById.set(task.id, { id: task.id, entity_id: entityId, updated_at: safeUpdatedAt });
1039
- }
1040
- }
1041
- for (const event of bundle.events) {
1042
- await this.db.runAsync(
1043
- `INSERT OR IGNORE INTO ${this.prefix}events (id, entity_id, event_type, summary, related_entry_id, created_at)
1044
- VALUES (?, ?, ?, ?, ?, ?)`,
1045
- [event.id, entityId, event.event_type, event.summary, event.related_entry_id ?? null, event.created_at]
1046
- );
1047
- }
1048
- });
1049
- }
1050
- }
1051
- async forget(entityId, params) {
1052
- const now = Date.now();
1053
- let deletedEntries = 0;
1054
- let deletedTasks = 0;
1055
- if (params.clearAll) {
1056
- const [entriesRes, tasksRes] = await Promise.all([
1057
- this.db.runAsync(`UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE entity_id = ? AND deleted_at IS NULL`, [now, now, entityId]),
1058
- this.db.runAsync(`UPDATE ${this.prefix}tasks SET deleted_at = ?, updated_at = ? WHERE entity_id = ? AND deleted_at IS NULL`, [now, now, entityId])
1059
- ]);
1060
- await this.db.runAsync(`UPDATE ${this.prefix}checkpoints SET memory_checkpoint = 0, heal_checkpoint = 0 WHERE entity_id = ?`, [entityId]);
1061
- deletedEntries = entriesRes.changes;
1062
- deletedTasks = tasksRes.changes;
1063
- } else {
1064
- const hasIdSelectors = params.entryId !== void 0 || params.taskId !== void 0;
1065
- const hasSourceSelectors = params.sourceRef !== void 0 || params.sourceHash !== void 0;
1066
- if (hasIdSelectors && hasSourceSelectors) {
1067
- throw new Error("forget() params are mutually exclusive: use entryId/taskId together, or sourceRef/sourceHash together, but not both in the same call");
1068
- }
1069
- const sourceRef = params.sourceRef !== void 0 ? normalizeSourceRef(params.sourceRef) : null;
1070
- if (params.sourceRef !== void 0 && !sourceRef) throw new Error("Invalid sourceRef");
1071
- const sourceHash = params.sourceHash !== void 0 ? normalizeSourceHash(params.sourceHash) : null;
1072
- if (params.sourceHash !== void 0 && !sourceHash) throw new Error("Invalid sourceHash (must be 64-char hex string)");
1073
- const entryPromise = params.entryId ? this.db.runAsync(`UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE id = ? AND entity_id = ? AND deleted_at IS NULL`, [now, now, params.entryId, entityId]) : null;
1074
- const taskPromise = params.taskId ? this.db.runAsync(`UPDATE ${this.prefix}tasks SET deleted_at = ?, updated_at = ? WHERE id = ? AND entity_id = ? AND deleted_at IS NULL`, [now, now, params.taskId, entityId]) : null;
1075
- let refPromise = null;
1076
- if (sourceRef || sourceHash) {
1077
- let q = `UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE entity_id = ? AND deleted_at IS NULL`;
1078
- const args = [now, now, entityId];
1079
- if (sourceRef) {
1080
- q += ` AND source_ref = ?`;
1081
- args.push(sourceRef);
1082
- }
1083
- if (sourceHash) {
1084
- q += ` AND source_hash = ?`;
1085
- args.push(sourceHash);
1086
- }
1087
- refPromise = this.db.runAsync(q, args);
1088
- }
1089
- const [entryResult, taskResult, refResult] = await Promise.all([
1090
- entryPromise ?? Promise.resolve(null),
1091
- taskPromise ?? Promise.resolve(null),
1092
- refPromise ?? Promise.resolve(null)
1093
- ]);
1094
- if (entryResult) deletedEntries += entryResult.changes;
1095
- if (taskResult) deletedTasks += taskResult.changes;
1096
- if (refResult) deletedEntries += refResult.changes;
1097
- }
1098
- return { deleted: { entries: deletedEntries, tasks: deletedTasks } };
1099
- }
1100
- async ingestDocument(entityId, params) {
1101
- const sourceRef = normalizeSourceRef(params.sourceRef);
1102
- if (!sourceRef) throw new Error("Invalid sourceRef");
1103
- const sourceHash = normalizeSourceHash(params.sourceHash);
1104
- if (!sourceHash) throw new Error("Invalid sourceHash (must be 64-char hex string)");
1105
- const maxChunkLength = params.maxChunkLength ?? this.options.config?.maxChunkLength ?? 12e3;
1106
- const rawOverlap = params.chunkOverlap ?? this.options.config?.chunkOverlap ?? 400;
1107
- const chunkOverlap = Math.min(
1108
- Number.isFinite(rawOverlap) && rawOverlap >= 0 ? Math.floor(rawOverlap) : 400,
1109
- maxChunkLength - 1
1110
- );
1111
- const rawConcurrency = params.chunkConcurrency ?? this.options.config?.chunkConcurrency ?? 1;
1112
- const chunkConcurrency = Number.isFinite(rawConcurrency) && rawConcurrency >= 1 ? Math.floor(rawConcurrency) : 1;
1113
- if (typeof params.documentChunk !== "string") {
1114
- throw new Error(`documentChunk must be a string, received ${typeof params.documentChunk}`);
1115
- }
1116
- const jobKey = `${this.prefix}:${entityId}:${sourceRef}`;
1117
- if (this.activeIngestJobs.has(jobKey)) {
1118
- throw new WikiBusyError("ingest", entityId);
1119
- }
1120
- if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) {
1121
- throw new WikiBusyError("prune", entityId);
1122
- }
1123
- this.activeIngestJobs.add(jobKey);
1124
- try {
1125
- const { chunks, truncated } = chunkText(params.documentChunk, maxChunkLength, chunkOverlap);
1126
- if (chunks.length === 0) {
1127
- return { truncated: false, chunks: 0 };
1128
- }
1129
- const chunkResults = await withConcurrency(
1130
- chunks.map((chunk) => async () => {
1131
- const userPrompt = `Document Chunk:
1132
- ${chunk}`;
1133
- const responseText = await this.options.llmProvider.generateText({
1134
- systemPrompt: INGEST_SYSTEM_PROMPT,
1135
- userPrompt
1136
- });
1137
- const result = parseJsonResponse(responseText);
1138
- return (Array.isArray(result.facts) ? result.facts : []).map(validateFact).filter((f) => f !== null);
1139
- }),
1140
- chunkConcurrency
1141
- );
1142
- const seen = /* @__PURE__ */ new Set();
1143
- const allValidFacts = [];
1144
- for (const facts of chunkResults) {
1145
- for (const fact of facts) {
1146
- const normalized = fact.title.trim().toLowerCase().replace(/\s+/g, " ");
1147
- if (!seen.has(normalized)) {
1148
- seen.add(normalized);
1149
- allValidFacts.push(fact);
1150
- }
1151
- }
1152
- }
1153
- const now = Date.now();
1154
- await this.db.withTransactionAsync(async () => {
1155
- await this.db.runAsync(
1156
- `UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE source_ref = ? AND entity_id = ? AND deleted_at IS NULL`,
1157
- [now, now, sourceRef, entityId]
1158
- );
1159
- for (const fact of allValidFacts) {
1160
- const id = generateId("fact_");
1161
- await this.db.runAsync(
1162
- `INSERT INTO ${this.prefix}entries (id, entity_id, title, body, tags, confidence, source_type, source_hash, source_ref, created_at, updated_at)
1163
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
1164
- [id, entityId, fact.title, fact.body, JSON.stringify(fact.tags), fact.confidence, "user_document", sourceHash, sourceRef, now, now]
1165
- );
1166
- }
1167
- });
1168
- return { truncated, chunks: chunks.length };
1169
- } finally {
1170
- this.activeIngestJobs.delete(jobKey);
1171
- }
1172
- }
1173
- };
1174
-
1175
- // src/utils/formatMemoryDump.ts
1176
- function renderFact(f) {
1177
- const tags = (f.tags || []).join(", ");
1178
- const source = f.source_ref ?? f.source_type;
1179
- return `### ${f.title}
1180
- **Tags:** ${tags}
1181
- **Confidence:** ${f.confidence}
1182
- **Source:** ${source}
1183
-
1184
- ${f.body}
1185
-
1186
- ---
1187
- `;
1188
- }
1189
- function renderTask(t) {
1190
- const checked = t.status === "done" ? "x" : " ";
1191
- const note = t.status === "done" ? " (done)" : t.status === "abandoned" ? " (abandoned)" : t.status === "in_progress" ? " (in progress)" : "";
1192
- return `- [${checked}] ${t.description}${note}
1193
- `;
1194
- }
1195
- function renderEvent(e) {
1196
- const ts = new Date(e.created_at).toISOString();
1197
- return `- [${ts}] (${e.event_type}) ${e.summary}
1198
- `;
1199
- }
1200
- function renderEntity(entityId, bundle, generatedAt) {
1201
- const lines = [];
1202
- lines.push(`# Memory Dump: ${entityId}`);
1203
- lines.push(`Generated: ${new Date(generatedAt).toISOString()}`);
1204
- lines.push("");
1205
- lines.push("## Facts");
1206
- lines.push("");
1207
- if (bundle.facts.length === 0) {
1208
- lines.push("_(none)_\n");
1209
- } else {
1210
- for (const f of bundle.facts) lines.push(renderFact(f));
1211
- }
1212
- lines.push("## Tasks");
1213
- lines.push("");
1214
- if (bundle.tasks.length === 0) {
1215
- lines.push("_(none)_\n");
1216
- } else {
1217
- for (const t of bundle.tasks) lines.push(renderTask(t));
1218
- }
1219
- lines.push("");
1220
- lines.push("## Recent Events");
1221
- lines.push("");
1222
- if (bundle.events.length === 0) {
1223
- lines.push("_(none)_\n");
1224
- } else {
1225
- for (const e of bundle.events) lines.push(renderEvent(e));
1226
- }
1227
- return lines.join("\n");
1228
- }
1229
- function shortHash(value) {
1230
- let h1 = 5381;
1231
- let h2 = 52711;
1232
- for (let i = 0; i < value.length; i += 1) {
1233
- const c = value.charCodeAt(i);
1234
- h1 = Math.imul(h1, 33) ^ c;
1235
- h2 = Math.imul(h2, 31) ^ c;
1236
- }
1237
- return (h1 >>> 0).toString(16).padStart(8, "0") + (h2 >>> 0).toString(16).padStart(8, "0");
1238
- }
1239
- function formatEntityFileName(entityId) {
1240
- const normalized = entityId.normalize("NFKC");
1241
- const sanitized = normalized.replace(/[^A-Za-z0-9._-]+/g, "_").replace(/^\.+/, "_").replace(/_+/g, "_").replace(/^[_-]+|[_-]+$/g, "");
1242
- const MAX_BASE = 200;
1243
- const trimmed = sanitized.length > MAX_BASE ? sanitized.slice(0, MAX_BASE) : sanitized;
1244
- const baseName = trimmed && trimmed !== "." && trimmed !== ".." ? trimmed : "entity";
1245
- const needsSuffix = baseName !== entityId || sanitized.length > MAX_BASE;
1246
- const uniqueBaseName = needsSuffix ? `${baseName}-${shortHash(entityId)}` : baseName;
1247
- return `${uniqueBaseName}.md`;
1248
- }
1249
- function formatMemoryDump(dump) {
1250
- const files = Object.entries(dump.entities).map(([entityId, bundle]) => ({
1251
- name: formatEntityFileName(entityId),
1252
- content: renderEntity(entityId, bundle, dump.generatedAt)
1253
- }));
1254
- return {
1255
- manifest: JSON.stringify(dump, null, 2),
1256
- files
1257
- };
1258
- }
1259
-
1260
- // src/utils/formatContext.ts
1261
- function validateMaxOption(value, name) {
1262
- if (!isFinite(value) || value < 0) {
1263
- throw new Error(`Invalid ${name}: must be a non-negative finite number`);
1264
- }
1265
- }
1266
- var CONFIDENCE_WEIGHT = {
1267
- certain: 1,
1268
- inferred: 0.6,
1269
- tentative: 0.3
1270
- };
1271
- function scoreFactFor(fact, weights, now) {
1272
- const confW = CONFIDENCE_WEIGHT[fact.confidence] ?? 0.3;
1273
- const ageDays = (now - fact.updated_at) / 864e5;
1274
- const recencyDecay = Math.exp(-ageDays / 30);
1275
- return confW * weights.confidence + Math.log(1 + fact.access_count) * weights.accessCount + recencyDecay * weights.recency;
1276
- }
1277
- function renderFactMarkdown(fact, includeConfidence, includeTags) {
1278
- const confPart = includeConfidence ? ` (${fact.confidence})` : "";
1279
- const tagPart = includeTags && fact.tags.length > 0 ? ` [${fact.tags.join(", ")}]` : "";
1280
- return `- **${fact.title}**${confPart}${tagPart}
1281
- ${fact.body.replace(/\n/g, "\n ")}`;
1282
- }
1283
- function renderFactPlain(fact, includeConfidence, includeTags) {
1284
- const confPart = includeConfidence ? ` (${fact.confidence})` : "";
1285
- const tagPart = includeTags && fact.tags.length > 0 ? ` [${fact.tags.join(", ")}]` : "";
1286
- return `${fact.title}${confPart}${tagPart}: ${fact.body}`;
1287
- }
1288
- function renderTaskMarkdown(task) {
1289
- return `- [P${task.priority}] ${task.description.replace(/\n/g, "\n ")} (${task.status})`;
1290
- }
1291
- function renderTaskPlain(task) {
1292
- return `[P${task.priority}] ${task.description} (${task.status})`;
1293
- }
1294
- function renderEventMarkdown(event) {
1295
- const ts = new Date(event.created_at).toISOString();
1296
- return `- [${event.event_type} @ ${ts}] ${event.summary.replace(/\n/g, "\n ")}`;
1297
- }
1298
- function renderEventPlain(event) {
1299
- const ts = new Date(event.created_at).toISOString();
1300
- return `[${event.event_type} @ ${ts}] ${event.summary}`;
1301
- }
1302
- function formatContext(bundle, options) {
1303
- const opts = {
1304
- format: options?.format ?? "markdown",
1305
- maxFacts: options?.maxFacts ?? 10,
1306
- maxTasks: options?.maxTasks ?? 10,
1307
- maxEvents: options?.maxEvents ?? 10,
1308
- includeConfidence: options?.includeConfidence ?? true,
1309
- includeTags: options?.includeTags ?? true,
1310
- factWeights: {
1311
- confidence: options?.factWeights?.confidence ?? 1,
1312
- accessCount: options?.factWeights?.accessCount ?? 0.3,
1313
- recency: options?.factWeights?.recency ?? 0.5
1314
- }
1315
- };
1316
- validateMaxOption(opts.maxFacts, "maxFacts");
1317
- validateMaxOption(opts.maxTasks, "maxTasks");
1318
- validateMaxOption(opts.maxEvents, "maxEvents");
1319
- const weights = opts.factWeights;
1320
- const now = Date.now();
1321
- const sortedFacts = [...bundle.facts].sort((a, b) => scoreFactFor(b, weights, now) - scoreFactFor(a, weights, now)).slice(0, opts.maxFacts);
1322
- const sortedTasks = [...bundle.tasks].sort((a, b) => b.priority - a.priority || a.created_at - b.created_at).slice(0, opts.maxTasks);
1323
- const sortedEvents = [...bundle.events].sort((a, b) => b.created_at - a.created_at).slice(0, opts.maxEvents);
1324
- if (sortedFacts.length === 0 && sortedTasks.length === 0 && sortedEvents.length === 0) {
1325
- return "";
1326
- }
1327
- const isMarkdown = opts.format === "markdown";
1328
- const lines = [];
1329
- if (isMarkdown) {
1330
- lines.push("## Memory");
1331
- if (sortedFacts.length > 0) {
1332
- lines.push("");
1333
- lines.push("### Known Facts");
1334
- for (const fact of sortedFacts) {
1335
- lines.push(renderFactMarkdown(fact, opts.includeConfidence, opts.includeTags));
1336
- }
1337
- }
1338
- if (sortedTasks.length > 0) {
1339
- lines.push("");
1340
- lines.push("### Open Tasks");
1341
- for (const task of sortedTasks) {
1342
- lines.push(renderTaskMarkdown(task));
1343
- }
1344
- }
1345
- if (sortedEvents.length > 0) {
1346
- lines.push("");
1347
- lines.push("### Recent Events");
1348
- for (const event of sortedEvents) {
1349
- lines.push(renderEventMarkdown(event));
1350
- }
1351
- }
1352
- } else {
1353
- if (sortedFacts.length > 0) {
1354
- lines.push("KNOWN FACTS:");
1355
- for (const fact of sortedFacts) {
1356
- lines.push(renderFactPlain(fact, opts.includeConfidence, opts.includeTags));
1357
- }
1358
- }
1359
- if (sortedTasks.length > 0) {
1360
- lines.push("OPEN TASKS:");
1361
- for (const task of sortedTasks) {
1362
- lines.push(renderTaskPlain(task));
1363
- }
1364
- }
1365
- if (sortedEvents.length > 0) {
1366
- lines.push("RECENT EVENTS:");
1367
- for (const event of sortedEvents) {
1368
- lines.push(renderEventPlain(event));
1369
- }
1370
- }
1371
- }
1372
- return lines.join("\n");
1373
- }
1
+ import {
2
+ createExpoAdapter
3
+ } from "./chunk-6DVBBIYR.mjs";
1374
4
 
1375
5
  // src/index.ts
6
+ import { WikiMemory } from "@equationalapplications/core-llm-wiki";
7
+ export * from "@equationalapplications/core-llm-wiki";
8
+ export * from "@equationalapplications/react-llm-wiki";
1376
9
  function createWiki(db, options) {
1377
- return new WikiMemory(db, options);
10
+ return new WikiMemory(createExpoAdapter(db), options);
1378
11
  }
1379
12
  export {
1380
- WikiBusyError,
1381
- WikiMemory,
1382
- createWiki,
1383
- formatContext,
1384
- formatMemoryDump
13
+ createWiki
1385
14
  };
15
+ //# sourceMappingURL=index.mjs.map