@equationalapplications/expo-llm-wiki 0.0.0-development → 2.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -1
- package/dist/{WikiMemory-BI2Aizwv.d.mts → WikiMemory-B54Gaa-K.d.mts} +37 -1
- package/dist/{WikiMemory-BI2Aizwv.d.ts → WikiMemory-B54Gaa-K.d.ts} +37 -1
- package/dist/index.d.mts +5 -3
- package/dist/index.d.ts +5 -3
- package/dist/index.js +441 -65
- package/dist/index.mjs +438 -64
- package/dist/react/index.d.mts +9 -2
- package/dist/react/index.d.ts +9 -2
- package/dist/react/index.js +30 -0
- package/dist/react/index.mjs +29 -0
- package/package.json +9 -5
- package/dist/WikiMemory-B-yFw9Dc.d.mts +0 -118
- package/dist/WikiMemory-B-yFw9Dc.d.ts +0 -118
- package/dist/WikiMemory-BWTt1Ynm.d.mts +0 -103
- package/dist/WikiMemory-BWTt1Ynm.d.ts +0 -103
package/dist/index.mjs
CHANGED
|
@@ -83,11 +83,23 @@ async function setupDatabase(db, prefix) {
|
|
|
83
83
|
`);
|
|
84
84
|
}
|
|
85
85
|
|
|
86
|
+
// src/types.ts
|
|
87
|
+
var WikiBusyError = class extends Error {
|
|
88
|
+
operation;
|
|
89
|
+
entityId;
|
|
90
|
+
constructor(operation, entityId) {
|
|
91
|
+
super(`${operation} already running for entity ${entityId}`);
|
|
92
|
+
this.name = "WikiBusyError";
|
|
93
|
+
this.operation = operation;
|
|
94
|
+
this.entityId = entityId;
|
|
95
|
+
}
|
|
96
|
+
};
|
|
97
|
+
|
|
86
98
|
// src/prompts.ts
|
|
87
99
|
var LIBRARIAN_SYSTEM_PROMPT = `You are a knowledge extraction agent. Your job is to analyze recent episodic events and extract stable facts and actionable tasks about the user or entity.
|
|
88
100
|
Return ONLY a valid JSON object matching this schema:
|
|
89
101
|
{
|
|
90
|
-
"facts": [{ "title": "string (max 80 chars)", "body": "string (max
|
|
102
|
+
"facts": [{ "title": "string (max 80 chars)", "body": "string (max 800 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative" }],
|
|
91
103
|
"tasks": [{ "description": "string", "priority": "number (0-10)" }]
|
|
92
104
|
}
|
|
93
105
|
Keep facts concise. Do not return markdown, just raw JSON.`;
|
|
@@ -96,13 +108,13 @@ Return ONLY a valid JSON object matching this schema:
|
|
|
96
108
|
{
|
|
97
109
|
"downgraded": ["string (fact IDs)"],
|
|
98
110
|
"deleted": ["string (fact IDs)"],
|
|
99
|
-
"newFacts": [{ "title": "string", "body": "string", "tags": ["string"], "confidence": "certain|inferred|tentative" }]
|
|
111
|
+
"newFacts": [{ "title": "string (max 80 chars)", "body": "string (max 800 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative" }]
|
|
100
112
|
}
|
|
101
113
|
Do not return markdown, just raw JSON.`;
|
|
102
114
|
var INGEST_SYSTEM_PROMPT = `You are a document ingestion agent. Your job is to extract factual knowledge from the provided document chunk.
|
|
103
115
|
Return ONLY a valid JSON object matching this schema:
|
|
104
116
|
{
|
|
105
|
-
"facts": [{ "title": "string (max 80 chars)", "body": "string (max
|
|
117
|
+
"facts": [{ "title": "string (max 80 chars)", "body": "string (max 800 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative" }]
|
|
106
118
|
}
|
|
107
119
|
Extract verbatim factual content. Do not return markdown, just raw JSON.`;
|
|
108
120
|
|
|
@@ -176,6 +188,84 @@ function safeSlice(value, start, end) {
|
|
|
176
188
|
}
|
|
177
189
|
return value.slice(safeStart, safeEnd);
|
|
178
190
|
}
|
|
191
|
+
function chunkText(input, maxChunkLength, overlap) {
|
|
192
|
+
const text = input.trim();
|
|
193
|
+
if (text.length === 0) return { chunks: [], truncated: false };
|
|
194
|
+
if (!Number.isInteger(maxChunkLength) || maxChunkLength < 2) {
|
|
195
|
+
throw new Error("maxChunkLength must be an integer >= 2");
|
|
196
|
+
}
|
|
197
|
+
if (!Number.isInteger(overlap) || overlap < 0 || overlap >= maxChunkLength) {
|
|
198
|
+
throw new Error("overlap must be a non-negative integer < maxChunkLength");
|
|
199
|
+
}
|
|
200
|
+
const chunks = [];
|
|
201
|
+
let truncated = false;
|
|
202
|
+
let cursor = 0;
|
|
203
|
+
const halfMax = Math.floor(maxChunkLength / 2);
|
|
204
|
+
while (cursor < text.length) {
|
|
205
|
+
const remaining = text.length - cursor;
|
|
206
|
+
if (remaining <= maxChunkLength) {
|
|
207
|
+
chunks.push(safeSlice(text, cursor, text.length));
|
|
208
|
+
break;
|
|
209
|
+
}
|
|
210
|
+
const windowEnd = cursor + maxChunkLength;
|
|
211
|
+
const minSplit = cursor + halfMax;
|
|
212
|
+
let splitPoint = -1;
|
|
213
|
+
const paraIdx = text.lastIndexOf("\n\n", windowEnd);
|
|
214
|
+
if (paraIdx >= minSplit && paraIdx + 2 <= windowEnd) {
|
|
215
|
+
splitPoint = paraIdx + 2;
|
|
216
|
+
}
|
|
217
|
+
if (splitPoint === -1) {
|
|
218
|
+
let lastTerm = -1;
|
|
219
|
+
for (let i = minSplit; i < windowEnd - 1; i++) {
|
|
220
|
+
const ch = text[i];
|
|
221
|
+
if ((ch === "." || ch === "!" || ch === "?") && /\s/.test(text[i + 1])) {
|
|
222
|
+
lastTerm = i + 2;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
if (lastTerm !== -1 && lastTerm <= windowEnd) splitPoint = lastTerm;
|
|
226
|
+
}
|
|
227
|
+
if (splitPoint === -1) {
|
|
228
|
+
for (let i = windowEnd - 1; i >= minSplit; i--) {
|
|
229
|
+
if (/\s/.test(text[i])) {
|
|
230
|
+
splitPoint = i + 1;
|
|
231
|
+
break;
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
if (splitPoint === -1) {
|
|
236
|
+
truncated = true;
|
|
237
|
+
splitPoint = windowEnd;
|
|
238
|
+
}
|
|
239
|
+
chunks.push(safeSlice(text, cursor, splitPoint));
|
|
240
|
+
const next = Math.max(splitPoint - overlap, cursor + 1);
|
|
241
|
+
cursor = next;
|
|
242
|
+
}
|
|
243
|
+
return { chunks, truncated };
|
|
244
|
+
}
|
|
245
|
+
async function withConcurrency(tasks, limit) {
|
|
246
|
+
const results = new Array(tasks.length);
|
|
247
|
+
let index = 0;
|
|
248
|
+
let failed = false;
|
|
249
|
+
let firstError;
|
|
250
|
+
async function worker() {
|
|
251
|
+
while (index < tasks.length && !failed) {
|
|
252
|
+
const i = index++;
|
|
253
|
+
try {
|
|
254
|
+
results[i] = await tasks[i]();
|
|
255
|
+
} catch (e) {
|
|
256
|
+
if (!failed) {
|
|
257
|
+
failed = true;
|
|
258
|
+
firstError = e;
|
|
259
|
+
}
|
|
260
|
+
return;
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
const workerCount = tasks.length === 0 ? 0 : Math.min(Math.max(limit, 1), tasks.length);
|
|
265
|
+
await Promise.allSettled(Array.from({ length: workerCount }, worker));
|
|
266
|
+
if (failed) throw firstError;
|
|
267
|
+
return results;
|
|
268
|
+
}
|
|
179
269
|
function clip(value, max) {
|
|
180
270
|
if (typeof value !== "string") return "";
|
|
181
271
|
const s = value.trim();
|
|
@@ -188,7 +278,7 @@ function validateTags(tags) {
|
|
|
188
278
|
function validateFact(fact) {
|
|
189
279
|
if (typeof fact?.title !== "string" || typeof fact?.body !== "string") return null;
|
|
190
280
|
const title = clip(fact.title, 80);
|
|
191
|
-
const body = clip(fact.body,
|
|
281
|
+
const body = clip(fact.body, 800);
|
|
192
282
|
if (!title || !body) return null;
|
|
193
283
|
let confidence = fact.confidence;
|
|
194
284
|
if (confidence !== "certain" && confidence !== "tentative") confidence = "inferred";
|
|
@@ -237,6 +327,16 @@ var WikiMemory = class {
|
|
|
237
327
|
prefix;
|
|
238
328
|
options;
|
|
239
329
|
activeMaintenanceJobs = /* @__PURE__ */ new Set();
|
|
330
|
+
activeIngestJobs = /* @__PURE__ */ new Set();
|
|
331
|
+
_librarianKey(entityId) {
|
|
332
|
+
return `${this.prefix}:${entityId}:librarian`;
|
|
333
|
+
}
|
|
334
|
+
_healKey(entityId) {
|
|
335
|
+
return `${this.prefix}:${entityId}:heal`;
|
|
336
|
+
}
|
|
337
|
+
_warnCrossEntityCollision(type, id, existingEntityId, targetEntityId) {
|
|
338
|
+
console.warn(`[WikiMemory] importDump: ${type} id "${id}" already belongs to entity "${existingEntityId}"; skipping for entity "${targetEntityId}"`);
|
|
339
|
+
}
|
|
240
340
|
constructor(db, options) {
|
|
241
341
|
this.db = db;
|
|
242
342
|
this.options = options;
|
|
@@ -322,6 +422,9 @@ var WikiMemory = class {
|
|
|
322
422
|
}));
|
|
323
423
|
return { facts, tasks, events: events.reverse() };
|
|
324
424
|
}
|
|
425
|
+
async getMemoryBundle(entityId) {
|
|
426
|
+
return this._getFullBundle(entityId, { maxEvents: 10 });
|
|
427
|
+
}
|
|
325
428
|
async write(entityId, event) {
|
|
326
429
|
const id = generateId("evt_");
|
|
327
430
|
const now = Date.now();
|
|
@@ -342,7 +445,7 @@ var WikiMemory = class {
|
|
|
342
445
|
let memoryCheckpoint = cp?.memory_checkpoint || 0;
|
|
343
446
|
if (memoryCheckpoint > count) memoryCheckpoint = 0;
|
|
344
447
|
if (count - memoryCheckpoint >= threshold) {
|
|
345
|
-
const jobKey =
|
|
448
|
+
const jobKey = this._librarianKey(entityId);
|
|
346
449
|
if (!this.activeMaintenanceJobs.has(jobKey)) {
|
|
347
450
|
this.activeMaintenanceJobs.add(jobKey);
|
|
348
451
|
this.runLibrarianThenMaybeHeal(entityId, count).catch(console.error).finally(() => this.activeMaintenanceJobs.delete(jobKey));
|
|
@@ -361,12 +464,20 @@ var WikiMemory = class {
|
|
|
361
464
|
let healCheckpoint = cp?.heal_checkpoint || 0;
|
|
362
465
|
if (healCheckpoint > currentEventCount) healCheckpoint = 0;
|
|
363
466
|
if (currentEventCount - healCheckpoint >= autoHealThreshold) {
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
467
|
+
const healKey = this._healKey(entityId);
|
|
468
|
+
if (!this.activeMaintenanceJobs.has(healKey)) {
|
|
469
|
+
this.activeMaintenanceJobs.add(healKey);
|
|
470
|
+
try {
|
|
471
|
+
await this._doRunHeal(entityId);
|
|
472
|
+
await this.db.runAsync(`
|
|
473
|
+
INSERT INTO ${this.prefix}checkpoints (entity_id, heal_checkpoint)
|
|
474
|
+
VALUES (?, ?)
|
|
475
|
+
ON CONFLICT(entity_id) DO UPDATE SET heal_checkpoint = ?
|
|
476
|
+
`, [entityId, currentEventCount, currentEventCount]);
|
|
477
|
+
} finally {
|
|
478
|
+
this.activeMaintenanceJobs.delete(healKey);
|
|
479
|
+
}
|
|
480
|
+
}
|
|
370
481
|
}
|
|
371
482
|
}
|
|
372
483
|
async _doRunLibrarian(entityId) {
|
|
@@ -509,8 +620,10 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
509
620
|
});
|
|
510
621
|
}
|
|
511
622
|
async runLibrarian(entityId) {
|
|
512
|
-
const jobKey =
|
|
513
|
-
if (this.activeMaintenanceJobs.has(jobKey))
|
|
623
|
+
const jobKey = this._librarianKey(entityId);
|
|
624
|
+
if (this.activeMaintenanceJobs.has(jobKey)) {
|
|
625
|
+
throw new WikiBusyError("librarian", entityId);
|
|
626
|
+
}
|
|
514
627
|
this.activeMaintenanceJobs.add(jobKey);
|
|
515
628
|
try {
|
|
516
629
|
await this._doRunLibrarian(entityId);
|
|
@@ -519,8 +632,10 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
519
632
|
}
|
|
520
633
|
}
|
|
521
634
|
async runHeal(entityId) {
|
|
522
|
-
const jobKey =
|
|
523
|
-
if (this.activeMaintenanceJobs.has(jobKey))
|
|
635
|
+
const jobKey = this._healKey(entityId);
|
|
636
|
+
if (this.activeMaintenanceJobs.has(jobKey)) {
|
|
637
|
+
throw new WikiBusyError("heal", entityId);
|
|
638
|
+
}
|
|
524
639
|
this.activeMaintenanceJobs.add(jobKey);
|
|
525
640
|
try {
|
|
526
641
|
await this._doRunHeal(entityId);
|
|
@@ -528,6 +643,170 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
528
643
|
this.activeMaintenanceJobs.delete(jobKey);
|
|
529
644
|
}
|
|
530
645
|
}
|
|
646
|
+
getEntityStatus(entityId) {
|
|
647
|
+
const ingestPrefix = `${this.prefix}:${entityId}:`;
|
|
648
|
+
let ingesting = false;
|
|
649
|
+
for (const k of this.activeIngestJobs) {
|
|
650
|
+
if (k.startsWith(ingestPrefix)) {
|
|
651
|
+
ingesting = true;
|
|
652
|
+
break;
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
return {
|
|
656
|
+
ingesting,
|
|
657
|
+
librarian: this.activeMaintenanceJobs.has(this._librarianKey(entityId)),
|
|
658
|
+
heal: this.activeMaintenanceJobs.has(this._healKey(entityId))
|
|
659
|
+
};
|
|
660
|
+
}
|
|
661
|
+
async _getFullBundle(entityId, opts) {
|
|
662
|
+
const maxEvents = opts?.maxEvents;
|
|
663
|
+
const eventsQuery = maxEvents != null ? `SELECT * FROM ${this.prefix}events WHERE entity_id = ? ORDER BY created_at DESC LIMIT ?` : `SELECT * FROM ${this.prefix}events WHERE entity_id = ? ORDER BY created_at ASC`;
|
|
664
|
+
const eventsParams = maxEvents != null ? [entityId, maxEvents] : [entityId];
|
|
665
|
+
const [factsRaw, tasks, eventsRaw] = await Promise.all([
|
|
666
|
+
this.db.getAllAsync(
|
|
667
|
+
`SELECT * FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL ORDER BY updated_at DESC`,
|
|
668
|
+
[entityId]
|
|
669
|
+
),
|
|
670
|
+
this.db.getAllAsync(
|
|
671
|
+
`SELECT * FROM ${this.prefix}tasks WHERE entity_id = ? AND deleted_at IS NULL ORDER BY priority DESC, created_at ASC`,
|
|
672
|
+
[entityId]
|
|
673
|
+
),
|
|
674
|
+
this.db.getAllAsync(eventsQuery, eventsParams)
|
|
675
|
+
]);
|
|
676
|
+
const facts = factsRaw.map((f) => ({
|
|
677
|
+
...f,
|
|
678
|
+
tags: typeof f.tags === "string" ? JSON.parse(f.tags) : f.tags
|
|
679
|
+
}));
|
|
680
|
+
const events = maxEvents != null ? eventsRaw.slice().reverse() : eventsRaw;
|
|
681
|
+
return { facts, tasks, events };
|
|
682
|
+
}
|
|
683
|
+
async exportDump(entityIds) {
|
|
684
|
+
let ids;
|
|
685
|
+
if (entityIds && entityIds.length > 0) {
|
|
686
|
+
ids = Array.from(new Set(entityIds));
|
|
687
|
+
} else {
|
|
688
|
+
const rows = await this.db.getAllAsync(`
|
|
689
|
+
SELECT DISTINCT entity_id FROM (
|
|
690
|
+
SELECT entity_id FROM ${this.prefix}entries WHERE deleted_at IS NULL
|
|
691
|
+
UNION
|
|
692
|
+
SELECT entity_id FROM ${this.prefix}tasks WHERE deleted_at IS NULL
|
|
693
|
+
UNION
|
|
694
|
+
SELECT entity_id FROM ${this.prefix}events
|
|
695
|
+
) ORDER BY entity_id
|
|
696
|
+
`);
|
|
697
|
+
ids = rows.map((r) => r.entity_id);
|
|
698
|
+
}
|
|
699
|
+
const entities = {};
|
|
700
|
+
const BATCH = 3;
|
|
701
|
+
for (let i = 0; i < ids.length; i += BATCH) {
|
|
702
|
+
const batch = ids.slice(i, i + BATCH);
|
|
703
|
+
const batchResults = await Promise.all(
|
|
704
|
+
batch.map(async (id) => [id, await this._getFullBundle(id)])
|
|
705
|
+
);
|
|
706
|
+
for (const [id, bundle] of batchResults) {
|
|
707
|
+
entities[id] = bundle;
|
|
708
|
+
}
|
|
709
|
+
}
|
|
710
|
+
return { generatedAt: Date.now(), entities };
|
|
711
|
+
}
|
|
712
|
+
async importDump(dump, opts) {
|
|
713
|
+
const merge = opts?.merge ?? false;
|
|
714
|
+
for (const [entityId, bundle] of Object.entries(dump.entities)) {
|
|
715
|
+
await this.db.withTransactionAsync(async () => {
|
|
716
|
+
if (!merge) {
|
|
717
|
+
const now = Date.now();
|
|
718
|
+
await this.db.runAsync(
|
|
719
|
+
`UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE entity_id = ? AND deleted_at IS NULL`,
|
|
720
|
+
[now, now, entityId]
|
|
721
|
+
);
|
|
722
|
+
await this.db.runAsync(
|
|
723
|
+
`UPDATE ${this.prefix}tasks SET deleted_at = ?, updated_at = ? WHERE entity_id = ? AND deleted_at IS NULL`,
|
|
724
|
+
[now, now, entityId]
|
|
725
|
+
);
|
|
726
|
+
await this.db.runAsync(
|
|
727
|
+
`DELETE FROM ${this.prefix}checkpoints WHERE entity_id = ?`,
|
|
728
|
+
[entityId]
|
|
729
|
+
);
|
|
730
|
+
}
|
|
731
|
+
const factIds = bundle.facts.map((fact) => fact.id);
|
|
732
|
+
const existingFactsById = /* @__PURE__ */ new Map();
|
|
733
|
+
const factLookupChunkSize = 500;
|
|
734
|
+
for (let i = 0; i < factIds.length; i += factLookupChunkSize) {
|
|
735
|
+
const factIdChunk = factIds.slice(i, i + factLookupChunkSize);
|
|
736
|
+
if (factIdChunk.length === 0) continue;
|
|
737
|
+
const placeholders = factIdChunk.map(() => "?").join(", ");
|
|
738
|
+
const existingFacts = await this.db.getAllAsync(
|
|
739
|
+
`SELECT id, entity_id FROM ${this.prefix}entries WHERE id IN (${placeholders})`,
|
|
740
|
+
factIdChunk
|
|
741
|
+
);
|
|
742
|
+
for (const existingFact of existingFacts) {
|
|
743
|
+
existingFactsById.set(existingFact.id, existingFact);
|
|
744
|
+
}
|
|
745
|
+
}
|
|
746
|
+
for (const fact of bundle.facts) {
|
|
747
|
+
const tagsJson = JSON.stringify(Array.isArray(fact.tags) ? fact.tags : []);
|
|
748
|
+
const existing = existingFactsById.get(fact.id);
|
|
749
|
+
if (existing) {
|
|
750
|
+
if (existing.entity_id !== entityId) {
|
|
751
|
+
this._warnCrossEntityCollision("entry", fact.id, existing.entity_id, entityId);
|
|
752
|
+
continue;
|
|
753
|
+
}
|
|
754
|
+
if (merge) continue;
|
|
755
|
+
await this.db.runAsync(
|
|
756
|
+
`UPDATE ${this.prefix}entries SET entity_id = ?, title = ?, body = ?, tags = ?, confidence = ?, source_type = ?, source_hash = ?, source_ref = ?, created_at = ?, updated_at = ?, last_accessed_at = ?, access_count = ?, deleted_at = ? WHERE id = ?`,
|
|
757
|
+
[entityId, fact.title, fact.body, tagsJson, fact.confidence, fact.source_type, fact.source_hash, fact.source_ref, fact.created_at, fact.updated_at, fact.last_accessed_at, fact.access_count, fact.deleted_at, fact.id]
|
|
758
|
+
);
|
|
759
|
+
} else {
|
|
760
|
+
await this.db.runAsync(
|
|
761
|
+
`INSERT INTO ${this.prefix}entries (id, entity_id, title, body, tags, confidence, source_type, source_hash, source_ref, created_at, updated_at, last_accessed_at, access_count, deleted_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
762
|
+
[fact.id, entityId, fact.title, fact.body, tagsJson, fact.confidence, fact.source_type, fact.source_hash, fact.source_ref, fact.created_at, fact.updated_at, fact.last_accessed_at, fact.access_count, fact.deleted_at]
|
|
763
|
+
);
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
const taskIds = bundle.tasks.map((task) => task.id);
|
|
767
|
+
const existingTasksById = /* @__PURE__ */ new Map();
|
|
768
|
+
const taskLookupChunkSize = 500;
|
|
769
|
+
for (let i = 0; i < taskIds.length; i += taskLookupChunkSize) {
|
|
770
|
+
const taskIdChunk = taskIds.slice(i, i + taskLookupChunkSize);
|
|
771
|
+
if (taskIdChunk.length === 0) continue;
|
|
772
|
+
const placeholders = taskIdChunk.map(() => "?").join(", ");
|
|
773
|
+
const existingTasks = await this.db.getAllAsync(
|
|
774
|
+
`SELECT id, entity_id FROM ${this.prefix}tasks WHERE id IN (${placeholders})`,
|
|
775
|
+
taskIdChunk
|
|
776
|
+
);
|
|
777
|
+
for (const existingTask of existingTasks) {
|
|
778
|
+
existingTasksById.set(existingTask.id, existingTask);
|
|
779
|
+
}
|
|
780
|
+
}
|
|
781
|
+
for (const task of bundle.tasks) {
|
|
782
|
+
const existing = existingTasksById.get(task.id);
|
|
783
|
+
if (existing) {
|
|
784
|
+
if (existing.entity_id !== entityId) {
|
|
785
|
+
this._warnCrossEntityCollision("task", task.id, existing.entity_id, entityId);
|
|
786
|
+
continue;
|
|
787
|
+
}
|
|
788
|
+
if (merge) continue;
|
|
789
|
+
await this.db.runAsync(
|
|
790
|
+
`UPDATE ${this.prefix}tasks SET entity_id = ?, description = ?, status = ?, priority = ?, created_at = ?, updated_at = ?, resolved_at = ?, deleted_at = ? WHERE id = ?`,
|
|
791
|
+
[entityId, task.description, task.status, task.priority, task.created_at, task.updated_at, task.resolved_at, task.deleted_at, task.id]
|
|
792
|
+
);
|
|
793
|
+
} else {
|
|
794
|
+
await this.db.runAsync(
|
|
795
|
+
`INSERT INTO ${this.prefix}tasks (id, entity_id, description, status, priority, created_at, updated_at, resolved_at, deleted_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
796
|
+
[task.id, entityId, task.description, task.status, task.priority, task.created_at, task.updated_at, task.resolved_at, task.deleted_at]
|
|
797
|
+
);
|
|
798
|
+
}
|
|
799
|
+
}
|
|
800
|
+
for (const event of bundle.events) {
|
|
801
|
+
await this.db.runAsync(
|
|
802
|
+
`INSERT OR IGNORE INTO ${this.prefix}events (id, entity_id, event_type, summary, related_entry_id, created_at)
|
|
803
|
+
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
804
|
+
[event.id, entityId, event.event_type, event.summary, event.related_entry_id ?? null, event.created_at]
|
|
805
|
+
);
|
|
806
|
+
}
|
|
807
|
+
});
|
|
808
|
+
}
|
|
809
|
+
}
|
|
531
810
|
async forget(entityId, params) {
|
|
532
811
|
const now = Date.now();
|
|
533
812
|
let deletedEntries = 0;
|
|
@@ -582,70 +861,165 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
582
861
|
if (!sourceRef) throw new Error("Invalid sourceRef");
|
|
583
862
|
const sourceHash = normalizeSourceHash(params.sourceHash);
|
|
584
863
|
if (!sourceHash) throw new Error("Invalid sourceHash (must be 64-char hex string)");
|
|
585
|
-
const maxChunkLength = params.maxChunkLength ?? this.options.config?.maxChunkLength ??
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
864
|
+
const maxChunkLength = params.maxChunkLength ?? this.options.config?.maxChunkLength ?? 12e3;
|
|
865
|
+
const rawOverlap = params.chunkOverlap ?? this.options.config?.chunkOverlap ?? 400;
|
|
866
|
+
const chunkOverlap = Math.min(
|
|
867
|
+
Number.isFinite(rawOverlap) && rawOverlap >= 0 ? Math.floor(rawOverlap) : 400,
|
|
868
|
+
maxChunkLength - 1
|
|
869
|
+
);
|
|
870
|
+
const rawConcurrency = params.chunkConcurrency ?? this.options.config?.chunkConcurrency ?? 1;
|
|
871
|
+
const chunkConcurrency = Number.isFinite(rawConcurrency) && rawConcurrency >= 1 ? Math.floor(rawConcurrency) : 1;
|
|
589
872
|
if (typeof params.documentChunk !== "string") {
|
|
590
873
|
throw new Error(`documentChunk must be a string, received ${typeof params.documentChunk}`);
|
|
591
874
|
}
|
|
592
|
-
const
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
if (text.length === 0) {
|
|
596
|
-
return { truncated: false, chunks: 0 };
|
|
875
|
+
const jobKey = `${this.prefix}:${entityId}:${sourceRef}`;
|
|
876
|
+
if (this.activeIngestJobs.has(jobKey)) {
|
|
877
|
+
throw new WikiBusyError("ingest", entityId);
|
|
597
878
|
}
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
const searchArea = text.slice(0, maxChunkLength + 1);
|
|
604
|
-
const match = searchArea.match(/[.!?]\s+(?![\s\S]*[.!?]\s+)/);
|
|
605
|
-
if (match && match.index !== void 0) {
|
|
606
|
-
const splitPoint = Math.min(match.index + match[0].length, maxChunkLength);
|
|
607
|
-
const chunk = safeSlice(text, 0, splitPoint);
|
|
608
|
-
chunks.push(chunk);
|
|
609
|
-
text = text.slice(chunk.length);
|
|
610
|
-
} else {
|
|
611
|
-
truncated = true;
|
|
612
|
-
const chunk = safeSlice(text, 0, maxChunkLength);
|
|
613
|
-
chunks.push(chunk);
|
|
614
|
-
text = text.slice(chunk.length);
|
|
879
|
+
this.activeIngestJobs.add(jobKey);
|
|
880
|
+
try {
|
|
881
|
+
const { chunks, truncated } = chunkText(params.documentChunk, maxChunkLength, chunkOverlap);
|
|
882
|
+
if (chunks.length === 0) {
|
|
883
|
+
return { truncated: false, chunks: 0 };
|
|
615
884
|
}
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
const userPrompt = `Document Chunk:
|
|
885
|
+
const chunkResults = await withConcurrency(
|
|
886
|
+
chunks.map((chunk) => async () => {
|
|
887
|
+
const userPrompt = `Document Chunk:
|
|
620
888
|
${chunk}`;
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
889
|
+
const responseText = await this.options.llmProvider.generateText({
|
|
890
|
+
systemPrompt: INGEST_SYSTEM_PROMPT,
|
|
891
|
+
userPrompt
|
|
892
|
+
});
|
|
893
|
+
const result = parseJsonResponse(responseText);
|
|
894
|
+
return (Array.isArray(result.facts) ? result.facts : []).map(validateFact).filter((f) => f !== null);
|
|
895
|
+
}),
|
|
896
|
+
chunkConcurrency
|
|
897
|
+
);
|
|
898
|
+
const seen = /* @__PURE__ */ new Set();
|
|
899
|
+
const allValidFacts = [];
|
|
900
|
+
for (const facts of chunkResults) {
|
|
901
|
+
for (const fact of facts) {
|
|
902
|
+
const normalized = fact.title.trim().toLowerCase().replace(/\s+/g, " ");
|
|
903
|
+
if (!seen.has(normalized)) {
|
|
904
|
+
seen.add(normalized);
|
|
905
|
+
allValidFacts.push(fact);
|
|
906
|
+
}
|
|
907
|
+
}
|
|
908
|
+
}
|
|
909
|
+
const now = Date.now();
|
|
910
|
+
await this.db.withTransactionAsync(async () => {
|
|
911
|
+
await this.db.runAsync(
|
|
912
|
+
`UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE source_ref = ? AND entity_id = ? AND deleted_at IS NULL`,
|
|
913
|
+
[now, now, sourceRef, entityId]
|
|
914
|
+
);
|
|
915
|
+
for (const fact of allValidFacts) {
|
|
916
|
+
const id = generateId("fact_");
|
|
917
|
+
await this.db.runAsync(
|
|
918
|
+
`INSERT INTO ${this.prefix}entries (id, entity_id, title, body, tags, confidence, source_type, source_hash, source_ref, created_at, updated_at)
|
|
919
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
920
|
+
[id, entityId, fact.title, fact.body, JSON.stringify(fact.tags), fact.confidence, "user_document", sourceHash, sourceRef, now, now]
|
|
921
|
+
);
|
|
922
|
+
}
|
|
624
923
|
});
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
924
|
+
return { truncated, chunks: chunks.length };
|
|
925
|
+
} finally {
|
|
926
|
+
this.activeIngestJobs.delete(jobKey);
|
|
628
927
|
}
|
|
629
|
-
const now = Date.now();
|
|
630
|
-
await this.db.withTransactionAsync(async () => {
|
|
631
|
-
await this.db.runAsync(`UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE source_ref = ? AND entity_id = ? AND deleted_at IS NULL`, [now, now, sourceRef, entityId]);
|
|
632
|
-
for (const fact of allValidFacts) {
|
|
633
|
-
const id = generateId("fact_");
|
|
634
|
-
await this.db.runAsync(`
|
|
635
|
-
INSERT INTO ${this.prefix}entries (id, entity_id, title, body, tags, confidence, source_type, source_hash, source_ref, created_at, updated_at)
|
|
636
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
637
|
-
`, [id, entityId, fact.title, fact.body, JSON.stringify(fact.tags), fact.confidence, "user_document", sourceHash, sourceRef, now, now]);
|
|
638
|
-
}
|
|
639
|
-
});
|
|
640
|
-
return { truncated, chunks: chunks.length };
|
|
641
928
|
}
|
|
642
929
|
};
|
|
643
930
|
|
|
931
|
+
// src/utils/formatMemoryDump.ts
|
|
932
|
+
function renderFact(f) {
|
|
933
|
+
const tags = (f.tags || []).join(", ");
|
|
934
|
+
const source = f.source_ref ?? f.source_type;
|
|
935
|
+
return `### ${f.title}
|
|
936
|
+
**Tags:** ${tags}
|
|
937
|
+
**Confidence:** ${f.confidence}
|
|
938
|
+
**Source:** ${source}
|
|
939
|
+
|
|
940
|
+
${f.body}
|
|
941
|
+
|
|
942
|
+
---
|
|
943
|
+
`;
|
|
944
|
+
}
|
|
945
|
+
function renderTask(t) {
|
|
946
|
+
const checked = t.status === "done" ? "x" : " ";
|
|
947
|
+
const note = t.status === "done" ? " (done)" : t.status === "abandoned" ? " (abandoned)" : t.status === "in_progress" ? " (in progress)" : "";
|
|
948
|
+
return `- [${checked}] ${t.description}${note}
|
|
949
|
+
`;
|
|
950
|
+
}
|
|
951
|
+
function renderEvent(e) {
|
|
952
|
+
const ts = new Date(e.created_at).toISOString();
|
|
953
|
+
return `- [${ts}] (${e.event_type}) ${e.summary}
|
|
954
|
+
`;
|
|
955
|
+
}
|
|
956
|
+
function renderEntity(entityId, bundle, generatedAt) {
|
|
957
|
+
const lines = [];
|
|
958
|
+
lines.push(`# Memory Dump: ${entityId}`);
|
|
959
|
+
lines.push(`Generated: ${new Date(generatedAt).toISOString()}`);
|
|
960
|
+
lines.push("");
|
|
961
|
+
lines.push("## Facts");
|
|
962
|
+
lines.push("");
|
|
963
|
+
if (bundle.facts.length === 0) {
|
|
964
|
+
lines.push("_(none)_\n");
|
|
965
|
+
} else {
|
|
966
|
+
for (const f of bundle.facts) lines.push(renderFact(f));
|
|
967
|
+
}
|
|
968
|
+
lines.push("## Tasks");
|
|
969
|
+
lines.push("");
|
|
970
|
+
if (bundle.tasks.length === 0) {
|
|
971
|
+
lines.push("_(none)_\n");
|
|
972
|
+
} else {
|
|
973
|
+
for (const t of bundle.tasks) lines.push(renderTask(t));
|
|
974
|
+
}
|
|
975
|
+
lines.push("");
|
|
976
|
+
lines.push("## Recent Events");
|
|
977
|
+
lines.push("");
|
|
978
|
+
if (bundle.events.length === 0) {
|
|
979
|
+
lines.push("_(none)_\n");
|
|
980
|
+
} else {
|
|
981
|
+
for (const e of bundle.events) lines.push(renderEvent(e));
|
|
982
|
+
}
|
|
983
|
+
return lines.join("\n");
|
|
984
|
+
}
|
|
985
|
+
function shortHash(value) {
|
|
986
|
+
let h1 = 5381;
|
|
987
|
+
let h2 = 52711;
|
|
988
|
+
for (let i = 0; i < value.length; i += 1) {
|
|
989
|
+
const c = value.charCodeAt(i);
|
|
990
|
+
h1 = Math.imul(h1, 33) ^ c;
|
|
991
|
+
h2 = Math.imul(h2, 31) ^ c;
|
|
992
|
+
}
|
|
993
|
+
return (h1 >>> 0).toString(16).padStart(8, "0") + (h2 >>> 0).toString(16).padStart(8, "0");
|
|
994
|
+
}
|
|
995
|
+
function formatEntityFileName(entityId) {
|
|
996
|
+
const normalized = entityId.normalize("NFKC");
|
|
997
|
+
const sanitized = normalized.replace(/[^A-Za-z0-9._-]+/g, "_").replace(/^\.+/, "_").replace(/_+/g, "_").replace(/^[_-]+|[_-]+$/g, "");
|
|
998
|
+
const MAX_BASE = 200;
|
|
999
|
+
const trimmed = sanitized.length > MAX_BASE ? sanitized.slice(0, MAX_BASE) : sanitized;
|
|
1000
|
+
const baseName = trimmed && trimmed !== "." && trimmed !== ".." ? trimmed : "entity";
|
|
1001
|
+
const needsSuffix = baseName !== entityId || sanitized.length > MAX_BASE;
|
|
1002
|
+
const uniqueBaseName = needsSuffix ? `${baseName}-${shortHash(entityId)}` : baseName;
|
|
1003
|
+
return `${uniqueBaseName}.md`;
|
|
1004
|
+
}
|
|
1005
|
+
function formatMemoryDump(dump) {
|
|
1006
|
+
const files = Object.entries(dump.entities).map(([entityId, bundle]) => ({
|
|
1007
|
+
name: formatEntityFileName(entityId),
|
|
1008
|
+
content: renderEntity(entityId, bundle, dump.generatedAt)
|
|
1009
|
+
}));
|
|
1010
|
+
return {
|
|
1011
|
+
manifest: JSON.stringify(dump, null, 2),
|
|
1012
|
+
files
|
|
1013
|
+
};
|
|
1014
|
+
}
|
|
1015
|
+
|
|
644
1016
|
// src/index.ts
|
|
645
1017
|
function createWiki(db, options) {
|
|
646
1018
|
return new WikiMemory(db, options);
|
|
647
1019
|
}
|
|
648
1020
|
export {
|
|
1021
|
+
WikiBusyError,
|
|
649
1022
|
WikiMemory,
|
|
650
|
-
createWiki
|
|
1023
|
+
createWiki,
|
|
1024
|
+
formatMemoryDump
|
|
651
1025
|
};
|
package/dist/react/index.d.mts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import * as react_jsx_runtime from 'react/jsx-runtime';
|
|
2
2
|
import { ReactNode } from 'react';
|
|
3
|
-
import { a as WikiMemory,
|
|
3
|
+
import { a as WikiMemory, d as MemoryBundle, h as WikiEvent, M as MemoryDump } from '../WikiMemory-B54Gaa-K.mjs';
|
|
4
4
|
import 'expo-sqlite';
|
|
5
5
|
|
|
6
6
|
declare function WikiProvider({ wiki, children }: {
|
|
@@ -69,4 +69,11 @@ declare function useWikiForget(): {
|
|
|
69
69
|
error: Error | null;
|
|
70
70
|
};
|
|
71
71
|
|
|
72
|
-
|
|
72
|
+
declare function useWikiExport(): {
|
|
73
|
+
execute: (entityIds?: string[]) => Promise<MemoryDump>;
|
|
74
|
+
lastResult: MemoryDump | null;
|
|
75
|
+
isPending: boolean;
|
|
76
|
+
error: Error | null;
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
export { WikiProvider, useMemoryRead, useWiki, useWikiExport, useWikiForget, useWikiIngest, useWikiMaintenance, useWikiWrite };
|
package/dist/react/index.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import * as react_jsx_runtime from 'react/jsx-runtime';
|
|
2
2
|
import { ReactNode } from 'react';
|
|
3
|
-
import { a as WikiMemory,
|
|
3
|
+
import { a as WikiMemory, d as MemoryBundle, h as WikiEvent, M as MemoryDump } from '../WikiMemory-B54Gaa-K.js';
|
|
4
4
|
import 'expo-sqlite';
|
|
5
5
|
|
|
6
6
|
declare function WikiProvider({ wiki, children }: {
|
|
@@ -69,4 +69,11 @@ declare function useWikiForget(): {
|
|
|
69
69
|
error: Error | null;
|
|
70
70
|
};
|
|
71
71
|
|
|
72
|
-
|
|
72
|
+
declare function useWikiExport(): {
|
|
73
|
+
execute: (entityIds?: string[]) => Promise<MemoryDump>;
|
|
74
|
+
lastResult: MemoryDump | null;
|
|
75
|
+
isPending: boolean;
|
|
76
|
+
error: Error | null;
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
export { WikiProvider, useMemoryRead, useWiki, useWikiExport, useWikiForget, useWikiIngest, useWikiMaintenance, useWikiWrite };
|