@equationalapplications/expo-llm-wiki 0.0.0-development → 2.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -83,11 +83,23 @@ async function setupDatabase(db, prefix) {
83
83
  `);
84
84
  }
85
85
 
86
+ // src/types.ts
87
+ var WikiBusyError = class extends Error {
88
+ operation;
89
+ entityId;
90
+ constructor(operation, entityId) {
91
+ super(`${operation} already running for entity ${entityId}`);
92
+ this.name = "WikiBusyError";
93
+ this.operation = operation;
94
+ this.entityId = entityId;
95
+ }
96
+ };
97
+
86
98
  // src/prompts.ts
87
99
  var LIBRARIAN_SYSTEM_PROMPT = `You are a knowledge extraction agent. Your job is to analyze recent episodic events and extract stable facts and actionable tasks about the user or entity.
88
100
  Return ONLY a valid JSON object matching this schema:
89
101
  {
90
- "facts": [{ "title": "string (max 80 chars)", "body": "string (max 200 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative" }],
102
+ "facts": [{ "title": "string (max 80 chars)", "body": "string (max 800 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative" }],
91
103
  "tasks": [{ "description": "string", "priority": "number (0-10)" }]
92
104
  }
93
105
  Keep facts concise. Do not return markdown, just raw JSON.`;
@@ -96,13 +108,13 @@ Return ONLY a valid JSON object matching this schema:
96
108
  {
97
109
  "downgraded": ["string (fact IDs)"],
98
110
  "deleted": ["string (fact IDs)"],
99
- "newFacts": [{ "title": "string", "body": "string", "tags": ["string"], "confidence": "certain|inferred|tentative" }]
111
+ "newFacts": [{ "title": "string (max 80 chars)", "body": "string (max 800 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative" }]
100
112
  }
101
113
  Do not return markdown, just raw JSON.`;
102
114
  var INGEST_SYSTEM_PROMPT = `You are a document ingestion agent. Your job is to extract factual knowledge from the provided document chunk.
103
115
  Return ONLY a valid JSON object matching this schema:
104
116
  {
105
- "facts": [{ "title": "string (max 80 chars)", "body": "string (max 200 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative" }]
117
+ "facts": [{ "title": "string (max 80 chars)", "body": "string (max 800 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative" }]
106
118
  }
107
119
  Extract verbatim factual content. Do not return markdown, just raw JSON.`;
108
120
 
@@ -176,6 +188,84 @@ function safeSlice(value, start, end) {
176
188
  }
177
189
  return value.slice(safeStart, safeEnd);
178
190
  }
191
+ function chunkText(input, maxChunkLength, overlap) {
192
+ const text = input.trim();
193
+ if (text.length === 0) return { chunks: [], truncated: false };
194
+ if (!Number.isInteger(maxChunkLength) || maxChunkLength < 2) {
195
+ throw new Error("maxChunkLength must be an integer >= 2");
196
+ }
197
+ if (!Number.isInteger(overlap) || overlap < 0 || overlap >= maxChunkLength) {
198
+ throw new Error("overlap must be a non-negative integer < maxChunkLength");
199
+ }
200
+ const chunks = [];
201
+ let truncated = false;
202
+ let cursor = 0;
203
+ const halfMax = Math.floor(maxChunkLength / 2);
204
+ while (cursor < text.length) {
205
+ const remaining = text.length - cursor;
206
+ if (remaining <= maxChunkLength) {
207
+ chunks.push(safeSlice(text, cursor, text.length));
208
+ break;
209
+ }
210
+ const windowEnd = cursor + maxChunkLength;
211
+ const minSplit = cursor + halfMax;
212
+ let splitPoint = -1;
213
+ const paraIdx = text.lastIndexOf("\n\n", windowEnd);
214
+ if (paraIdx >= minSplit && paraIdx + 2 <= windowEnd) {
215
+ splitPoint = paraIdx + 2;
216
+ }
217
+ if (splitPoint === -1) {
218
+ let lastTerm = -1;
219
+ for (let i = minSplit; i < windowEnd - 1; i++) {
220
+ const ch = text[i];
221
+ if ((ch === "." || ch === "!" || ch === "?") && /\s/.test(text[i + 1])) {
222
+ lastTerm = i + 2;
223
+ }
224
+ }
225
+ if (lastTerm !== -1 && lastTerm <= windowEnd) splitPoint = lastTerm;
226
+ }
227
+ if (splitPoint === -1) {
228
+ for (let i = windowEnd - 1; i >= minSplit; i--) {
229
+ if (/\s/.test(text[i])) {
230
+ splitPoint = i + 1;
231
+ break;
232
+ }
233
+ }
234
+ }
235
+ if (splitPoint === -1) {
236
+ truncated = true;
237
+ splitPoint = windowEnd;
238
+ }
239
+ chunks.push(safeSlice(text, cursor, splitPoint));
240
+ const next = Math.max(splitPoint - overlap, cursor + 1);
241
+ cursor = next;
242
+ }
243
+ return { chunks, truncated };
244
+ }
245
+ async function withConcurrency(tasks, limit) {
246
+ const results = new Array(tasks.length);
247
+ let index = 0;
248
+ let failed = false;
249
+ let firstError;
250
+ async function worker() {
251
+ while (index < tasks.length && !failed) {
252
+ const i = index++;
253
+ try {
254
+ results[i] = await tasks[i]();
255
+ } catch (e) {
256
+ if (!failed) {
257
+ failed = true;
258
+ firstError = e;
259
+ }
260
+ return;
261
+ }
262
+ }
263
+ }
264
+ const workerCount = tasks.length === 0 ? 0 : Math.min(Math.max(limit, 1), tasks.length);
265
+ await Promise.allSettled(Array.from({ length: workerCount }, worker));
266
+ if (failed) throw firstError;
267
+ return results;
268
+ }
179
269
  function clip(value, max) {
180
270
  if (typeof value !== "string") return "";
181
271
  const s = value.trim();
@@ -188,7 +278,7 @@ function validateTags(tags) {
188
278
  function validateFact(fact) {
189
279
  if (typeof fact?.title !== "string" || typeof fact?.body !== "string") return null;
190
280
  const title = clip(fact.title, 80);
191
- const body = clip(fact.body, 200);
281
+ const body = clip(fact.body, 800);
192
282
  if (!title || !body) return null;
193
283
  let confidence = fact.confidence;
194
284
  if (confidence !== "certain" && confidence !== "tentative") confidence = "inferred";
@@ -237,6 +327,16 @@ var WikiMemory = class {
237
327
  prefix;
238
328
  options;
239
329
  activeMaintenanceJobs = /* @__PURE__ */ new Set();
330
+ activeIngestJobs = /* @__PURE__ */ new Set();
331
+ _librarianKey(entityId) {
332
+ return `${this.prefix}:${entityId}:librarian`;
333
+ }
334
+ _healKey(entityId) {
335
+ return `${this.prefix}:${entityId}:heal`;
336
+ }
337
+ _warnCrossEntityCollision(type, id, existingEntityId, targetEntityId) {
338
+ console.warn(`[WikiMemory] importDump: ${type} id "${id}" already belongs to entity "${existingEntityId}"; skipping for entity "${targetEntityId}"`);
339
+ }
240
340
  constructor(db, options) {
241
341
  this.db = db;
242
342
  this.options = options;
@@ -322,6 +422,9 @@ var WikiMemory = class {
322
422
  }));
323
423
  return { facts, tasks, events: events.reverse() };
324
424
  }
425
+ async getMemoryBundle(entityId) {
426
+ return this._getFullBundle(entityId, { maxEvents: 10 });
427
+ }
325
428
  async write(entityId, event) {
326
429
  const id = generateId("evt_");
327
430
  const now = Date.now();
@@ -342,7 +445,7 @@ var WikiMemory = class {
342
445
  let memoryCheckpoint = cp?.memory_checkpoint || 0;
343
446
  if (memoryCheckpoint > count) memoryCheckpoint = 0;
344
447
  if (count - memoryCheckpoint >= threshold) {
345
- const jobKey = `${this.prefix}:${entityId}`;
448
+ const jobKey = this._librarianKey(entityId);
346
449
  if (!this.activeMaintenanceJobs.has(jobKey)) {
347
450
  this.activeMaintenanceJobs.add(jobKey);
348
451
  this.runLibrarianThenMaybeHeal(entityId, count).catch(console.error).finally(() => this.activeMaintenanceJobs.delete(jobKey));
@@ -361,12 +464,20 @@ var WikiMemory = class {
361
464
  let healCheckpoint = cp?.heal_checkpoint || 0;
362
465
  if (healCheckpoint > currentEventCount) healCheckpoint = 0;
363
466
  if (currentEventCount - healCheckpoint >= autoHealThreshold) {
364
- await this._doRunHeal(entityId);
365
- await this.db.runAsync(`
366
- INSERT INTO ${this.prefix}checkpoints (entity_id, heal_checkpoint)
367
- VALUES (?, ?)
368
- ON CONFLICT(entity_id) DO UPDATE SET heal_checkpoint = ?
369
- `, [entityId, currentEventCount, currentEventCount]);
467
+ const healKey = this._healKey(entityId);
468
+ if (!this.activeMaintenanceJobs.has(healKey)) {
469
+ this.activeMaintenanceJobs.add(healKey);
470
+ try {
471
+ await this._doRunHeal(entityId);
472
+ await this.db.runAsync(`
473
+ INSERT INTO ${this.prefix}checkpoints (entity_id, heal_checkpoint)
474
+ VALUES (?, ?)
475
+ ON CONFLICT(entity_id) DO UPDATE SET heal_checkpoint = ?
476
+ `, [entityId, currentEventCount, currentEventCount]);
477
+ } finally {
478
+ this.activeMaintenanceJobs.delete(healKey);
479
+ }
480
+ }
370
481
  }
371
482
  }
372
483
  async _doRunLibrarian(entityId) {
@@ -509,8 +620,10 @@ The following document anchors are provided for contradiction detection only. Do
509
620
  });
510
621
  }
511
622
  async runLibrarian(entityId) {
512
- const jobKey = `${this.prefix}:${entityId}`;
513
- if (this.activeMaintenanceJobs.has(jobKey)) return;
623
+ const jobKey = this._librarianKey(entityId);
624
+ if (this.activeMaintenanceJobs.has(jobKey)) {
625
+ throw new WikiBusyError("librarian", entityId);
626
+ }
514
627
  this.activeMaintenanceJobs.add(jobKey);
515
628
  try {
516
629
  await this._doRunLibrarian(entityId);
@@ -519,8 +632,10 @@ The following document anchors are provided for contradiction detection only. Do
519
632
  }
520
633
  }
521
634
  async runHeal(entityId) {
522
- const jobKey = `${this.prefix}:${entityId}`;
523
- if (this.activeMaintenanceJobs.has(jobKey)) return;
635
+ const jobKey = this._healKey(entityId);
636
+ if (this.activeMaintenanceJobs.has(jobKey)) {
637
+ throw new WikiBusyError("heal", entityId);
638
+ }
524
639
  this.activeMaintenanceJobs.add(jobKey);
525
640
  try {
526
641
  await this._doRunHeal(entityId);
@@ -528,6 +643,170 @@ The following document anchors are provided for contradiction detection only. Do
528
643
  this.activeMaintenanceJobs.delete(jobKey);
529
644
  }
530
645
  }
646
+ getEntityStatus(entityId) {
647
+ const ingestPrefix = `${this.prefix}:${entityId}:`;
648
+ let ingesting = false;
649
+ for (const k of this.activeIngestJobs) {
650
+ if (k.startsWith(ingestPrefix)) {
651
+ ingesting = true;
652
+ break;
653
+ }
654
+ }
655
+ return {
656
+ ingesting,
657
+ librarian: this.activeMaintenanceJobs.has(this._librarianKey(entityId)),
658
+ heal: this.activeMaintenanceJobs.has(this._healKey(entityId))
659
+ };
660
+ }
661
+ async _getFullBundle(entityId, opts) {
662
+ const maxEvents = opts?.maxEvents;
663
+ const eventsQuery = maxEvents != null ? `SELECT * FROM ${this.prefix}events WHERE entity_id = ? ORDER BY created_at DESC LIMIT ?` : `SELECT * FROM ${this.prefix}events WHERE entity_id = ? ORDER BY created_at ASC`;
664
+ const eventsParams = maxEvents != null ? [entityId, maxEvents] : [entityId];
665
+ const [factsRaw, tasks, eventsRaw] = await Promise.all([
666
+ this.db.getAllAsync(
667
+ `SELECT * FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL ORDER BY updated_at DESC`,
668
+ [entityId]
669
+ ),
670
+ this.db.getAllAsync(
671
+ `SELECT * FROM ${this.prefix}tasks WHERE entity_id = ? AND deleted_at IS NULL ORDER BY priority DESC, created_at ASC`,
672
+ [entityId]
673
+ ),
674
+ this.db.getAllAsync(eventsQuery, eventsParams)
675
+ ]);
676
+ const facts = factsRaw.map((f) => ({
677
+ ...f,
678
+ tags: typeof f.tags === "string" ? JSON.parse(f.tags) : f.tags
679
+ }));
680
+ const events = maxEvents != null ? eventsRaw.slice().reverse() : eventsRaw;
681
+ return { facts, tasks, events };
682
+ }
683
+ async exportDump(entityIds) {
684
+ let ids;
685
+ if (entityIds && entityIds.length > 0) {
686
+ ids = Array.from(new Set(entityIds));
687
+ } else {
688
+ const rows = await this.db.getAllAsync(`
689
+ SELECT DISTINCT entity_id FROM (
690
+ SELECT entity_id FROM ${this.prefix}entries WHERE deleted_at IS NULL
691
+ UNION
692
+ SELECT entity_id FROM ${this.prefix}tasks WHERE deleted_at IS NULL
693
+ UNION
694
+ SELECT entity_id FROM ${this.prefix}events
695
+ ) ORDER BY entity_id
696
+ `);
697
+ ids = rows.map((r) => r.entity_id);
698
+ }
699
+ const entities = {};
700
+ const BATCH = 3;
701
+ for (let i = 0; i < ids.length; i += BATCH) {
702
+ const batch = ids.slice(i, i + BATCH);
703
+ const batchResults = await Promise.all(
704
+ batch.map(async (id) => [id, await this._getFullBundle(id)])
705
+ );
706
+ for (const [id, bundle] of batchResults) {
707
+ entities[id] = bundle;
708
+ }
709
+ }
710
+ return { generatedAt: Date.now(), entities };
711
+ }
712
+ async importDump(dump, opts) {
713
+ const merge = opts?.merge ?? false;
714
+ for (const [entityId, bundle] of Object.entries(dump.entities)) {
715
+ await this.db.withTransactionAsync(async () => {
716
+ if (!merge) {
717
+ const now = Date.now();
718
+ await this.db.runAsync(
719
+ `UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE entity_id = ? AND deleted_at IS NULL`,
720
+ [now, now, entityId]
721
+ );
722
+ await this.db.runAsync(
723
+ `UPDATE ${this.prefix}tasks SET deleted_at = ?, updated_at = ? WHERE entity_id = ? AND deleted_at IS NULL`,
724
+ [now, now, entityId]
725
+ );
726
+ await this.db.runAsync(
727
+ `DELETE FROM ${this.prefix}checkpoints WHERE entity_id = ?`,
728
+ [entityId]
729
+ );
730
+ }
731
+ const factIds = bundle.facts.map((fact) => fact.id);
732
+ const existingFactsById = /* @__PURE__ */ new Map();
733
+ const factLookupChunkSize = 500;
734
+ for (let i = 0; i < factIds.length; i += factLookupChunkSize) {
735
+ const factIdChunk = factIds.slice(i, i + factLookupChunkSize);
736
+ if (factIdChunk.length === 0) continue;
737
+ const placeholders = factIdChunk.map(() => "?").join(", ");
738
+ const existingFacts = await this.db.getAllAsync(
739
+ `SELECT id, entity_id FROM ${this.prefix}entries WHERE id IN (${placeholders})`,
740
+ factIdChunk
741
+ );
742
+ for (const existingFact of existingFacts) {
743
+ existingFactsById.set(existingFact.id, existingFact);
744
+ }
745
+ }
746
+ for (const fact of bundle.facts) {
747
+ const tagsJson = JSON.stringify(Array.isArray(fact.tags) ? fact.tags : []);
748
+ const existing = existingFactsById.get(fact.id);
749
+ if (existing) {
750
+ if (existing.entity_id !== entityId) {
751
+ this._warnCrossEntityCollision("entry", fact.id, existing.entity_id, entityId);
752
+ continue;
753
+ }
754
+ if (merge) continue;
755
+ await this.db.runAsync(
756
+ `UPDATE ${this.prefix}entries SET entity_id = ?, title = ?, body = ?, tags = ?, confidence = ?, source_type = ?, source_hash = ?, source_ref = ?, created_at = ?, updated_at = ?, last_accessed_at = ?, access_count = ?, deleted_at = ? WHERE id = ?`,
757
+ [entityId, fact.title, fact.body, tagsJson, fact.confidence, fact.source_type, fact.source_hash, fact.source_ref, fact.created_at, fact.updated_at, fact.last_accessed_at, fact.access_count, fact.deleted_at, fact.id]
758
+ );
759
+ } else {
760
+ await this.db.runAsync(
761
+ `INSERT INTO ${this.prefix}entries (id, entity_id, title, body, tags, confidence, source_type, source_hash, source_ref, created_at, updated_at, last_accessed_at, access_count, deleted_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
762
+ [fact.id, entityId, fact.title, fact.body, tagsJson, fact.confidence, fact.source_type, fact.source_hash, fact.source_ref, fact.created_at, fact.updated_at, fact.last_accessed_at, fact.access_count, fact.deleted_at]
763
+ );
764
+ }
765
+ }
766
+ const taskIds = bundle.tasks.map((task) => task.id);
767
+ const existingTasksById = /* @__PURE__ */ new Map();
768
+ const taskLookupChunkSize = 500;
769
+ for (let i = 0; i < taskIds.length; i += taskLookupChunkSize) {
770
+ const taskIdChunk = taskIds.slice(i, i + taskLookupChunkSize);
771
+ if (taskIdChunk.length === 0) continue;
772
+ const placeholders = taskIdChunk.map(() => "?").join(", ");
773
+ const existingTasks = await this.db.getAllAsync(
774
+ `SELECT id, entity_id FROM ${this.prefix}tasks WHERE id IN (${placeholders})`,
775
+ taskIdChunk
776
+ );
777
+ for (const existingTask of existingTasks) {
778
+ existingTasksById.set(existingTask.id, existingTask);
779
+ }
780
+ }
781
+ for (const task of bundle.tasks) {
782
+ const existing = existingTasksById.get(task.id);
783
+ if (existing) {
784
+ if (existing.entity_id !== entityId) {
785
+ this._warnCrossEntityCollision("task", task.id, existing.entity_id, entityId);
786
+ continue;
787
+ }
788
+ if (merge) continue;
789
+ await this.db.runAsync(
790
+ `UPDATE ${this.prefix}tasks SET entity_id = ?, description = ?, status = ?, priority = ?, created_at = ?, updated_at = ?, resolved_at = ?, deleted_at = ? WHERE id = ?`,
791
+ [entityId, task.description, task.status, task.priority, task.created_at, task.updated_at, task.resolved_at, task.deleted_at, task.id]
792
+ );
793
+ } else {
794
+ await this.db.runAsync(
795
+ `INSERT INTO ${this.prefix}tasks (id, entity_id, description, status, priority, created_at, updated_at, resolved_at, deleted_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
796
+ [task.id, entityId, task.description, task.status, task.priority, task.created_at, task.updated_at, task.resolved_at, task.deleted_at]
797
+ );
798
+ }
799
+ }
800
+ for (const event of bundle.events) {
801
+ await this.db.runAsync(
802
+ `INSERT OR IGNORE INTO ${this.prefix}events (id, entity_id, event_type, summary, related_entry_id, created_at)
803
+ VALUES (?, ?, ?, ?, ?, ?)`,
804
+ [event.id, entityId, event.event_type, event.summary, event.related_entry_id ?? null, event.created_at]
805
+ );
806
+ }
807
+ });
808
+ }
809
+ }
531
810
  async forget(entityId, params) {
532
811
  const now = Date.now();
533
812
  let deletedEntries = 0;
@@ -582,70 +861,165 @@ The following document anchors are provided for contradiction detection only. Do
582
861
  if (!sourceRef) throw new Error("Invalid sourceRef");
583
862
  const sourceHash = normalizeSourceHash(params.sourceHash);
584
863
  if (!sourceHash) throw new Error("Invalid sourceHash (must be 64-char hex string)");
585
- const maxChunkLength = params.maxChunkLength ?? this.options.config?.maxChunkLength ?? 6e3;
586
- if (!Number.isInteger(maxChunkLength) || maxChunkLength < 2) {
587
- throw new Error("maxChunkLength must be an integer greater than or equal to 2");
588
- }
864
+ const maxChunkLength = params.maxChunkLength ?? this.options.config?.maxChunkLength ?? 12e3;
865
+ const rawOverlap = params.chunkOverlap ?? this.options.config?.chunkOverlap ?? 400;
866
+ const chunkOverlap = Math.min(
867
+ Number.isFinite(rawOverlap) && rawOverlap >= 0 ? Math.floor(rawOverlap) : 400,
868
+ maxChunkLength - 1
869
+ );
870
+ const rawConcurrency = params.chunkConcurrency ?? this.options.config?.chunkConcurrency ?? 1;
871
+ const chunkConcurrency = Number.isFinite(rawConcurrency) && rawConcurrency >= 1 ? Math.floor(rawConcurrency) : 1;
589
872
  if (typeof params.documentChunk !== "string") {
590
873
  throw new Error(`documentChunk must be a string, received ${typeof params.documentChunk}`);
591
874
  }
592
- const chunks = [];
593
- let truncated = false;
594
- let text = params.documentChunk.trim();
595
- if (text.length === 0) {
596
- return { truncated: false, chunks: 0 };
875
+ const jobKey = `${this.prefix}:${entityId}:${sourceRef}`;
876
+ if (this.activeIngestJobs.has(jobKey)) {
877
+ throw new WikiBusyError("ingest", entityId);
597
878
  }
598
- while (text.length > 0) {
599
- if (text.length <= maxChunkLength) {
600
- chunks.push(text);
601
- break;
602
- }
603
- const searchArea = text.slice(0, maxChunkLength + 1);
604
- const match = searchArea.match(/[.!?]\s+(?![\s\S]*[.!?]\s+)/);
605
- if (match && match.index !== void 0) {
606
- const splitPoint = Math.min(match.index + match[0].length, maxChunkLength);
607
- const chunk = safeSlice(text, 0, splitPoint);
608
- chunks.push(chunk);
609
- text = text.slice(chunk.length);
610
- } else {
611
- truncated = true;
612
- const chunk = safeSlice(text, 0, maxChunkLength);
613
- chunks.push(chunk);
614
- text = text.slice(chunk.length);
879
+ this.activeIngestJobs.add(jobKey);
880
+ try {
881
+ const { chunks, truncated } = chunkText(params.documentChunk, maxChunkLength, chunkOverlap);
882
+ if (chunks.length === 0) {
883
+ return { truncated: false, chunks: 0 };
615
884
  }
616
- }
617
- const allValidFacts = [];
618
- for (const chunk of chunks) {
619
- const userPrompt = `Document Chunk:
885
+ const chunkResults = await withConcurrency(
886
+ chunks.map((chunk) => async () => {
887
+ const userPrompt = `Document Chunk:
620
888
  ${chunk}`;
621
- const responseText = await this.options.llmProvider.generateText({
622
- systemPrompt: INGEST_SYSTEM_PROMPT,
623
- userPrompt
889
+ const responseText = await this.options.llmProvider.generateText({
890
+ systemPrompt: INGEST_SYSTEM_PROMPT,
891
+ userPrompt
892
+ });
893
+ const result = parseJsonResponse(responseText);
894
+ return (Array.isArray(result.facts) ? result.facts : []).map(validateFact).filter((f) => f !== null);
895
+ }),
896
+ chunkConcurrency
897
+ );
898
+ const seen = /* @__PURE__ */ new Set();
899
+ const allValidFacts = [];
900
+ for (const facts of chunkResults) {
901
+ for (const fact of facts) {
902
+ const normalized = fact.title.trim().toLowerCase().replace(/\s+/g, " ");
903
+ if (!seen.has(normalized)) {
904
+ seen.add(normalized);
905
+ allValidFacts.push(fact);
906
+ }
907
+ }
908
+ }
909
+ const now = Date.now();
910
+ await this.db.withTransactionAsync(async () => {
911
+ await this.db.runAsync(
912
+ `UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE source_ref = ? AND entity_id = ? AND deleted_at IS NULL`,
913
+ [now, now, sourceRef, entityId]
914
+ );
915
+ for (const fact of allValidFacts) {
916
+ const id = generateId("fact_");
917
+ await this.db.runAsync(
918
+ `INSERT INTO ${this.prefix}entries (id, entity_id, title, body, tags, confidence, source_type, source_hash, source_ref, created_at, updated_at)
919
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
920
+ [id, entityId, fact.title, fact.body, JSON.stringify(fact.tags), fact.confidence, "user_document", sourceHash, sourceRef, now, now]
921
+ );
922
+ }
624
923
  });
625
- const result = parseJsonResponse(responseText);
626
- const validFacts = (Array.isArray(result.facts) ? result.facts : []).map(validateFact).filter((f) => f !== null);
627
- allValidFacts.push(...validFacts);
924
+ return { truncated, chunks: chunks.length };
925
+ } finally {
926
+ this.activeIngestJobs.delete(jobKey);
628
927
  }
629
- const now = Date.now();
630
- await this.db.withTransactionAsync(async () => {
631
- await this.db.runAsync(`UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE source_ref = ? AND entity_id = ? AND deleted_at IS NULL`, [now, now, sourceRef, entityId]);
632
- for (const fact of allValidFacts) {
633
- const id = generateId("fact_");
634
- await this.db.runAsync(`
635
- INSERT INTO ${this.prefix}entries (id, entity_id, title, body, tags, confidence, source_type, source_hash, source_ref, created_at, updated_at)
636
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
637
- `, [id, entityId, fact.title, fact.body, JSON.stringify(fact.tags), fact.confidence, "user_document", sourceHash, sourceRef, now, now]);
638
- }
639
- });
640
- return { truncated, chunks: chunks.length };
641
928
  }
642
929
  };
643
930
 
931
+ // src/utils/formatMemoryDump.ts
932
+ function renderFact(f) {
933
+ const tags = (f.tags || []).join(", ");
934
+ const source = f.source_ref ?? f.source_type;
935
+ return `### ${f.title}
936
+ **Tags:** ${tags}
937
+ **Confidence:** ${f.confidence}
938
+ **Source:** ${source}
939
+
940
+ ${f.body}
941
+
942
+ ---
943
+ `;
944
+ }
945
+ function renderTask(t) {
946
+ const checked = t.status === "done" ? "x" : " ";
947
+ const note = t.status === "done" ? " (done)" : t.status === "abandoned" ? " (abandoned)" : t.status === "in_progress" ? " (in progress)" : "";
948
+ return `- [${checked}] ${t.description}${note}
949
+ `;
950
+ }
951
+ function renderEvent(e) {
952
+ const ts = new Date(e.created_at).toISOString();
953
+ return `- [${ts}] (${e.event_type}) ${e.summary}
954
+ `;
955
+ }
956
+ function renderEntity(entityId, bundle, generatedAt) {
957
+ const lines = [];
958
+ lines.push(`# Memory Dump: ${entityId}`);
959
+ lines.push(`Generated: ${new Date(generatedAt).toISOString()}`);
960
+ lines.push("");
961
+ lines.push("## Facts");
962
+ lines.push("");
963
+ if (bundle.facts.length === 0) {
964
+ lines.push("_(none)_\n");
965
+ } else {
966
+ for (const f of bundle.facts) lines.push(renderFact(f));
967
+ }
968
+ lines.push("## Tasks");
969
+ lines.push("");
970
+ if (bundle.tasks.length === 0) {
971
+ lines.push("_(none)_\n");
972
+ } else {
973
+ for (const t of bundle.tasks) lines.push(renderTask(t));
974
+ }
975
+ lines.push("");
976
+ lines.push("## Recent Events");
977
+ lines.push("");
978
+ if (bundle.events.length === 0) {
979
+ lines.push("_(none)_\n");
980
+ } else {
981
+ for (const e of bundle.events) lines.push(renderEvent(e));
982
+ }
983
+ return lines.join("\n");
984
+ }
985
+ function shortHash(value) {
986
+ let h1 = 5381;
987
+ let h2 = 52711;
988
+ for (let i = 0; i < value.length; i += 1) {
989
+ const c = value.charCodeAt(i);
990
+ h1 = Math.imul(h1, 33) ^ c;
991
+ h2 = Math.imul(h2, 31) ^ c;
992
+ }
993
+ return (h1 >>> 0).toString(16).padStart(8, "0") + (h2 >>> 0).toString(16).padStart(8, "0");
994
+ }
995
+ function formatEntityFileName(entityId) {
996
+ const normalized = entityId.normalize("NFKC");
997
+ const sanitized = normalized.replace(/[^A-Za-z0-9._-]+/g, "_").replace(/^\.+/, "_").replace(/_+/g, "_").replace(/^[_-]+|[_-]+$/g, "");
998
+ const MAX_BASE = 200;
999
+ const trimmed = sanitized.length > MAX_BASE ? sanitized.slice(0, MAX_BASE) : sanitized;
1000
+ const baseName = trimmed && trimmed !== "." && trimmed !== ".." ? trimmed : "entity";
1001
+ const needsSuffix = baseName !== entityId || sanitized.length > MAX_BASE;
1002
+ const uniqueBaseName = needsSuffix ? `${baseName}-${shortHash(entityId)}` : baseName;
1003
+ return `${uniqueBaseName}.md`;
1004
+ }
1005
+ function formatMemoryDump(dump) {
1006
+ const files = Object.entries(dump.entities).map(([entityId, bundle]) => ({
1007
+ name: formatEntityFileName(entityId),
1008
+ content: renderEntity(entityId, bundle, dump.generatedAt)
1009
+ }));
1010
+ return {
1011
+ manifest: JSON.stringify(dump, null, 2),
1012
+ files
1013
+ };
1014
+ }
1015
+
644
1016
  // src/index.ts
645
1017
  function createWiki(db, options) {
646
1018
  return new WikiMemory(db, options);
647
1019
  }
648
1020
  export {
1021
+ WikiBusyError,
649
1022
  WikiMemory,
650
- createWiki
1023
+ createWiki,
1024
+ formatMemoryDump
651
1025
  };
@@ -1,6 +1,6 @@
1
1
  import * as react_jsx_runtime from 'react/jsx-runtime';
2
2
  import { ReactNode } from 'react';
3
- import { a as WikiMemory, M as MemoryBundle, e as WikiEvent } from '../WikiMemory-BI2Aizwv.mjs';
3
+ import { a as WikiMemory, d as MemoryBundle, h as WikiEvent, M as MemoryDump } from '../WikiMemory-B54Gaa-K.mjs';
4
4
  import 'expo-sqlite';
5
5
 
6
6
  declare function WikiProvider({ wiki, children }: {
@@ -69,4 +69,11 @@ declare function useWikiForget(): {
69
69
  error: Error | null;
70
70
  };
71
71
 
72
- export { WikiProvider, useMemoryRead, useWiki, useWikiForget, useWikiIngest, useWikiMaintenance, useWikiWrite };
72
+ declare function useWikiExport(): {
73
+ execute: (entityIds?: string[]) => Promise<MemoryDump>;
74
+ lastResult: MemoryDump | null;
75
+ isPending: boolean;
76
+ error: Error | null;
77
+ };
78
+
79
+ export { WikiProvider, useMemoryRead, useWiki, useWikiExport, useWikiForget, useWikiIngest, useWikiMaintenance, useWikiWrite };
@@ -1,6 +1,6 @@
1
1
  import * as react_jsx_runtime from 'react/jsx-runtime';
2
2
  import { ReactNode } from 'react';
3
- import { a as WikiMemory, M as MemoryBundle, e as WikiEvent } from '../WikiMemory-BI2Aizwv.js';
3
+ import { a as WikiMemory, d as MemoryBundle, h as WikiEvent, M as MemoryDump } from '../WikiMemory-B54Gaa-K.js';
4
4
  import 'expo-sqlite';
5
5
 
6
6
  declare function WikiProvider({ wiki, children }: {
@@ -69,4 +69,11 @@ declare function useWikiForget(): {
69
69
  error: Error | null;
70
70
  };
71
71
 
72
- export { WikiProvider, useMemoryRead, useWiki, useWikiForget, useWikiIngest, useWikiMaintenance, useWikiWrite };
72
+ declare function useWikiExport(): {
73
+ execute: (entityIds?: string[]) => Promise<MemoryDump>;
74
+ lastResult: MemoryDump | null;
75
+ isPending: boolean;
76
+ error: Error | null;
77
+ };
78
+
79
+ export { WikiProvider, useMemoryRead, useWiki, useWikiExport, useWikiForget, useWikiIngest, useWikiMaintenance, useWikiWrite };