@equationalapplications/expo-llm-wiki 0.0.0-development → 2.1.0

package/dist/index.mjs

@@ -29,7 +29,8 @@ async function setupDatabase(db, prefix) {
       body,
       tags,
       content='${prefix}entries',
-      content_rowid='rowid'
+      content_rowid='rowid',
+      tokenize='porter unicode61'
     );
 
     -- Triggers to keep FTS5 in sync with entries
@@ -83,11 +84,23 @@ async function setupDatabase(db, prefix) {
   `);
 }
 
+// src/types.ts
+var WikiBusyError = class extends Error {
+  operation;
+  entityId;
+  constructor(operation, entityId) {
+    super(`${operation} already running for entity ${entityId}`);
+    this.name = "WikiBusyError";
+    this.operation = operation;
+    this.entityId = entityId;
+  }
+};
+
 // src/prompts.ts
 var LIBRARIAN_SYSTEM_PROMPT = `You are a knowledge extraction agent. Your job is to analyze recent episodic events and extract stable facts and actionable tasks about the user or entity.
 Return ONLY a valid JSON object matching this schema:
 {
-  "facts": [{ "title": "string (max 80 chars)", "body": "string (max 200 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative" }],
+  "facts": [{ "title": "string (max 80 chars)", "body": "string (max 800 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative" }],
   "tasks": [{ "description": "string", "priority": "number (0-10)" }]
 }
 Keep facts concise. Do not return markdown, just raw JSON.`;
@@ -96,13 +109,13 @@ Return ONLY a valid JSON object matching this schema:
 {
   "downgraded": ["string (fact IDs)"],
   "deleted": ["string (fact IDs)"],
-  "newFacts": [{ "title": "string", "body": "string", "tags": ["string"], "confidence": "certain|inferred|tentative" }]
+  "newFacts": [{ "title": "string (max 80 chars)", "body": "string (max 800 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative" }]
 }
 Do not return markdown, just raw JSON.`;
 var INGEST_SYSTEM_PROMPT = `You are a document ingestion agent. Your job is to extract factual knowledge from the provided document chunk.
 Return ONLY a valid JSON object matching this schema:
 {
-  "facts": [{ "title": "string (max 80 chars)", "body": "string (max 200 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative" }]
+  "facts": [{ "title": "string (max 80 chars)", "body": "string (max 800 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative" }]
 }
 Extract verbatim factual content. Do not return markdown, just raw JSON.`;
 
@@ -176,6 +189,84 @@ function safeSlice(value, start, end) {
   }
   return value.slice(safeStart, safeEnd);
 }
+function chunkText(input, maxChunkLength, overlap) {
+  const text = input.trim();
+  if (text.length === 0) return { chunks: [], truncated: false };
+  if (!Number.isInteger(maxChunkLength) || maxChunkLength < 2) {
+    throw new Error("maxChunkLength must be an integer >= 2");
+  }
+  if (!Number.isInteger(overlap) || overlap < 0 || overlap >= maxChunkLength) {
+    throw new Error("overlap must be a non-negative integer < maxChunkLength");
+  }
+  const chunks = [];
+  let truncated = false;
+  let cursor = 0;
+  const halfMax = Math.floor(maxChunkLength / 2);
+  while (cursor < text.length) {
+    const remaining = text.length - cursor;
+    if (remaining <= maxChunkLength) {
+      chunks.push(safeSlice(text, cursor, text.length));
+      break;
+    }
+    const windowEnd = cursor + maxChunkLength;
+    const minSplit = cursor + halfMax;
+    let splitPoint = -1;
+    const paraIdx = text.lastIndexOf("\n\n", windowEnd);
+    if (paraIdx >= minSplit && paraIdx + 2 <= windowEnd) {
+      splitPoint = paraIdx + 2;
+    }
+    if (splitPoint === -1) {
+      let lastTerm = -1;
+      for (let i = minSplit; i < windowEnd - 1; i++) {
+        const ch = text[i];
+        if ((ch === "." || ch === "!" || ch === "?") && /\s/.test(text[i + 1])) {
+          lastTerm = i + 2;
+        }
+      }
+      if (lastTerm !== -1 && lastTerm <= windowEnd) splitPoint = lastTerm;
+    }
+    if (splitPoint === -1) {
+      for (let i = windowEnd - 1; i >= minSplit; i--) {
+        if (/\s/.test(text[i])) {
+          splitPoint = i + 1;
+          break;
+        }
+      }
+    }
+    if (splitPoint === -1) {
+      truncated = true;
+      splitPoint = windowEnd;
+    }
+    chunks.push(safeSlice(text, cursor, splitPoint));
+    const next = Math.max(splitPoint - overlap, cursor + 1);
+    cursor = next;
+  }
+  return { chunks, truncated };
+}
+async function withConcurrency(tasks, limit) {
+  const results = new Array(tasks.length);
+  let index = 0;
+  let failed = false;
+  let firstError;
+  async function worker() {
+    while (index < tasks.length && !failed) {
+      const i = index++;
+      try {
+        results[i] = await tasks[i]();
+      } catch (e) {
+        if (!failed) {
+          failed = true;
+          firstError = e;
+        }
+        return;
+      }
+    }
+  }
+  const workerCount = tasks.length === 0 ? 0 : Math.min(Math.max(limit, 1), tasks.length);
+  await Promise.allSettled(Array.from({ length: workerCount }, worker));
+  if (failed) throw firstError;
+  return results;
+}
 function clip(value, max) {
   if (typeof value !== "string") return "";
   const s = value.trim();
@@ -188,7 +279,7 @@ function validateTags(tags) {
 function validateFact(fact) {
   if (typeof fact?.title !== "string" || typeof fact?.body !== "string") return null;
   const title = clip(fact.title, 80);
-  const body = clip(fact.body, 200);
+  const body = clip(fact.body, 800);
   if (!title || !body) return null;
   let confidence = fact.confidence;
   if (confidence !== "certain" && confidence !== "tentative") confidence = "inferred";
@@ -237,6 +328,16 @@ var WikiMemory = class {
   prefix;
   options;
   activeMaintenanceJobs = /* @__PURE__ */ new Set();
+  activeIngestJobs = /* @__PURE__ */ new Set();
+  _librarianKey(entityId) {
+    return `${this.prefix}:${entityId}:librarian`;
+  }
+  _healKey(entityId) {
+    return `${this.prefix}:${entityId}:heal`;
+  }
+  _warnCrossEntityCollision(type, id, existingEntityId, targetEntityId) {
+    console.warn(`[WikiMemory] importDump: ${type} id "${id}" already belongs to entity "${existingEntityId}"; skipping for entity "${targetEntityId}"`);
+  }
   constructor(db, options) {
     this.db = db;
     this.options = options;
@@ -244,6 +345,45 @@ var WikiMemory = class {
   }
   async setup() {
     await setupDatabase(this.db, this.prefix);
+    const ftsMeta = await this.db.getFirstAsync(
+      `SELECT sql FROM sqlite_master WHERE type='table' AND name=?`,
+      [`${this.prefix}entries_fts`]
+    );
+    const hasPorterTokenizer = /tokenize\s*=\s*['"]porter\s+unicode61['"]/i.test(ftsMeta?.sql ?? "");
+    if (ftsMeta?.sql && !hasPorterTokenizer) {
+      await this.db.withTransactionAsync(async () => {
+        await this.db.execAsync(`
+          DROP TRIGGER IF EXISTS ${this.prefix}entries_ai;
+          DROP TRIGGER IF EXISTS ${this.prefix}entries_ad;
+          DROP TRIGGER IF EXISTS ${this.prefix}entries_au;
+          DROP TABLE IF EXISTS ${this.prefix}entries_fts;
+          CREATE VIRTUAL TABLE ${this.prefix}entries_fts USING fts5(
+            title,
+            body,
+            tags,
+            content='${this.prefix}entries',
+            content_rowid='rowid',
+            tokenize='porter unicode61'
+          );
+          INSERT INTO ${this.prefix}entries_fts(rowid, title, body, tags)
+            SELECT rowid, title, body, tags FROM ${this.prefix}entries;
+          CREATE TRIGGER ${this.prefix}entries_ai AFTER INSERT ON ${this.prefix}entries BEGIN
+            INSERT INTO ${this.prefix}entries_fts(rowid, title, body, tags)
+            VALUES (new.rowid, new.title, new.body, new.tags);
+          END;
+          CREATE TRIGGER ${this.prefix}entries_ad AFTER DELETE ON ${this.prefix}entries BEGIN
+            INSERT INTO ${this.prefix}entries_fts(${this.prefix}entries_fts, rowid, title, body, tags)
+            VALUES ('delete', old.rowid, old.title, old.body, old.tags);
+          END;
+          CREATE TRIGGER ${this.prefix}entries_au AFTER UPDATE ON ${this.prefix}entries BEGIN
+            INSERT INTO ${this.prefix}entries_fts(${this.prefix}entries_fts, rowid, title, body, tags)
+            VALUES ('delete', old.rowid, old.title, old.body, old.tags);
+            INSERT INTO ${this.prefix}entries_fts(rowid, title, body, tags)
+            VALUES (new.rowid, new.title, new.body, new.tags);
+          END;
+        `);
+      });
+    }
     const rows = await this.db.getAllAsync(`
       SELECT rowid, source_ref FROM ${this.prefix}entries
       WHERE source_ref IS NOT NULL
@@ -268,9 +408,35 @@ var WikiMemory = class {
     });
   }
   formatSearchQuery(query) {
-    const tokens = query.toLowerCase().replace(/[^a-z0-9\s]/g, "").split(/\s+/).filter((t) => t.length >= 3).slice(0, 6);
-    if (tokens.length === 0) return "";
-    return tokens.map((t) => `"${t}"*`).join(" OR ");
+    const normalizeTokens = (value) => value.toLowerCase().replace(/[^a-z0-9\s]/g, "").split(/\s+/).filter((t) => t.length >= 3);
+    const baseTokens = normalizeTokens(query);
+    if (baseTokens.length === 0) return "";
+    const synonymMap = this.options.config?.synonymMap;
+    const expanded = [];
+    const seen = /* @__PURE__ */ new Set();
+    const pushNormalized = (value) => {
+      for (const token of normalizeTokens(value)) {
+        if (expanded.length >= 12) return false;
+        if (seen.has(token)) continue;
+        seen.add(token);
+        expanded.push(token);
+      }
+      return true;
+    };
+    for (const t of baseTokens) {
+      if (!pushNormalized(t)) break;
+      if (synonymMap) {
+        const synonyms = synonymMap[t];
+        if (Array.isArray(synonyms)) {
+          for (const s of synonyms) {
+            if (typeof s === "string") {
+              if (!pushNormalized(s)) break;
+            }
+          }
+        }
+      }
+    }
+    return expanded.map((t) => `"${t}"*`).join(" OR ");
   }
   async read(entityId, query) {
     const ftsQuery = this.formatSearchQuery(query);
@@ -322,6 +488,9 @@ var WikiMemory = class {
     }));
     return { facts, tasks, events: events.reverse() };
   }
+  async getMemoryBundle(entityId) {
+    return this._getFullBundle(entityId, { maxEvents: 10 });
+  }
   async write(entityId, event) {
     const id = generateId("evt_");
     const now = Date.now();
@@ -342,7 +511,7 @@ var WikiMemory = class {
     let memoryCheckpoint = cp?.memory_checkpoint || 0;
     if (memoryCheckpoint > count) memoryCheckpoint = 0;
     if (count - memoryCheckpoint >= threshold) {
-      const jobKey = `${this.prefix}:${entityId}`;
+      const jobKey = this._librarianKey(entityId);
       if (!this.activeMaintenanceJobs.has(jobKey)) {
         this.activeMaintenanceJobs.add(jobKey);
         this.runLibrarianThenMaybeHeal(entityId, count).catch(console.error).finally(() => this.activeMaintenanceJobs.delete(jobKey));
@@ -361,12 +530,20 @@ var WikiMemory = class {
     let healCheckpoint = cp?.heal_checkpoint || 0;
     if (healCheckpoint > currentEventCount) healCheckpoint = 0;
     if (currentEventCount - healCheckpoint >= autoHealThreshold) {
-      await this._doRunHeal(entityId);
-      await this.db.runAsync(`
-        INSERT INTO ${this.prefix}checkpoints (entity_id, heal_checkpoint) 
-        VALUES (?, ?) 
-        ON CONFLICT(entity_id) DO UPDATE SET heal_checkpoint = ?
-      `, [entityId, currentEventCount, currentEventCount]);
+      const healKey = this._healKey(entityId);
+      if (!this.activeMaintenanceJobs.has(healKey)) {
+        this.activeMaintenanceJobs.add(healKey);
+        try {
+          await this._doRunHeal(entityId);
+          await this.db.runAsync(`
+            INSERT INTO ${this.prefix}checkpoints (entity_id, heal_checkpoint) 
+            VALUES (?, ?) 
+            ON CONFLICT(entity_id) DO UPDATE SET heal_checkpoint = ?
+          `, [entityId, currentEventCount, currentEventCount]);
+        } finally {
+          this.activeMaintenanceJobs.delete(healKey);
+        }
+      }
     }
   }
   async _doRunLibrarian(entityId) {
@@ -509,8 +686,10 @@ The following document anchors are provided for contradiction detection only. Do
     });
   }
   async runLibrarian(entityId) {
-    const jobKey = `${this.prefix}:${entityId}`;
-    if (this.activeMaintenanceJobs.has(jobKey)) return;
+    const jobKey = this._librarianKey(entityId);
+    if (this.activeMaintenanceJobs.has(jobKey)) {
+      throw new WikiBusyError("librarian", entityId);
+    }
     this.activeMaintenanceJobs.add(jobKey);
     try {
       await this._doRunLibrarian(entityId);
@@ -519,8 +698,10 @@ The following document anchors are provided for contradiction detection only. Do
     }
   }
   async runHeal(entityId) {
-    const jobKey = `${this.prefix}:${entityId}`;
-    if (this.activeMaintenanceJobs.has(jobKey)) return;
+    const jobKey = this._healKey(entityId);
+    if (this.activeMaintenanceJobs.has(jobKey)) {
+      throw new WikiBusyError("heal", entityId);
+    }
     this.activeMaintenanceJobs.add(jobKey);
     try {
       await this._doRunHeal(entityId);
@@ -528,6 +709,180 @@ The following document anchors are provided for contradiction detection only. Do
       this.activeMaintenanceJobs.delete(jobKey);
     }
   }
+  getEntityStatus(entityId) {
+    const ingestPrefix = `${this.prefix}:${entityId}:`;
+    let ingesting = false;
+    for (const k of this.activeIngestJobs) {
+      if (k.startsWith(ingestPrefix)) {
+        ingesting = true;
+        break;
+      }
+    }
+    return {
+      ingesting,
+      librarian: this.activeMaintenanceJobs.has(this._librarianKey(entityId)),
+      heal: this.activeMaintenanceJobs.has(this._healKey(entityId))
+    };
+  }
+  async _getFullBundle(entityId, opts) {
+    const maxEvents = opts?.maxEvents;
+    const eventsQuery = maxEvents != null ? `SELECT * FROM ${this.prefix}events WHERE entity_id = ? ORDER BY created_at DESC LIMIT ?` : `SELECT * FROM ${this.prefix}events WHERE entity_id = ? ORDER BY created_at ASC`;
+    const eventsParams = maxEvents != null ? [entityId, maxEvents] : [entityId];
+    const [factsRaw, tasks, eventsRaw] = await Promise.all([
+      this.db.getAllAsync(
+        `SELECT * FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL ORDER BY updated_at DESC`,
+        [entityId]
+      ),
+      this.db.getAllAsync(
+        `SELECT * FROM ${this.prefix}tasks WHERE entity_id = ? AND deleted_at IS NULL ORDER BY priority DESC, created_at ASC`,
+        [entityId]
+      ),
+      this.db.getAllAsync(eventsQuery, eventsParams)
+    ]);
+    const facts = factsRaw.map((f) => ({
+      ...f,
+      tags: typeof f.tags === "string" ? JSON.parse(f.tags) : f.tags
+    }));
+    const events = maxEvents != null ? eventsRaw.slice().reverse() : eventsRaw;
+    return { facts, tasks, events };
+  }
+  async exportDump(entityIds) {
+    let ids;
+    if (entityIds && entityIds.length > 0) {
+      ids = Array.from(new Set(entityIds));
+    } else {
+      const rows = await this.db.getAllAsync(`
+        SELECT DISTINCT entity_id FROM (
+          SELECT entity_id FROM ${this.prefix}entries WHERE deleted_at IS NULL
+          UNION
+          SELECT entity_id FROM ${this.prefix}tasks WHERE deleted_at IS NULL
+          UNION
+          SELECT entity_id FROM ${this.prefix}events
+        ) ORDER BY entity_id
+      `);
+      ids = rows.map((r) => r.entity_id);
+    }
+    const entities = {};
+    const BATCH = 3;
+    for (let i = 0; i < ids.length; i += BATCH) {
+      const batch = ids.slice(i, i + BATCH);
+      const batchResults = await Promise.all(
+        batch.map(async (id) => [id, await this._getFullBundle(id)])
+      );
+      for (const [id, bundle] of batchResults) {
+        entities[id] = bundle;
+      }
+    }
+    return { generatedAt: Date.now(), entities };
+  }
+  async importDump(dump, opts) {
+    const merge = opts?.merge ?? false;
+    for (const [entityId, bundle] of Object.entries(dump.entities)) {
+      await this.db.withTransactionAsync(async () => {
+        if (!merge) {
+          const now = Date.now();
+          await this.db.runAsync(
+            `UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE entity_id = ? AND deleted_at IS NULL`,
+            [now, now, entityId]
+          );
+          await this.db.runAsync(
+            `UPDATE ${this.prefix}tasks SET deleted_at = ?, updated_at = ? WHERE entity_id = ? AND deleted_at IS NULL`,
+            [now, now, entityId]
+          );
+          await this.db.runAsync(
+            `DELETE FROM ${this.prefix}checkpoints WHERE entity_id = ?`,
+            [entityId]
+          );
+        }
+        const factIds = bundle.facts.map((fact) => fact.id);
+        const existingFactsById = /* @__PURE__ */ new Map();
+        const factLookupChunkSize = 500;
+        for (let i = 0; i < factIds.length; i += factLookupChunkSize) {
+          const factIdChunk = factIds.slice(i, i + factLookupChunkSize);
+          if (factIdChunk.length === 0) continue;
+          const placeholders = factIdChunk.map(() => "?").join(", ");
+          const existingFacts = await this.db.getAllAsync(
+            `SELECT id, entity_id, updated_at FROM ${this.prefix}entries WHERE id IN (${placeholders})`,
+            factIdChunk
+          );
+          for (const existingFact of existingFacts) {
+            existingFactsById.set(existingFact.id, existingFact);
+          }
+        }
+        for (const fact of bundle.facts) {
+          const tagsJson = JSON.stringify(Array.isArray(fact.tags) ? fact.tags : []);
+          const safeUpdatedAt = Number.isFinite(fact.updated_at) ? fact.updated_at : 0;
+          const existing = existingFactsById.get(fact.id);
+          if (existing) {
+            if (existing.entity_id !== entityId) {
+              this._warnCrossEntityCollision("entry", fact.id, existing.entity_id, entityId);
+              continue;
+            }
+            if (merge) {
+              if (safeUpdatedAt <= existing.updated_at) continue;
+            }
+            await this.db.runAsync(
+              `UPDATE ${this.prefix}entries SET entity_id = ?, title = ?, body = ?, tags = ?, confidence = ?, source_type = ?, source_hash = ?, source_ref = ?, created_at = ?, updated_at = ?, last_accessed_at = ?, access_count = ?, deleted_at = ? WHERE id = ?`,
+              [entityId, fact.title, fact.body, tagsJson, fact.confidence, fact.source_type, fact.source_hash, fact.source_ref, fact.created_at, safeUpdatedAt, fact.last_accessed_at, fact.access_count, fact.deleted_at, fact.id]
+            );
+            existingFactsById.set(fact.id, { id: fact.id, entity_id: entityId, updated_at: safeUpdatedAt });
+          } else {
+            await this.db.runAsync(
+              `INSERT INTO ${this.prefix}entries (id, entity_id, title, body, tags, confidence, source_type, source_hash, source_ref, created_at, updated_at, last_accessed_at, access_count, deleted_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+              [fact.id, entityId, fact.title, fact.body, tagsJson, fact.confidence, fact.source_type, fact.source_hash, fact.source_ref, fact.created_at, safeUpdatedAt, fact.last_accessed_at, fact.access_count, fact.deleted_at]
+            );
+            existingFactsById.set(fact.id, { id: fact.id, entity_id: entityId, updated_at: safeUpdatedAt });
+          }
+        }
+        const taskIds = bundle.tasks.map((task) => task.id);
+        const existingTasksById = /* @__PURE__ */ new Map();
+        const taskLookupChunkSize = 500;
+        for (let i = 0; i < taskIds.length; i += taskLookupChunkSize) {
+          const taskIdChunk = taskIds.slice(i, i + taskLookupChunkSize);
+          if (taskIdChunk.length === 0) continue;
+          const placeholders = taskIdChunk.map(() => "?").join(", ");
+          const existingTasks = await this.db.getAllAsync(
+            `SELECT id, entity_id, updated_at FROM ${this.prefix}tasks WHERE id IN (${placeholders})`,
+            taskIdChunk
+          );
+          for (const existingTask of existingTasks) {
+            existingTasksById.set(existingTask.id, existingTask);
+          }
+        }
+        for (const task of bundle.tasks) {
+          const safeUpdatedAt = Number.isFinite(task.updated_at) ? task.updated_at : 0;
+          const existing = existingTasksById.get(task.id);
+          if (existing) {
+            if (existing.entity_id !== entityId) {
+              this._warnCrossEntityCollision("task", task.id, existing.entity_id, entityId);
+              continue;
+            }
+            if (merge) {
+              if (safeUpdatedAt <= existing.updated_at) continue;
+            }
+            await this.db.runAsync(
+              `UPDATE ${this.prefix}tasks SET entity_id = ?, description = ?, status = ?, priority = ?, created_at = ?, updated_at = ?, resolved_at = ?, deleted_at = ? WHERE id = ?`,
+              [entityId, task.description, task.status, task.priority, task.created_at, safeUpdatedAt, task.resolved_at, task.deleted_at, task.id]
+            );
+            existingTasksById.set(task.id, { id: task.id, entity_id: entityId, updated_at: safeUpdatedAt });
+          } else {
+            await this.db.runAsync(
+              `INSERT INTO ${this.prefix}tasks (id, entity_id, description, status, priority, created_at, updated_at, resolved_at, deleted_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+              [task.id, entityId, task.description, task.status, task.priority, task.created_at, safeUpdatedAt, task.resolved_at, task.deleted_at]
+            );
+            existingTasksById.set(task.id, { id: task.id, entity_id: entityId, updated_at: safeUpdatedAt });
+          }
+        }
+        for (const event of bundle.events) {
+          await this.db.runAsync(
+            `INSERT OR IGNORE INTO ${this.prefix}events (id, entity_id, event_type, summary, related_entry_id, created_at)
+             VALUES (?, ?, ?, ?, ?, ?)`,
+            [event.id, entityId, event.event_type, event.summary, event.related_entry_id ?? null, event.created_at]
+          );
+        }
+      });
+    }
+  }
   async forget(entityId, params) {
     const now = Date.now();
     let deletedEntries = 0;
@@ -582,70 +937,165 @@ The following document anchors are provided for contradiction detection only. Do
     if (!sourceRef) throw new Error("Invalid sourceRef");
     const sourceHash = normalizeSourceHash(params.sourceHash);
     if (!sourceHash) throw new Error("Invalid sourceHash (must be 64-char hex string)");
-    const maxChunkLength = params.maxChunkLength ?? this.options.config?.maxChunkLength ?? 6e3;
-    if (!Number.isInteger(maxChunkLength) || maxChunkLength < 2) {
-      throw new Error("maxChunkLength must be an integer greater than or equal to 2");
-    }
+    const maxChunkLength = params.maxChunkLength ?? this.options.config?.maxChunkLength ?? 12e3;
+    const rawOverlap = params.chunkOverlap ?? this.options.config?.chunkOverlap ?? 400;
+    const chunkOverlap = Math.min(
+      Number.isFinite(rawOverlap) && rawOverlap >= 0 ? Math.floor(rawOverlap) : 400,
+      maxChunkLength - 1
+    );
+    const rawConcurrency = params.chunkConcurrency ?? this.options.config?.chunkConcurrency ?? 1;
+    const chunkConcurrency = Number.isFinite(rawConcurrency) && rawConcurrency >= 1 ? Math.floor(rawConcurrency) : 1;
     if (typeof params.documentChunk !== "string") {
       throw new Error(`documentChunk must be a string, received ${typeof params.documentChunk}`);
     }
-    const chunks = [];
-    let truncated = false;
-    let text = params.documentChunk.trim();
-    if (text.length === 0) {
-      return { truncated: false, chunks: 0 };
+    const jobKey = `${this.prefix}:${entityId}:${sourceRef}`;
+    if (this.activeIngestJobs.has(jobKey)) {
+      throw new WikiBusyError("ingest", entityId);
     }
-    while (text.length > 0) {
-      if (text.length <= maxChunkLength) {
-        chunks.push(text);
-        break;
-      }
-      const searchArea = text.slice(0, maxChunkLength + 1);
-      const match = searchArea.match(/[.!?]\s+(?![\s\S]*[.!?]\s+)/);
-      if (match && match.index !== void 0) {
-        const splitPoint = Math.min(match.index + match[0].length, maxChunkLength);
-        const chunk = safeSlice(text, 0, splitPoint);
-        chunks.push(chunk);
-        text = text.slice(chunk.length);
-      } else {
-        truncated = true;
-        const chunk = safeSlice(text, 0, maxChunkLength);
-        chunks.push(chunk);
-        text = text.slice(chunk.length);
+    this.activeIngestJobs.add(jobKey);
+    try {
+      const { chunks, truncated } = chunkText(params.documentChunk, maxChunkLength, chunkOverlap);
+      if (chunks.length === 0) {
+        return { truncated: false, chunks: 0 };
       }
-    }
-    const allValidFacts = [];
-    for (const chunk of chunks) {
-      const userPrompt = `Document Chunk:
+      const chunkResults = await withConcurrency(
+        chunks.map((chunk) => async () => {
+          const userPrompt = `Document Chunk:
 ${chunk}`;
-      const responseText = await this.options.llmProvider.generateText({
-        systemPrompt: INGEST_SYSTEM_PROMPT,
-        userPrompt
+          const responseText = await this.options.llmProvider.generateText({
+            systemPrompt: INGEST_SYSTEM_PROMPT,
+            userPrompt
+          });
+          const result = parseJsonResponse(responseText);
+          return (Array.isArray(result.facts) ? result.facts : []).map(validateFact).filter((f) => f !== null);
+        }),
+        chunkConcurrency
+      );
+      const seen = /* @__PURE__ */ new Set();
+      const allValidFacts = [];
+      for (const facts of chunkResults) {
+        for (const fact of facts) {
+          const normalized = fact.title.trim().toLowerCase().replace(/\s+/g, " ");
+          if (!seen.has(normalized)) {
+            seen.add(normalized);
+            allValidFacts.push(fact);
+          }
+        }
+      }
+      const now = Date.now();
+      await this.db.withTransactionAsync(async () => {
+        await this.db.runAsync(
+          `UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE source_ref = ? AND entity_id = ? AND deleted_at IS NULL`,
+          [now, now, sourceRef, entityId]
+        );
+        for (const fact of allValidFacts) {
+          const id = generateId("fact_");
+          await this.db.runAsync(
+            `INSERT INTO ${this.prefix}entries (id, entity_id, title, body, tags, confidence, source_type, source_hash, source_ref, created_at, updated_at)
+           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+            [id, entityId, fact.title, fact.body, JSON.stringify(fact.tags), fact.confidence, "user_document", sourceHash, sourceRef, now, now]
+          );
+        }
       });
-      const result = parseJsonResponse(responseText);
-      const validFacts = (Array.isArray(result.facts) ? result.facts : []).map(validateFact).filter((f) => f !== null);
-      allValidFacts.push(...validFacts);
+      return { truncated, chunks: chunks.length };
+    } finally {
+      this.activeIngestJobs.delete(jobKey);
     }
-    const now = Date.now();
-    await this.db.withTransactionAsync(async () => {
-      await this.db.runAsync(`UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE source_ref = ? AND entity_id = ? AND deleted_at IS NULL`, [now, now, sourceRef, entityId]);
-      for (const fact of allValidFacts) {
-        const id = generateId("fact_");
-        await this.db.runAsync(`
-                INSERT INTO ${this.prefix}entries (id, entity_id, title, body, tags, confidence, source_type, source_hash, source_ref, created_at, updated_at)
-                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-            `, [id, entityId, fact.title, fact.body, JSON.stringify(fact.tags), fact.confidence, "user_document", sourceHash, sourceRef, now, now]);
-      }
-    });
-    return { truncated, chunks: chunks.length };
   }
 };
 
+// src/utils/formatMemoryDump.ts
+function renderFact(f) {
+  const tags = (f.tags || []).join(", ");
+  const source = f.source_ref ?? f.source_type;
+  return `### ${f.title}
+**Tags:** ${tags}
+**Confidence:** ${f.confidence}
+**Source:** ${source}
+
+${f.body}
+
+---
+`;
+}
+function renderTask(t) {
+  const checked = t.status === "done" ? "x" : " ";
+  const note = t.status === "done" ? " (done)" : t.status === "abandoned" ? " (abandoned)" : t.status === "in_progress" ? " (in progress)" : "";
+  return `- [${checked}] ${t.description}${note}
+`;
+}
+function renderEvent(e) {
+  const ts = new Date(e.created_at).toISOString();
+  return `- [${ts}] (${e.event_type}) ${e.summary}
+`;
+}
+function renderEntity(entityId, bundle, generatedAt) {
+  const lines = [];
+  lines.push(`# Memory Dump: ${entityId}`);
+  lines.push(`Generated: ${new Date(generatedAt).toISOString()}`);
+  lines.push("");
+  lines.push("## Facts");
+  lines.push("");
+  if (bundle.facts.length === 0) {
+    lines.push("_(none)_\n");
+  } else {
+    for (const f of bundle.facts) lines.push(renderFact(f));
+  }
+  lines.push("## Tasks");
+  lines.push("");
+  if (bundle.tasks.length === 0) {
+    lines.push("_(none)_\n");
+  } else {
+    for (const t of bundle.tasks) lines.push(renderTask(t));
+  }
+  lines.push("");
+  lines.push("## Recent Events");
+  lines.push("");
+  if (bundle.events.length === 0) {
+    lines.push("_(none)_\n");
+  } else {
+    for (const e of bundle.events) lines.push(renderEvent(e));
+  }
+  return lines.join("\n");
+}
+function shortHash(value) {
+  let h1 = 5381;
+  let h2 = 52711;
+  for (let i = 0; i < value.length; i += 1) {
+    const c = value.charCodeAt(i);
+    h1 = Math.imul(h1, 33) ^ c;
+    h2 = Math.imul(h2, 31) ^ c;
+  }
+  return (h1 >>> 0).toString(16).padStart(8, "0") + (h2 >>> 0).toString(16).padStart(8, "0");
+}
+function formatEntityFileName(entityId) {
+  const normalized = entityId.normalize("NFKC");
+  const sanitized = normalized.replace(/[^A-Za-z0-9._-]+/g, "_").replace(/^\.+/, "_").replace(/_+/g, "_").replace(/^[_-]+|[_-]+$/g, "");
+  const MAX_BASE = 200;
+  const trimmed = sanitized.length > MAX_BASE ? sanitized.slice(0, MAX_BASE) : sanitized;
+  const baseName = trimmed && trimmed !== "." && trimmed !== ".." ? trimmed : "entity";
+  const needsSuffix = baseName !== entityId || sanitized.length > MAX_BASE;
+  const uniqueBaseName = needsSuffix ? `${baseName}-${shortHash(entityId)}` : baseName;
+  return `${uniqueBaseName}.md`;
+}
+function formatMemoryDump(dump) {
+  const files = Object.entries(dump.entities).map(([entityId, bundle]) => ({
+    name: formatEntityFileName(entityId),
+    content: renderEntity(entityId, bundle, dump.generatedAt)
+  }));
+  return {
+    manifest: JSON.stringify(dump, null, 2),
+    files
+  };
+}
+
 // src/index.ts
 function createWiki(db, options) {
   return new WikiMemory(db, options);
 }
 export {
+  WikiBusyError,
   WikiMemory,
-  createWiki
+  createWiki,
+  formatMemoryDump
 };