@equationalapplications/expo-llm-wiki 2.2.0 → 2.3.0

package/dist/index.js

@@ -23,6 +23,7 @@ __export(index_exports, {
   WikiBusyError: () => WikiBusyError,
   WikiMemory: () => WikiMemory,
   createWiki: () => createWiki,
+  formatContext: () => formatContext,
   formatMemoryDump: () => formatMemoryDump
 });
 module.exports = __toCommonJS(index_exports);
@@ -110,9 +111,64 @@ async function setupDatabase(db, prefix) {
       heal_checkpoint INTEGER NOT NULL DEFAULT 0,
       memory_checkpoint INTEGER NOT NULL DEFAULT 0
     );
+
+    CREATE TABLE IF NOT EXISTS ${prefix}meta (
+      key TEXT PRIMARY KEY,
+      value TEXT NOT NULL
+    );
   `);
 }
 
+// src/db/migrations.ts
+var MIGRATIONS = [
+  {
+    version: 1,
+    description: "Rebuild FTS5 with porter unicode61 tokenizer",
+    run: async (db, prefix) => {
+      await db.withTransactionAsync(async () => {
+        await db.execAsync(`
+          DROP TRIGGER IF EXISTS ${prefix}entries_ai;
+          DROP TRIGGER IF EXISTS ${prefix}entries_ad;
+          DROP TRIGGER IF EXISTS ${prefix}entries_au;
+          DROP TABLE IF EXISTS ${prefix}entries_fts;
+          CREATE VIRTUAL TABLE ${prefix}entries_fts USING fts5(
+            title,
+            body,
+            tags,
+            content='${prefix}entries',
+            content_rowid='rowid',
+            tokenize='porter unicode61'
+          );
+          INSERT INTO ${prefix}entries_fts(rowid, title, body, tags)
+            SELECT rowid, title, body, tags FROM ${prefix}entries;
+          CREATE TRIGGER ${prefix}entries_ai AFTER INSERT ON ${prefix}entries BEGIN
+            INSERT INTO ${prefix}entries_fts(rowid, title, body, tags)
+            VALUES (new.rowid, new.title, new.body, new.tags);
+          END;
+          CREATE TRIGGER ${prefix}entries_ad AFTER DELETE ON ${prefix}entries BEGIN
+            INSERT INTO ${prefix}entries_fts(${prefix}entries_fts, rowid, title, body, tags)
+            VALUES ('delete', old.rowid, old.title, old.body, old.tags);
+          END;
+          CREATE TRIGGER ${prefix}entries_au AFTER UPDATE ON ${prefix}entries BEGIN
+            INSERT INTO ${prefix}entries_fts(${prefix}entries_fts, rowid, title, body, tags)
+            VALUES ('delete', old.rowid, old.title, old.body, old.tags);
+            INSERT INTO ${prefix}entries_fts(rowid, title, body, tags)
+            VALUES (new.rowid, new.title, new.body, new.tags);
+          END;
+        `);
+      });
+    }
+  }
+];
+for (let i = 1; i < MIGRATIONS.length; i++) {
+  if (MIGRATIONS[i].version <= MIGRATIONS[i - 1].version) {
+    throw new Error(
+      `migrations.ts: MIGRATIONS must be in strictly ascending version order. Found version ${MIGRATIONS[i].version} after ${MIGRATIONS[i - 1].version} at index ${i}.`
+    );
+  }
+}
+var CURRENT_SCHEMA_VERSION = MIGRATIONS.length > 0 ? MIGRATIONS[MIGRATIONS.length - 1].version : 0;
+
 // src/types.ts
 var WikiBusyError = class extends Error {
   operation;
@@ -373,45 +429,54 @@ var WikiMemory = class {
     this.prefix = options.config?.tablePrefix || "llm_wiki_";
   }
   async setup() {
-    await setupDatabase(this.db, this.prefix);
-    const ftsMeta = await this.db.getFirstAsync(
-      `SELECT sql FROM sqlite_master WHERE type='table' AND name=?`,
-      [`${this.prefix}entries_fts`]
+    const entriesExistedBeforeSetup = await this.db.getFirstAsync(
+      `SELECT name FROM sqlite_master WHERE type='table' AND name=?`,
+      [`${this.prefix}entries`]
     );
-    const hasPorterTokenizer = /tokenize\s*=\s*['"]porter\s+unicode61['"]/i.test(ftsMeta?.sql ?? "");
-    if (ftsMeta?.sql && !hasPorterTokenizer) {
-      await this.db.withTransactionAsync(async () => {
-        await this.db.execAsync(`
-          DROP TRIGGER IF EXISTS ${this.prefix}entries_ai;
-          DROP TRIGGER IF EXISTS ${this.prefix}entries_ad;
-          DROP TRIGGER IF EXISTS ${this.prefix}entries_au;
-          DROP TABLE IF EXISTS ${this.prefix}entries_fts;
-          CREATE VIRTUAL TABLE ${this.prefix}entries_fts USING fts5(
-            title,
-            body,
-            tags,
-            content='${this.prefix}entries',
-            content_rowid='rowid',
-            tokenize='porter unicode61'
-          );
-          INSERT INTO ${this.prefix}entries_fts(rowid, title, body, tags)
-            SELECT rowid, title, body, tags FROM ${this.prefix}entries;
-          CREATE TRIGGER ${this.prefix}entries_ai AFTER INSERT ON ${this.prefix}entries BEGIN
-            INSERT INTO ${this.prefix}entries_fts(rowid, title, body, tags)
-            VALUES (new.rowid, new.title, new.body, new.tags);
-          END;
-          CREATE TRIGGER ${this.prefix}entries_ad AFTER DELETE ON ${this.prefix}entries BEGIN
-            INSERT INTO ${this.prefix}entries_fts(${this.prefix}entries_fts, rowid, title, body, tags)
-            VALUES ('delete', old.rowid, old.title, old.body, old.tags);
-          END;
-          CREATE TRIGGER ${this.prefix}entries_au AFTER UPDATE ON ${this.prefix}entries BEGIN
-            INSERT INTO ${this.prefix}entries_fts(${this.prefix}entries_fts, rowid, title, body, tags)
-            VALUES ('delete', old.rowid, old.title, old.body, old.tags);
-            INSERT INTO ${this.prefix}entries_fts(rowid, title, body, tags)
-            VALUES (new.rowid, new.title, new.body, new.tags);
-          END;
-        `);
-      });
+    await setupDatabase(this.db, this.prefix);
+    let currentVersion;
+    if (!entriesExistedBeforeSetup) {
+      await this.db.runAsync(
+        `INSERT OR REPLACE INTO ${this.prefix}meta (key, value) VALUES ('schema_version', ?)`,
+        [String(CURRENT_SCHEMA_VERSION)]
+      );
+      currentVersion = CURRENT_SCHEMA_VERSION;
+    } else {
+      const metaRow = await this.db.getFirstAsync(
+        `SELECT value FROM ${this.prefix}meta WHERE key = 'schema_version'`
+      );
+      if (metaRow) {
+        currentVersion = parseInt(metaRow.value, 10);
+        if (!Number.isFinite(currentVersion)) currentVersion = 0;
+      } else {
+        const ftsMeta = await this.db.getFirstAsync(
+          `SELECT sql FROM sqlite_master WHERE type='table' AND name=?`,
+          [`${this.prefix}entries_fts`]
+        );
+        const hasPorter = /tokenize\s*=\s*['"]porter\s+unicode61['"]/i.test(ftsMeta?.sql ?? "");
+        currentVersion = hasPorter ? 1 : 0;
+      }
+    }
+    for (const migration of MIGRATIONS) {
+      if (migration.version > currentVersion) {
+        await migration.run(this.db, this.prefix);
+        await this.db.runAsync(
+          `INSERT OR REPLACE INTO ${this.prefix}meta (key, value) VALUES ('schema_version', ?)`,
+          [String(migration.version)]
+        );
+        currentVersion = migration.version;
+      }
+    }
+    if (entriesExistedBeforeSetup) {
+      const metaCheck = await this.db.getFirstAsync(
+        `SELECT value FROM ${this.prefix}meta WHERE key = 'schema_version'`
+      );
+      if (!metaCheck) {
+        await this.db.runAsync(
+          `INSERT OR REPLACE INTO ${this.prefix}meta (key, value) VALUES ('schema_version', ?)`,
+          [String(currentVersion)]
+        );
+      }
     }
     const rows = await this.db.getAllAsync(`
       SELECT rowid, source_ref FROM ${this.prefix}entries
@@ -436,6 +501,101 @@ var WikiMemory = class {
       }
     });
   }
+  async hasChanged(entityId, sourceRef, sourceHash) {
+    const normalizedRef = normalizeSourceRef(sourceRef);
+    if (!normalizedRef) {
+      throw new Error(`Invalid sourceRef: "${sourceRef}"`);
+    }
+    const normalizedHash = normalizeSourceHash(sourceHash);
+    if (!normalizedHash) {
+      throw new Error(`Invalid sourceHash: must be a 64-character hex string (normalized to lowercase)`);
+    }
+    const row = await this.db.getFirstAsync(
+      `SELECT source_hash FROM ${this.prefix}entries
+       WHERE entity_id = ? AND source_ref = ? AND deleted_at IS NULL
+       ORDER BY updated_at DESC
+       LIMIT 1`,
+      [entityId, normalizedRef]
+    );
+    if (!row) return true;
+    const normalizedStoredHash = row.source_hash ? normalizeSourceHash(row.source_hash) : null;
+    return normalizedStoredHash !== normalizedHash;
+  }
+  _pruneKey(entityId) {
+    return `${this.prefix}:${entityId}:prune`;
+  }
+  _validatePruneDuration(value, name) {
+    if (value !== null && value !== void 0 && (typeof value !== "number" || !isFinite(value) || value < 0)) {
+      throw new Error(`Invalid ${name}: must be a non-negative finite number or null`);
+    }
+  }
+  async runPrune(entityId, options) {
+    const pruneKey = this._pruneKey(entityId);
+    const ingestPrefix = `${this.prefix}:${entityId}:`;
+    let isIngestRunning = false;
+    for (const k of this.activeIngestJobs) {
+      if (k.startsWith(ingestPrefix)) {
+        isIngestRunning = true;
+        break;
+      }
+    }
+    let blockingOperation = null;
+    if (this.activeMaintenanceJobs.has(pruneKey)) {
+      blockingOperation = "prune";
+    } else if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) {
+      blockingOperation = "librarian";
+    } else if (this.activeMaintenanceJobs.has(this._healKey(entityId))) {
+      blockingOperation = "heal";
+    } else if (isIngestRunning) {
+      blockingOperation = "ingest";
+    }
+    if (blockingOperation !== null) {
+      throw new WikiBusyError(blockingOperation, entityId);
+    }
+    this.activeMaintenanceJobs.add(pruneKey);
+    try {
+      const retainSoftDeletedFor = options?.retainSoftDeletedFor !== void 0 ? options.retainSoftDeletedFor : this.options.config?.pruneRetainSoftDeletedFor ?? 7;
+      const retainEventsFor = options?.retainEventsFor !== void 0 ? options.retainEventsFor : this.options.config?.pruneEventsAfter ?? 30;
+      const vacuum = options?.vacuum ?? false;
+      this._validatePruneDuration(retainSoftDeletedFor, "retainSoftDeletedFor");
+      this._validatePruneDuration(retainEventsFor, "retainEventsFor");
+      const now = Date.now();
+      let deletedEntries = 0;
+      let deletedTasks = 0;
+      let deletedEvents = 0;
+      if (retainSoftDeletedFor !== null) {
+        const cutoff = now - retainSoftDeletedFor * 864e5;
+        const entryResult = await this.db.runAsync(
+          `DELETE FROM ${this.prefix}entries
+           WHERE entity_id = ? AND deleted_at IS NOT NULL AND deleted_at < ?`,
+          [entityId, cutoff]
+        );
+        deletedEntries = entryResult.changes;
+        const taskResult = await this.db.runAsync(
+          `DELETE FROM ${this.prefix}tasks
+           WHERE entity_id = ? AND deleted_at IS NOT NULL AND deleted_at < ?`,
+          [entityId, cutoff]
+        );
+        deletedTasks = taskResult.changes;
+      }
+      if (retainEventsFor !== null) {
+        const cutoff = now - retainEventsFor * 864e5;
+        const eventResult = await this.db.runAsync(
+          `DELETE FROM ${this.prefix}events
+           WHERE entity_id = ? AND created_at < ?`,
+          [entityId, cutoff]
+        );
+        deletedEvents = eventResult.changes;
+      }
+      if (vacuum) {
+        await this.db.execAsync(`PRAGMA wal_checkpoint(TRUNCATE)`);
+        await this.db.execAsync(`VACUUM`);
+      }
+      return { entries: deletedEntries, tasks: deletedTasks, events: deletedEvents };
+    } finally {
+      this.activeMaintenanceJobs.delete(pruneKey);
+    }
+  }
   formatSearchQuery(query) {
     const normalizeTokens = (value) => value.toLowerCase().replace(/[^a-z0-9\s]/g, "").split(/\s+/).filter((t) => t.length >= 3);
     const baseTokens = normalizeTokens(query);
@@ -541,7 +701,7 @@ var WikiMemory = class {
     if (memoryCheckpoint > count) memoryCheckpoint = 0;
     if (count - memoryCheckpoint >= threshold) {
       const jobKey = this._librarianKey(entityId);
-      if (!this.activeMaintenanceJobs.has(jobKey)) {
+      if (!this.activeMaintenanceJobs.has(jobKey) && !this.activeMaintenanceJobs.has(this._pruneKey(entityId))) {
         this.activeMaintenanceJobs.add(jobKey);
         this.runLibrarianThenMaybeHeal(entityId, count).catch(console.error).finally(() => this.activeMaintenanceJobs.delete(jobKey));
       }
@@ -719,6 +879,9 @@ The following document anchors are provided for contradiction detection only. Do
     if (this.activeMaintenanceJobs.has(jobKey)) {
       throw new WikiBusyError("librarian", entityId);
     }
+    if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) {
+      throw new WikiBusyError("prune", entityId);
+    }
     this.activeMaintenanceJobs.add(jobKey);
     try {
       await this._doRunLibrarian(entityId);
@@ -731,6 +894,9 @@ The following document anchors are provided for contradiction detection only. Do
     if (this.activeMaintenanceJobs.has(jobKey)) {
       throw new WikiBusyError("heal", entityId);
     }
+    if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) {
+      throw new WikiBusyError("prune", entityId);
+    }
     this.activeMaintenanceJobs.add(jobKey);
     try {
       await this._doRunHeal(entityId);
@@ -981,6 +1147,9 @@ The following document anchors are provided for contradiction detection only. Do
     if (this.activeIngestJobs.has(jobKey)) {
       throw new WikiBusyError("ingest", entityId);
     }
+    if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) {
+      throw new WikiBusyError("prune", entityId);
+    }
     this.activeIngestJobs.add(jobKey);
     try {
       const { chunks, truncated } = chunkText(params.documentChunk, maxChunkLength, chunkOverlap);
@@ -1118,6 +1287,121 @@ function formatMemoryDump(dump) {
   };
 }
 
+// src/utils/formatContext.ts
+function validateMaxOption(value, name) {
+  if (!isFinite(value) || value < 0) {
+    throw new Error(`Invalid ${name}: must be a non-negative finite number`);
+  }
+}
+var CONFIDENCE_WEIGHT = {
+  certain: 1,
+  inferred: 0.6,
+  tentative: 0.3
+};
+function scoreFactFor(fact, weights, now) {
+  const confW = CONFIDENCE_WEIGHT[fact.confidence] ?? 0.3;
+  const ageDays = (now - fact.updated_at) / 864e5;
+  const recencyDecay = Math.exp(-ageDays / 30);
+  return confW * weights.confidence + Math.log(1 + fact.access_count) * weights.accessCount + recencyDecay * weights.recency;
+}
+function renderFactMarkdown(fact, includeConfidence, includeTags) {
+  const confPart = includeConfidence ? ` (${fact.confidence})` : "";
+  const tagPart = includeTags && fact.tags.length > 0 ? ` [${fact.tags.join(", ")}]` : "";
+  return `- **${fact.title}**${confPart}${tagPart}
+  ${fact.body.replace(/\n/g, "\n  ")}`;
+}
+function renderFactPlain(fact, includeConfidence, includeTags) {
+  const confPart = includeConfidence ? ` (${fact.confidence})` : "";
+  const tagPart = includeTags && fact.tags.length > 0 ? ` [${fact.tags.join(", ")}]` : "";
+  return `${fact.title}${confPart}${tagPart}: ${fact.body}`;
+}
+function renderTaskMarkdown(task) {
+  return `- [P${task.priority}] ${task.description.replace(/\n/g, "\n  ")} (${task.status})`;
+}
+function renderTaskPlain(task) {
+  return `[P${task.priority}] ${task.description} (${task.status})`;
+}
+function renderEventMarkdown(event) {
+  const ts = new Date(event.created_at).toISOString();
+  return `- [${event.event_type} @ ${ts}] ${event.summary.replace(/\n/g, "\n  ")}`;
+}
+function renderEventPlain(event) {
+  const ts = new Date(event.created_at).toISOString();
+  return `[${event.event_type} @ ${ts}] ${event.summary}`;
+}
+function formatContext(bundle, options) {
+  const opts = {
+    format: options?.format ?? "markdown",
+    maxFacts: options?.maxFacts ?? 10,
+    maxTasks: options?.maxTasks ?? 10,
+    maxEvents: options?.maxEvents ?? 10,
+    includeConfidence: options?.includeConfidence ?? true,
+    includeTags: options?.includeTags ?? true,
+    factWeights: {
+      confidence: options?.factWeights?.confidence ?? 1,
+      accessCount: options?.factWeights?.accessCount ?? 0.3,
+      recency: options?.factWeights?.recency ?? 0.5
+    }
+  };
+  validateMaxOption(opts.maxFacts, "maxFacts");
+  validateMaxOption(opts.maxTasks, "maxTasks");
+  validateMaxOption(opts.maxEvents, "maxEvents");
+  const weights = opts.factWeights;
+  const now = Date.now();
+  const sortedFacts = [...bundle.facts].sort((a, b) => scoreFactFor(b, weights, now) - scoreFactFor(a, weights, now)).slice(0, opts.maxFacts);
+  const sortedTasks = [...bundle.tasks].sort((a, b) => b.priority - a.priority || a.created_at - b.created_at).slice(0, opts.maxTasks);
+  const sortedEvents = [...bundle.events].sort((a, b) => b.created_at - a.created_at).slice(0, opts.maxEvents);
+  if (sortedFacts.length === 0 && sortedTasks.length === 0 && sortedEvents.length === 0) {
+    return "";
+  }
+  const isMarkdown = opts.format === "markdown";
+  const lines = [];
+  if (isMarkdown) {
+    lines.push("## Memory");
+    if (sortedFacts.length > 0) {
+      lines.push("");
+      lines.push("### Known Facts");
+      for (const fact of sortedFacts) {
+        lines.push(renderFactMarkdown(fact, opts.includeConfidence, opts.includeTags));
+      }
+    }
+    if (sortedTasks.length > 0) {
+      lines.push("");
+      lines.push("### Open Tasks");
+      for (const task of sortedTasks) {
+        lines.push(renderTaskMarkdown(task));
+      }
+    }
+    if (sortedEvents.length > 0) {
+      lines.push("");
+      lines.push("### Recent Events");
+      for (const event of sortedEvents) {
+        lines.push(renderEventMarkdown(event));
+      }
+    }
+  } else {
+    if (sortedFacts.length > 0) {
+      lines.push("KNOWN FACTS:");
+      for (const fact of sortedFacts) {
+        lines.push(renderFactPlain(fact, opts.includeConfidence, opts.includeTags));
+      }
+    }
+    if (sortedTasks.length > 0) {
+      lines.push("OPEN TASKS:");
+      for (const task of sortedTasks) {
+        lines.push(renderTaskPlain(task));
+      }
+    }
+    if (sortedEvents.length > 0) {
+      lines.push("RECENT EVENTS:");
+      for (const event of sortedEvents) {
+        lines.push(renderEventPlain(event));
+      }
+    }
+  }
+  return lines.join("\n");
+}
+
 // src/index.ts
 function createWiki(db, options) {
   return new WikiMemory(db, options);
@@ -1127,5 +1411,6 @@ function createWiki(db, options) {
   WikiBusyError,
   WikiMemory,
   createWiki,
+  formatContext,
   formatMemoryDump
 });