@equationalapplications/core-llm-wiki 3.2.0 → 4.1.0

package/dist/index.mjs

@@ -10,7 +10,7 @@ async function setupDatabase(db, prefix) {
       body TEXT NOT NULL,
       tags TEXT NOT NULL DEFAULT '[]',
       confidence TEXT NOT NULL DEFAULT 'inferred',
-      source_type TEXT NOT NULL DEFAULT 'agent_inferred',
+      source_type TEXT NOT NULL DEFAULT 'librarian_inferred',
       source_hash TEXT,
       source_ref TEXT,
       created_at INTEGER NOT NULL,
@@ -412,6 +412,7 @@ var _WikiMemory = class _WikiMemory {
   constructor(db, options) {
     this.activeMaintenanceJobs = /* @__PURE__ */ new Set();
     this.activeIngestJobs = /* @__PURE__ */ new Set();
+    this.statusSubscribers = /* @__PURE__ */ new Map();
     this.miniSearch = new MiniSearch({
       fields: ["title", "body", "tags"],
       storeFields: ["entity_id"],
@@ -587,6 +588,44 @@ var _WikiMemory = class _WikiMemory {
   _warnCrossEntityCollision(type, id, existingEntityId, targetEntityId) {
     console.warn(`[WikiMemory] importDump: ${type} id "${id}" already belongs to entity "${existingEntityId}"; skipping for entity "${targetEntityId}"`);
   }
+  /** Maps pre-rename enum strings from older dumps to current source_type values. */
+  _normalizeImportedSourceType(raw, ctx) {
+    if (raw === "user_document") return "immutable_document";
+    if (raw === "agent_inferred") return "librarian_inferred";
+    const allowed = ["user_stated", "librarian_inferred", "user_confirmed", "immutable_document"];
+    if (allowed.includes(raw)) return raw;
+    const where = ctx !== void 0 ? ` for entity "${ctx.entityId}" fact "${ctx.factId}"` : "";
+    throw new Error(
+      `importDump: invalid source_type "${raw}"${where} (expected one of: ${allowed.join(", ")}, or legacy aliases user_document / agent_inferred)`
+    );
+  }
+  async assertNoLegacySourceTypes() {
+    const legacyProbe = await this.db.getFirstAsync(
+      `SELECT 1 AS one FROM ${this.prefix}entries
+       WHERE source_type IN ('user_document', 'agent_inferred')
+       LIMIT 1`,
+      []
+    );
+    if (!legacyProbe) return;
+    const legacyCount = await this.db.getFirstAsync(
+      `SELECT COUNT(*) as count FROM ${this.prefix}entries
+       WHERE source_type IN ('user_document', 'agent_inferred')`,
+      []
+    );
+    const count = legacyCount?.count ?? 0;
+    const migrationSQL = `
+-- Migrate legacy source_type values (targets your WikiMemory prefix: ${this.prefix})
+UPDATE ${this.prefix}entries SET source_type = 'immutable_document' WHERE source_type = 'user_document';
+UPDATE ${this.prefix}entries SET source_type = 'librarian_inferred' WHERE source_type = 'agent_inferred';
+    `.trim();
+    throw new Error(
+      `Database contains ${count} entries with legacy source_type values ('user_document' or 'agent_inferred'). These enum values were renamed in this release. Running without migration would allow legacy 'user_document' facts to bypass immutability guards, causing data corruption.
+
+${migrationSQL}
+
+After running the migration SQL, restart your application.`
+    );
+  }
   async _notifyEmbeddingPersisted(entityId, factId, vector) {
     if (!this.options.vectorRanker?.onEmbeddingPersisted) return;
     const vectorCopy = vector ? vector.slice() : null;
@@ -688,6 +727,9 @@ var _WikiMemory = class _WikiMemory {
         );
       }
     }
+    if (entriesExistedBeforeSetup) {
+      await this.assertNoLegacySourceTypes();
+    }
     const rows = await this.db.getAllAsync(`
       SELECT rowid, source_ref FROM ${this.prefix}entries
       WHERE source_ref IS NOT NULL
@@ -775,6 +817,24 @@ var _WikiMemory = class _WikiMemory {
     }
     return false;
   }
+  _copyEntityStatus(s) {
+    return { ingesting: s.ingesting, librarian: s.librarian, heal: s.heal };
+  }
+  _notifyStatusSubscribers(entityId) {
+    const set = this.statusSubscribers.get(entityId);
+    if (!set || set.size === 0) return;
+    for (const entry of Array.from(set)) {
+      if (!set.has(entry)) continue;
+      const next = this.getEntityStatus(entityId);
+      if (entry.last.ingesting === next.ingesting && entry.last.librarian === next.librarian && entry.last.heal === next.heal) continue;
+      entry.last = this._copyEntityStatus(next);
+      try {
+        entry.callback(this._copyEntityStatus(next));
+      } catch (err) {
+        console.error(`[WikiMemory.subscribeEntityStatus] callback error for entityId="${entityId}" during transition emission`, err);
+      }
+    }
+  }
   _validatePruneDuration(value, name) {
     if (value !== null && value !== void 0 && (typeof value !== "number" || !isFinite(value) || value < 0)) {
       throw new Error(`Invalid ${name}: must be a non-negative finite number or null`);
@@ -824,7 +884,7 @@ var _WikiMemory = class _WikiMemory {
         const cutoff = now - retainSoftDeletedFor * 864e5;
         const entriesToDelete = await this.db.getAllAsync(
           `SELECT id, entity_id FROM ${this.prefix}entries
-           WHERE entity_id = ? AND deleted_at IS NOT NULL AND deleted_at < ?`,
+           WHERE entity_id = ? AND deleted_at IS NOT NULL AND deleted_at <= ?`,
           [entityId, cutoff]
         );
         const succeeded = [];
@@ -844,7 +904,7 @@ var _WikiMemory = class _WikiMemory {
             const chunk = succeeded.slice(i, i + chunkSize);
             const placeholders = chunk.map(() => "?").join(",");
             const entryResult = await this.db.runAsync(
-              `DELETE FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NOT NULL AND deleted_at < ? AND id IN (${placeholders})`,
+              `DELETE FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NOT NULL AND deleted_at <= ? AND id IN (${placeholders})`,
               [entityId, cutoff, ...chunk.map((r) => r.id)]
             );
             deletedEntries += entryResult.changes;
@@ -852,7 +912,7 @@ var _WikiMemory = class _WikiMemory {
         }
         const taskResult = await this.db.runAsync(
           `DELETE FROM ${this.prefix}tasks
-           WHERE entity_id = ? AND deleted_at IS NOT NULL AND deleted_at < ?`,
+           WHERE entity_id = ? AND deleted_at IS NOT NULL AND deleted_at <= ?`,
           [entityId, cutoff]
         );
         deletedTasks = taskResult.changes;
@@ -890,7 +950,7 @@ var _WikiMemory = class _WikiMemory {
         const cutoff = now - retainEventsFor * 864e5;
         const eventResult = await this.db.runAsync(
           `DELETE FROM ${this.prefix}events
-           WHERE entity_id = ? AND created_at < ?`,
+           WHERE entity_id = ? AND created_at <= ?`,
           [entityId, cutoff]
         );
         deletedEvents = eventResult.changes;
@@ -1517,7 +1577,11 @@ var _WikiMemory = class _WikiMemory {
       const jobKey = this._librarianKey(entityId);
       if (!this.activeMaintenanceJobs.has(jobKey) && !this.activeMaintenanceJobs.has(this._pruneKey(entityId)) && !this._isReembedActive(entityId) && !this._isImportActiveFor(entityId) && !this._isForgetActiveFor(entityId)) {
         this.activeMaintenanceJobs.add(jobKey);
-        this.runLibrarianThenMaybeHeal(entityId, count).catch(console.error).finally(() => this.activeMaintenanceJobs.delete(jobKey));
+        this._notifyStatusSubscribers(entityId);
+        this.runLibrarianThenMaybeHeal(entityId, count).catch(console.error).finally(() => {
+          this.activeMaintenanceJobs.delete(jobKey);
+          this._notifyStatusSubscribers(entityId);
+        });
       }
     }
   }
@@ -1536,6 +1600,7 @@ var _WikiMemory = class _WikiMemory {
       const healKey = this._healKey(entityId);
       if (!this.activeMaintenanceJobs.has(healKey)) {
         this.activeMaintenanceJobs.add(healKey);
+        this._notifyStatusSubscribers(entityId);
         try {
           await this._doRunHeal(entityId);
           await this.db.runAsync(`
@@ -1545,6 +1610,7 @@ var _WikiMemory = class _WikiMemory {
           `, [entityId, currentEventCount, currentEventCount]);
         } finally {
           this.activeMaintenanceJobs.delete(healKey);
+          this._notifyStatusSubscribers(entityId);
         }
       }
     }
@@ -1591,7 +1657,7 @@ ${JSON.stringify(currentFacts, null, 2)}`;
         let skip = false;
         if (newTokens.size >= MIN_TOKENS_TO_QUALIFY) {
           for (const existing of currentFactsRows) {
-            if (existing.source_type !== "agent_inferred") continue;
+            if (existing.source_type !== "librarian_inferred") continue;
             const existingTokens = titleTokens(existing.title);
             if (existingTokens.size >= MIN_TOKENS_TO_QUALIFY) {
               if (jaccardScore(newTokens, existingTokens) >= FUZZY_THRESHOLD) {
@@ -1606,7 +1672,7 @@ ${JSON.stringify(currentFacts, null, 2)}`;
         await this.db.runAsync(`
           INSERT INTO ${this.prefix}entries (id, entity_id, title, body, tags, confidence, source_type, created_at, updated_at)
           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
-        `, [id, entityId, fact.title, fact.body, JSON.stringify(fact.tags), fact.confidence, "agent_inferred", now, now]);
+        `, [id, entityId, fact.title, fact.body, JSON.stringify(fact.tags), fact.confidence, "librarian_inferred", now, now]);
         insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
       }
       for (const task of validTasks) {
@@ -1639,25 +1705,25 @@ ${JSON.stringify(currentFacts, null, 2)}`;
       if (orphanAfterDays !== null) {
         const orphanThreshold = now - orphanAfterDays * MS_PER_DAY;
         await this.db.runAsync(`
-          UPDATE ${this.prefix}entries 
-          SET deleted_at = ?, updated_at = ? 
-          WHERE entity_id = ? AND access_count = 0 AND created_at < ? AND source_type != 'user_document' AND deleted_at IS NULL
+          UPDATE ${this.prefix}entries
+          SET deleted_at = ?, updated_at = ?
+          WHERE entity_id = ? AND access_count = 0 AND created_at <= ? AND source_type != 'immutable_document' AND deleted_at IS NULL
         `, [now, now, entityId, orphanThreshold]);
       }
       if (staleInferredAfterDays !== null) {
         const staleThreshold = now - staleInferredAfterDays * MS_PER_DAY;
         await this.db.runAsync(`
-          UPDATE ${this.prefix}entries 
-          SET confidence = 'tentative', updated_at = ? 
-          WHERE entity_id = ? AND confidence = 'inferred' AND (last_accessed_at < ? OR (last_accessed_at IS NULL AND created_at < ?)) AND source_type != 'user_document' AND deleted_at IS NULL
+          UPDATE ${this.prefix}entries
+          SET confidence = 'tentative', updated_at = ?
+          WHERE entity_id = ? AND confidence = 'inferred' AND (last_accessed_at <= ? OR (last_accessed_at IS NULL AND created_at <= ?)) AND source_type != 'immutable_document' AND deleted_at IS NULL
         `, [now, entityId, staleThreshold, staleThreshold]);
       }
     });
     const allFactsRows = await this.db.getAllAsync(`SELECT * FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL`, [entityId]);
     const allTasks = await this.db.getAllAsync(`SELECT * FROM ${this.prefix}tasks WHERE entity_id = ? AND status IN ('pending', 'in_progress') AND deleted_at IS NULL`, [entityId]);
     const recentEvents = await this.db.getAllAsync(`SELECT * FROM ${this.prefix}events WHERE entity_id = ? ORDER BY created_at DESC LIMIT 20`, [entityId]);
-    const healCandidates = allFactsRows.filter((f) => f.source_type !== "user_document");
-    const documentAnchors = allFactsRows.filter((f) => f.source_type === "user_document").map(({ id, title, source_ref }) => ({ id, title, source_ref }));
+    const healCandidates = allFactsRows.filter((f) => f.source_type !== "immutable_document");
+    const documentAnchors = allFactsRows.filter((f) => f.source_type === "immutable_document").map(({ id, title, source_ref }) => ({ id, title, source_ref }));
     const userPrompt = `Heal Candidates:
 ${JSON.stringify(healCandidates.map((f) => {
       const { embedding: _embedding, embedding_blob: _blob, ...rest } = f;
@@ -1700,7 +1766,7 @@ The following document anchors are provided for contradiction detection only. Do
         await this.db.runAsync(`
           INSERT INTO ${this.prefix}entries (id, entity_id, title, body, tags, confidence, source_type, created_at, updated_at)
           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
-        `, [id, entityId, fact.title, fact.body, JSON.stringify(fact.tags), fact.confidence, "agent_inferred", now, now]);
+        `, [id, entityId, fact.title, fact.body, JSON.stringify(fact.tags), fact.confidence, "librarian_inferred", now, now]);
         insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
       }
     });
@@ -1736,10 +1802,12 @@ The following document anchors are provided for contradiction detection only. Do
       throw new WikiBusyError("forget", entityId);
     }
     this.activeMaintenanceJobs.add(jobKey);
+    this._notifyStatusSubscribers(entityId);
     try {
       await this._doRunLibrarian(entityId);
     } finally {
       this.activeMaintenanceJobs.delete(jobKey);
+      this._notifyStatusSubscribers(entityId);
     }
   }
   async runHeal(entityId) {
@@ -1760,10 +1828,12 @@ The following document anchors are provided for contradiction detection only. Do
       throw new WikiBusyError("forget", entityId);
     }
     this.activeMaintenanceJobs.add(jobKey);
+    this._notifyStatusSubscribers(entityId);
     try {
       await this._doRunHeal(entityId);
     } finally {
       this.activeMaintenanceJobs.delete(jobKey);
+      this._notifyStatusSubscribers(entityId);
     }
   }
   async runReembed(entityId, opts) {
@@ -1903,6 +1973,40 @@ The following document anchors are provided for contradiction detection only. Do
       heal: this.activeMaintenanceJobs.has(this._healKey(entityId))
     };
   }
+  /**
+   * Subscribe to {@link EntityStatus} changes for a single entity. The callback
+   * is invoked synchronously once with the current status before this method
+   * returns, then again on every transition where any of `ingesting`,
+   * `librarian`, or `heal` flips. No polling, no duplicate snapshots.
+   *
+   * Returns an idempotent unsubscribe function.
+   *
+   * See also {@link getEntityStatus} for a synchronous point-in-time read.
+   */
+  subscribeEntityStatus(entityId, callback) {
+    const initial = this.getEntityStatus(entityId);
+    let set = this.statusSubscribers.get(entityId);
+    if (!set) {
+      set = /* @__PURE__ */ new Set();
+      this.statusSubscribers.set(entityId, set);
+    }
+    const entry = { callback, last: this._copyEntityStatus(initial) };
+    set.add(entry);
+    try {
+      callback(this._copyEntityStatus(initial));
+    } catch (err) {
+      console.error(`[WikiMemory.subscribeEntityStatus] callback error for entityId="${entityId}" during initial emission`, err);
+    }
+    let active = true;
+    return () => {
+      if (!active) return;
+      active = false;
+      const s = this.statusSubscribers.get(entityId);
+      if (!s) return;
+      s.delete(entry);
+      if (s.size === 0) this.statusSubscribers.delete(entityId);
+    };
+  }
   clearVectorCache() {
     this.vectorCache.clear();
   }
@@ -2000,6 +2104,7 @@ The following document anchors are provided for contradiction detection only. Do
       this.activeMaintenanceJobs.add(this._importKey(entityId));
     }
     try {
+      await this.assertNoLegacySourceTypes();
       for (const [entityId, bundle] of Object.entries(dump.entities)) {
         await this._doImportEntity(entityId, bundle, merge);
       }
@@ -2053,6 +2158,10 @@ The following document anchors are provided for contradiction detection only. Do
         }
       }
       for (const fact of bundle.facts) {
+        const sourceType = this._normalizeImportedSourceType(String(fact.source_type), {
+          entityId,
+          factId: fact.id
+        });
         const tagsJson = JSON.stringify(Array.isArray(fact.tags) ? fact.tags : []);
         const safeUpdatedAt = Number.isFinite(fact.updated_at) ? fact.updated_at : 0;
         const existing = existingFactsById.get(fact.id);
@@ -2101,14 +2210,14 @@ The following document anchors are provided for contradiction detection only. Do
           if (blobData != null) {
             await this.db.runAsync(
               `UPDATE ${this.prefix}entries SET entity_id = ?, title = ?, body = ?, tags = ?, confidence = ?, source_type = ?, source_hash = ?, source_ref = ?, created_at = ?, updated_at = ?, last_accessed_at = ?, access_count = ?, deleted_at = ?, embedding_blob = ?, embedding = NULL WHERE id = ?`,
-              [entityId, fact.title, fact.body, tagsJson, fact.confidence, fact.source_type, fact.source_hash, fact.source_ref, fact.created_at, safeUpdatedAt, fact.last_accessed_at, fact.access_count, fact.deleted_at, blobData, fact.id]
+              [entityId, fact.title, fact.body, tagsJson, fact.confidence, sourceType, fact.source_hash, fact.source_ref, fact.created_at, safeUpdatedAt, fact.last_accessed_at, fact.access_count, fact.deleted_at, blobData, fact.id]
             );
             factsWithPreservedBlob.set(fact.id, blobData);
             if (!fact.deleted_at) preservedBlobDims.add(blobData.byteLength / 4);
           } else {
             await this.db.runAsync(
               `UPDATE ${this.prefix}entries SET entity_id = ?, title = ?, body = ?, tags = ?, confidence = ?, source_type = ?, source_hash = ?, source_ref = ?, created_at = ?, updated_at = ?, last_accessed_at = ?, access_count = ?, deleted_at = ?, embedding_blob = NULL, embedding = NULL WHERE id = ?`,
-              [entityId, fact.title, fact.body, tagsJson, fact.confidence, fact.source_type, fact.source_hash, fact.source_ref, fact.created_at, safeUpdatedAt, fact.last_accessed_at, fact.access_count, fact.deleted_at, fact.id]
+              [entityId, fact.title, fact.body, tagsJson, fact.confidence, sourceType, fact.source_hash, fact.source_ref, fact.created_at, safeUpdatedAt, fact.last_accessed_at, fact.access_count, fact.deleted_at, fact.id]
             );
           }
           existingFactsById.set(fact.id, { id: fact.id, entity_id: entityId, updated_at: safeUpdatedAt });
@@ -2118,14 +2227,14 @@ The following document anchors are provided for contradiction detection only. Do
           if (blobData != null) {
             await this.db.runAsync(
               `INSERT INTO ${this.prefix}entries (id, entity_id, title, body, tags, confidence, source_type, source_hash, source_ref, created_at, updated_at, last_accessed_at, access_count, deleted_at, embedding_blob) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-              [fact.id, entityId, fact.title, fact.body, tagsJson, fact.confidence, fact.source_type, fact.source_hash, fact.source_ref, fact.created_at, safeUpdatedAt, fact.last_accessed_at, fact.access_count, fact.deleted_at, blobData]
+              [fact.id, entityId, fact.title, fact.body, tagsJson, fact.confidence, sourceType, fact.source_hash, fact.source_ref, fact.created_at, safeUpdatedAt, fact.last_accessed_at, fact.access_count, fact.deleted_at, blobData]
             );
             factsWithPreservedBlob.set(fact.id, blobData);
             if (!fact.deleted_at) preservedBlobDims.add(blobData.byteLength / 4);
           } else {
             await this.db.runAsync(
               `INSERT INTO ${this.prefix}entries (id, entity_id, title, body, tags, confidence, source_type, source_hash, source_ref, created_at, updated_at, last_accessed_at, access_count, deleted_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-              [fact.id, entityId, fact.title, fact.body, tagsJson, fact.confidence, fact.source_type, fact.source_hash, fact.source_ref, fact.created_at, safeUpdatedAt, fact.last_accessed_at, fact.access_count, fact.deleted_at]
+              [fact.id, entityId, fact.title, fact.body, tagsJson, fact.confidence, sourceType, fact.source_hash, fact.source_ref, fact.created_at, safeUpdatedAt, fact.last_accessed_at, fact.access_count, fact.deleted_at]
             );
           }
           existingFactsById.set(fact.id, { id: fact.id, entity_id: entityId, updated_at: safeUpdatedAt });
@@ -2423,6 +2532,7 @@ The following document anchors are provided for contradiction detection only. Do
       throw new WikiBusyError("forget", entityId);
     }
     this.activeIngestJobs.add(jobKey);
+    this._notifyStatusSubscribers(entityId);
     try {
       const { chunks, truncated } = chunkText(params.documentChunk, maxChunkLength, chunkOverlap);
       if (chunks.length === 0) {
@@ -2472,7 +2582,7 @@ ${chunk}`;
           await this.db.runAsync(
             `INSERT INTO ${this.prefix}entries (id, entity_id, title, body, tags, confidence, source_type, source_hash, source_ref, created_at, updated_at)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-            [id, entityId, fact.title, fact.body, JSON.stringify(fact.tags), fact.confidence, "user_document", sourceHash, sourceRef, now, now]
+            [id, entityId, fact.title, fact.body, JSON.stringify(fact.tags), fact.confidence, "immutable_document", sourceHash, sourceRef, now, now]
           );
           insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
         }
@@ -2494,6 +2604,7 @@ ${chunk}`;
       return { truncated, chunks: chunks.length };
     } finally {
       this.activeIngestJobs.delete(jobKey);
+      this._notifyStatusSubscribers(entityId);
     }
   }
 };
@@ -2731,6 +2842,6 @@ function createWiki(db, options) {
   return new WikiMemory(db, options);
 }
 
-export { PrunePartialFailureError, WikiBusyError, WikiMemory, createWiki, formatContext, formatMemoryDump };
+export { PrunePartialFailureError, WikiBusyError, WikiMemory, createWiki, formatContext, formatMemoryDump, parseEmbedding };
 //# sourceMappingURL=index.mjs.map
 //# sourceMappingURL=index.mjs.map