@equationalapplications/core-llm-wiki 3.1.0 → 3.2.0

package/dist/index.mjs

@@ -126,6 +126,18 @@ var WikiBusyError = class extends Error {
     this.entityId = entityId;
   }
 };
+var PrunePartialFailureError = class extends Error {
+  constructor(deleted, failedAt, remaining, cause, deletedTasks = 0, deletedEvents = 0) {
+    super(`Prune partially failed: deleted ${deleted}, failed at ${failedAt}, ${remaining} remaining`);
+    this.name = "PrunePartialFailureError";
+    this.deleted = deleted;
+    this.failedAt = failedAt;
+    this.remaining = remaining;
+    this.deletedTasks = deletedTasks;
+    this.deletedEvents = deletedEvents;
+    this.cause = cause;
+  }
+};
 
 // src/prompts.ts
 var LIBRARIAN_SYSTEM_PROMPT = `You are a knowledge extraction agent. Your job is to analyze recent episodic events and extract stable facts and actionable tasks about the user or entity.
@@ -192,6 +204,7 @@ function parseEmbedding(blob, text) {
 }
 
 // src/WikiMemory.ts
+var HOOK_TIMEOUT_MARKER = /* @__PURE__ */ Symbol("WikiMemoryHookTimeout");
 function parseJsonResponse(text) {
   const firstBrace = text.indexOf("{");
   const firstBracket = text.indexOf("[");
@@ -575,7 +588,55 @@ var _WikiMemory = class _WikiMemory {
     console.warn(`[WikiMemory] importDump: ${type} id "${id}" already belongs to entity "${existingEntityId}"; skipping for entity "${targetEntityId}"`);
   }
   async _notifyEmbeddingPersisted(entityId, factId, vector) {
-    await this.options.vectorRanker?.onEmbeddingPersisted?.({ entityId, factId, vector });
+    if (!this.options.vectorRanker?.onEmbeddingPersisted) return;
+    const vectorCopy = vector ? vector.slice() : null;
+    await this.options.vectorRanker.onEmbeddingPersisted({
+      entityId,
+      factId,
+      vector: vectorCopy
+    });
+  }
+  /**
+   * GDPR-critical variant: awaits the hook with a timeout and rethrows failures.
+   * Use ONLY on deletion paths. forget() calls after soft-delete UPDATE; runPrune()
+   * calls before hard DELETE. For best-effort sync, use _notifyEmbeddingPersisted.
+   */
+  async _notifyEmbeddingPersistedOrThrow(entityId, factId, vector) {
+    if (!this.options.vectorRanker?.onEmbeddingPersisted) return;
+    if (this.options.forceDeleteIgnoreRankerHook === true) return;
+    const vectorCopy = vector ? vector.slice() : null;
+    const rawTimeout = this.options.deletionHookTimeoutMs ?? 3e4;
+    if (typeof rawTimeout !== "number" || !Number.isFinite(rawTimeout) || rawTimeout <= 0) {
+      throw new Error("Invalid deletionHookTimeoutMs: must be a positive finite number");
+    }
+    const timeoutMs = rawTimeout;
+    let timeoutHandle;
+    const timeoutPromise = new Promise((_, reject) => {
+      timeoutHandle = setTimeout(
+        () => {
+          const timeoutError = new Error(`onEmbeddingPersisted timed out after ${timeoutMs}ms`);
+          timeoutError[HOOK_TIMEOUT_MARKER] = true;
+          reject(timeoutError);
+        },
+        timeoutMs
+      );
+    });
+    const hookPromise = Promise.resolve(
+      this.options.vectorRanker.onEmbeddingPersisted({
+        entityId,
+        factId,
+        vector: vectorCopy
+      })
+    );
+    try {
+      await Promise.race([hookPromise, timeoutPromise]);
+    } catch (err) {
+      hookPromise.catch(() => {
+      });
+      throw err;
+    } finally {
+      if (timeoutHandle) clearTimeout(timeoutHandle);
+    }
   }
   async setup() {
     const entriesExistedBeforeSetup = await this.db.getFirstAsync(
@@ -759,27 +820,71 @@ var _WikiMemory = class _WikiMemory {
       let deletedEntries = 0;
       let deletedTasks = 0;
       let deletedEvents = 0;
-      const deletedEntryIds = [];
       if (retainSoftDeletedFor !== null) {
         const cutoff = now - retainSoftDeletedFor * 864e5;
         const entriesToDelete = await this.db.getAllAsync(
-          `SELECT id FROM ${this.prefix}entries
+          `SELECT id, entity_id FROM ${this.prefix}entries
            WHERE entity_id = ? AND deleted_at IS NOT NULL AND deleted_at < ?`,
           [entityId, cutoff]
         );
-        deletedEntryIds.push(...entriesToDelete.map((e) => e.id));
-        const entryResult = await this.db.runAsync(
-          `DELETE FROM ${this.prefix}entries
-           WHERE entity_id = ? AND deleted_at IS NOT NULL AND deleted_at < ?`,
-          [entityId, cutoff]
-        );
-        deletedEntries = entryResult.changes;
+        const succeeded = [];
+        let failure = null;
+        for (const row of entriesToDelete) {
+          try {
+            await this._notifyEmbeddingPersistedOrThrow(row.entity_id, row.id, null);
+            succeeded.push({ entity_id: row.entity_id, id: row.id });
+          } catch (err) {
+            failure = { factId: row.id, cause: err };
+            break;
+          }
+        }
+        if (succeeded.length > 0) {
+          const chunkSize = 500;
+          for (let i = 0; i < succeeded.length; i += chunkSize) {
+            const chunk = succeeded.slice(i, i + chunkSize);
+            const placeholders = chunk.map(() => "?").join(",");
+            const entryResult = await this.db.runAsync(
+              `DELETE FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NOT NULL AND deleted_at < ? AND id IN (${placeholders})`,
+              [entityId, cutoff, ...chunk.map((r) => r.id)]
+            );
+            deletedEntries += entryResult.changes;
+          }
+        }
         const taskResult = await this.db.runAsync(
           `DELETE FROM ${this.prefix}tasks
            WHERE entity_id = ? AND deleted_at IS NOT NULL AND deleted_at < ?`,
           [entityId, cutoff]
         );
         deletedTasks = taskResult.changes;
+        if (failure) {
+          await this.rebuildMiniSearchIndex(entityId);
+          this.vectorCache.delete(entityId);
+          const remaining = entriesToDelete.length - succeeded.length - 1;
+          const isTimeout = failure.cause?.[HOOK_TIMEOUT_MARKER] === true;
+          if (isTimeout) {
+            throw new PrunePartialFailureError(
+              succeeded.length,
+              failure.factId,
+              remaining,
+              new Error("Deletion hook timed out"),
+              deletedTasks,
+              0
+              // events not yet deleted at this point
+            );
+          }
+          const errMsg = failure.cause?.message ?? "";
+          const isValidationError = errMsg.startsWith("Invalid deletionHookTimeoutMs");
+          const sanitizedCause = isValidationError ? failure.cause : this._sanitizeRankerError(failure.cause);
+          throw new PrunePartialFailureError(
+            succeeded.length,
+            failure.factId,
+            remaining,
+            sanitizedCause,
+            deletedTasks,
+            0
+            // events not yet deleted at this point
+          );
+        }
       }
       if (retainEventsFor !== null) {
         const cutoff = now - retainEventsFor * 864e5;
@@ -796,14 +901,6 @@ var _WikiMemory = class _WikiMemory {
       }
       await this.rebuildMiniSearchIndex(entityId);
       this.vectorCache.delete(entityId);
-      const uniqueDeletedIds = Array.from(new Set(deletedEntryIds));
-      for (const factId of uniqueDeletedIds) {
-        try {
-          await this._notifyEmbeddingPersisted(entityId, factId, null);
-        } catch (hookErr) {
-          console.warn(`[WikiMemory] onEmbeddingPersisted hook failed during prune for ${factId}:`, hookErr);
-        }
-      }
       return { entries: deletedEntries, tasks: deletedTasks, events: deletedEvents };
     } finally {
       this.activeMaintenanceJobs.delete(pruneKey);
@@ -1057,7 +1154,10 @@ var _WikiMemory = class _WikiMemory {
               } catch (rankerErr) {
                 const rankerError = rankerErr instanceof Error ? rankerErr : new Error(String(rankerErr));
                 const policy = this.options.vectorRankerFallback ?? "js-cosine";
-                this.options.onVectorRankerFallback?.({ error: rankerError, policy });
+                this.options.onVectorRankerFallback?.({
+                  error: this._sanitizeRankerError(rankerError),
+                  policy
+                });
                 if (policy === "throw") {
                   rankerShouldRethrow = true;
                   throw rankerError;
@@ -1121,8 +1221,9 @@ var _WikiMemory = class _WikiMemory {
                   scored = [];
                 }
                 if (this.options.propagateRankerFailureToRetrievalFallback) {
-                  const mirrored = new Error("Vector ranker failed, falling back");
-                  mirrored.cause = rankerError;
+                  const mirrored = new Error("Vector ranker failed, falling back", {
+                    cause: this._sanitizeRankerError(rankerErr)
+                  });
                   pendingRankerFallbackError = mirrored;
                 }
               }
@@ -1282,12 +1383,31 @@ var _WikiMemory = class _WikiMemory {
     if (updatedAtDiff !== 0) return updatedAtDiff;
     return a.id.localeCompare(b.id);
   }
+  /**
+   * Strip potentially sensitive data from ranker errors before exposing to host callbacks.
+   * Preserves error type for debugging but removes message/stack that may contain credentials.
+   * Recursively sanitizes one level of .cause; deeper chains collapse to type only.
+   */
+  _sanitizeRankerError(err) {
+    if (this.options.sanitizeRankerErrors === false) {
+      return err instanceof Error ? err : new Error(String(err));
+    }
+    const typeName = err instanceof Error ? err.constructor?.name ?? "Error" : typeof err;
+    const innerCause = err instanceof Error && err.cause !== void 0 ? new Error(`Caused by: ${err.cause?.constructor?.name ?? typeof err.cause}`) : void 0;
+    const sanitized = new Error(
+      `VectorRanker ${typeName} (message scrubbed for security)`,
+      innerCause ? { cause: innerCause } : void 0
+    );
+    sanitized.name = typeName;
+    return sanitized;
+  }
   /**
    * Score candidate rows using in-process JS cosine similarity.
    * Applies hybrid blending (if weight set) and tie-break sorting before returning.
    */
   async _rankWithJsCosine(args) {
-    const { entityId, queryVec, candidateRows, weight, miniSearchScores, populateCache, limit } = args;
+    const queryVec = args.queryVec instanceof Float32Array ? args.queryVec.slice() : Array.from(args.queryVec);
+    const { entityId, candidateRows, weight, miniSearchScores, populateCache, limit } = args;
     let entityCache = this.vectorCache.get(entityId);
     const tooLarge = populateCache && candidateRows.length > _WikiMemory.MAX_VECTOR_CACHE_FACTS_PER_ENTITY;
     if (tooLarge && entityCache) {
@@ -1338,14 +1458,15 @@ var _WikiMemory = class _WikiMemory {
    * Returns scored results ready for hybrid blending and tie-break sorting.
    */
   async _rankWithVectorRanker(args) {
-    const { entityId, queryVec, candidateIds, weight, miniSearchScores, limit } = args;
+    const { entityId, candidateIds, weight, miniSearchScores, limit } = args;
     const ranker = this.options.vectorRanker;
     if (!ranker) {
       throw new Error("vectorRanker not configured");
     }
+    const queryVecCopy = args.queryVec instanceof Float32Array ? args.queryVec.slice() : Array.from(args.queryVec);
     const rankerResults = await ranker.rankBySimilarity({
       entityId,
-      queryVec,
+      queryVec: queryVecCopy,
       candidateIds,
       limit
     });
@@ -2165,11 +2286,15 @@ The following document anchors are provided for contradiction detection only. Do
       let deletedTasks = 0;
       const deletedEntryIds = [];
       if (params.clearAll) {
-        const entriesToDelete = await this.db.getAllAsync(
+        const newDeletions = await this.db.getAllAsync(
           `SELECT id FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL`,
           [entityId]
         );
-        deletedEntryIds.push(...entriesToDelete.map((e) => e.id));
+        const alreadySoftDeleted = await this.db.getAllAsync(
+          `SELECT id FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NOT NULL`,
+          [entityId]
+        );
+        deletedEntryIds.push(...newDeletions.map((e) => e.id), ...alreadySoftDeleted.map((e) => e.id));
         const [entriesRes, tasksRes] = await Promise.all([
           this.db.runAsync(`UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE entity_id = ? AND deleted_at IS NULL`, [now, now, entityId]),
           this.db.runAsync(`UPDATE ${this.prefix}tasks SET deleted_at = ?, updated_at = ? WHERE entity_id = ? AND deleted_at IS NULL`, [now, now, entityId])
@@ -2189,13 +2314,13 @@ The following document anchors are provided for contradiction detection only. Do
         if (params.sourceHash !== void 0 && !sourceHash) throw new Error("Invalid sourceHash (must be 64-char hex string)");
         if (params.entryId) {
           const entry = await this.db.getFirstAsync(
-            `SELECT id FROM ${this.prefix}entries WHERE id = ? AND entity_id = ? AND deleted_at IS NULL`,
+            `SELECT id FROM ${this.prefix}entries WHERE id = ? AND entity_id = ?`,
             [params.entryId, entityId]
           );
           if (entry) deletedEntryIds.push(entry.id);
         }
         if (sourceRef || sourceHash) {
-          let q = `SELECT id FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL`;
+          let q = `SELECT id FROM ${this.prefix}entries WHERE entity_id = ?`;
           const args = [entityId];
           if (sourceRef) {
             q += ` AND source_ref = ?`;
@@ -2238,9 +2363,26 @@ The following document anchors are provided for contradiction detection only. Do
       const uniqueDeletedIds = Array.from(new Set(deletedEntryIds));
       for (const factId of uniqueDeletedIds) {
         try {
-          await this._notifyEmbeddingPersisted(entityId, factId, null);
+          await this._notifyEmbeddingPersistedOrThrow(entityId, factId, null);
         } catch (hookErr) {
-          console.warn(`[WikiMemory] onEmbeddingPersisted hook failed during forget for ${factId}:`, hookErr);
+          const isTimeout = hookErr?.[HOOK_TIMEOUT_MARKER] === true;
+          if (isTimeout) {
+            throw new Error(
+              `forget(${entityId}/${factId}) failed: ${hookErr.message}`
+            );
+          }
+          const errMsg = hookErr?.message ?? "";
+          const isValidationError = errMsg.startsWith("Invalid deletionHookTimeoutMs");
+          if (isValidationError) {
+            throw new Error(
+              `forget(${entityId}/${factId}) failed: ${errMsg}`,
+              { cause: hookErr }
+            );
+          }
+          throw new Error(
+            `forget(${entityId}/${factId}) failed: ANN cleanup hook rejected`,
+            { cause: this._sanitizeRankerError(hookErr) }
+          );
         }
       }
       return { deleted: { entries: deletedEntries, tasks: deletedTasks } };
@@ -2589,6 +2731,6 @@ function createWiki(db, options) {
   return new WikiMemory(db, options);
 }
 
-export { WikiBusyError, WikiMemory, createWiki, formatContext, formatMemoryDump };
+export { PrunePartialFailureError, WikiBusyError, WikiMemory, createWiki, formatContext, formatMemoryDump };
 //# sourceMappingURL=index.mjs.map
 //# sourceMappingURL=index.mjs.map