@equationalapplications/core-llm-wiki 3.0.0 → 3.2.0

package/dist/index.mjs

@@ -126,6 +126,18 @@ var WikiBusyError = class extends Error {
     this.entityId = entityId;
   }
 };
+var PrunePartialFailureError = class extends Error {
+  constructor(deleted, failedAt, remaining, cause, deletedTasks = 0, deletedEvents = 0) {
+    super(`Prune partially failed: deleted ${deleted}, failed at ${failedAt}, ${remaining} remaining`);
+    this.name = "PrunePartialFailureError";
+    this.deleted = deleted;
+    this.failedAt = failedAt;
+    this.remaining = remaining;
+    this.deletedTasks = deletedTasks;
+    this.deletedEvents = deletedEvents;
+    this.cause = cause;
+  }
+};
 
 // src/prompts.ts
 var LIBRARIAN_SYSTEM_PROMPT = `You are a knowledge extraction agent. Your job is to analyze recent episodic events and extract stable facts and actionable tasks about the user or entity.
@@ -192,6 +204,7 @@ function parseEmbedding(blob, text) {
 }
 
 // src/WikiMemory.ts
+var HOOK_TIMEOUT_MARKER = /* @__PURE__ */ Symbol("WikiMemoryHookTimeout");
 function parseJsonResponse(text) {
   const firstBrace = text.indexOf("{");
   const firstBracket = text.indexOf("[");
@@ -554,6 +567,11 @@ var _WikiMemory = class _WikiMemory {
         `UPDATE ${this.prefix}entries SET embedding_blob = ?, embedding = NULL WHERE id = ?`,
         [blob, fact.id]
       );
+      try {
+        await this._notifyEmbeddingPersisted(fact.entity_id, fact.id, float32Vector);
+      } catch (hookErr) {
+        console.warn(`[WikiMemory] onEmbeddingPersisted hook failed for ${fact.id}:`, hookErr);
+      }
       return true;
     } catch (err) {
       console.warn(`[WikiMemory] embedFact failed for ${fact.id}:`, err);
@@ -569,6 +587,57 @@ var _WikiMemory = class _WikiMemory {
   _warnCrossEntityCollision(type, id, existingEntityId, targetEntityId) {
     console.warn(`[WikiMemory] importDump: ${type} id "${id}" already belongs to entity "${existingEntityId}"; skipping for entity "${targetEntityId}"`);
   }
+  async _notifyEmbeddingPersisted(entityId, factId, vector) {
+    if (!this.options.vectorRanker?.onEmbeddingPersisted) return;
+    const vectorCopy = vector ? vector.slice() : null;
+    await this.options.vectorRanker.onEmbeddingPersisted({
+      entityId,
+      factId,
+      vector: vectorCopy
+    });
+  }
+  /**
+   * GDPR-critical variant: awaits the hook with a timeout and rethrows failures.
+   * Use ONLY on deletion paths. forget() calls after soft-delete UPDATE; runPrune()
+   * calls before hard DELETE. For best-effort sync, use _notifyEmbeddingPersisted.
+   */
+  async _notifyEmbeddingPersistedOrThrow(entityId, factId, vector) {
+    if (!this.options.vectorRanker?.onEmbeddingPersisted) return;
+    if (this.options.forceDeleteIgnoreRankerHook === true) return;
+    const vectorCopy = vector ? vector.slice() : null;
+    const rawTimeout = this.options.deletionHookTimeoutMs ?? 3e4;
+    if (typeof rawTimeout !== "number" || !Number.isFinite(rawTimeout) || rawTimeout <= 0) {
+      throw new Error("Invalid deletionHookTimeoutMs: must be a positive finite number");
+    }
+    const timeoutMs = rawTimeout;
+    let timeoutHandle;
+    const timeoutPromise = new Promise((_, reject) => {
+      timeoutHandle = setTimeout(
+        () => {
+          const timeoutError = new Error(`onEmbeddingPersisted timed out after ${timeoutMs}ms`);
+          timeoutError[HOOK_TIMEOUT_MARKER] = true;
+          reject(timeoutError);
+        },
+        timeoutMs
+      );
+    });
+    const hookPromise = Promise.resolve(
+      this.options.vectorRanker.onEmbeddingPersisted({
+        entityId,
+        factId,
+        vector: vectorCopy
+      })
+    );
+    try {
+      await Promise.race([hookPromise, timeoutPromise]);
+    } catch (err) {
+      hookPromise.catch(() => {
+      });
+      throw err;
+    } finally {
+      if (timeoutHandle) clearTimeout(timeoutHandle);
+    }
+  }
   async setup() {
     const entriesExistedBeforeSetup = await this.db.getFirstAsync(
       `SELECT name FROM sqlite_master WHERE type='table' AND name=?`,
@@ -753,18 +822,69 @@ var _WikiMemory = class _WikiMemory {
       let deletedEvents = 0;
       if (retainSoftDeletedFor !== null) {
         const cutoff = now - retainSoftDeletedFor * 864e5;
-        const entryResult = await this.db.runAsync(
-          `DELETE FROM ${this.prefix}entries
+        const entriesToDelete = await this.db.getAllAsync(
+          `SELECT id, entity_id FROM ${this.prefix}entries
            WHERE entity_id = ? AND deleted_at IS NOT NULL AND deleted_at < ?`,
           [entityId, cutoff]
         );
-        deletedEntries = entryResult.changes;
+        const succeeded = [];
+        let failure = null;
+        for (const row of entriesToDelete) {
+          try {
+            await this._notifyEmbeddingPersistedOrThrow(row.entity_id, row.id, null);
+            succeeded.push({ entity_id: row.entity_id, id: row.id });
+          } catch (err) {
+            failure = { factId: row.id, cause: err };
+            break;
+          }
+        }
+        if (succeeded.length > 0) {
+          const chunkSize = 500;
+          for (let i = 0; i < succeeded.length; i += chunkSize) {
+            const chunk = succeeded.slice(i, i + chunkSize);
+            const placeholders = chunk.map(() => "?").join(",");
+            const entryResult = await this.db.runAsync(
+              `DELETE FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NOT NULL AND deleted_at < ? AND id IN (${placeholders})`,
+              [entityId, cutoff, ...chunk.map((r) => r.id)]
+            );
+            deletedEntries += entryResult.changes;
+          }
+        }
         const taskResult = await this.db.runAsync(
           `DELETE FROM ${this.prefix}tasks
            WHERE entity_id = ? AND deleted_at IS NOT NULL AND deleted_at < ?`,
           [entityId, cutoff]
         );
         deletedTasks = taskResult.changes;
+        if (failure) {
+          await this.rebuildMiniSearchIndex(entityId);
+          this.vectorCache.delete(entityId);
+          const remaining = entriesToDelete.length - succeeded.length - 1;
+          const isTimeout = failure.cause?.[HOOK_TIMEOUT_MARKER] === true;
+          if (isTimeout) {
+            throw new PrunePartialFailureError(
+              succeeded.length,
+              failure.factId,
+              remaining,
+              new Error("Deletion hook timed out"),
+              deletedTasks,
+              0
+              // events not yet deleted at this point
+            );
+          }
+          const errMsg = failure.cause?.message ?? "";
+          const isValidationError = errMsg.startsWith("Invalid deletionHookTimeoutMs");
+          const sanitizedCause = isValidationError ? failure.cause : this._sanitizeRankerError(failure.cause);
+          throw new PrunePartialFailureError(
+            succeeded.length,
+            failure.factId,
+            remaining,
+            sanitizedCause,
+            deletedTasks,
+            0
+            // events not yet deleted at this point
+          );
+        }
       }
       if (retainEventsFor !== null) {
         const cutoff = now - retainEventsFor * 864e5;
@@ -801,6 +921,10 @@ var _WikiMemory = class _WikiMemory {
     if (maxResults === 0) ; else if (trimmedQuery) {
       let usedEmbed = false;
       if (!skipEmbed && embedFn) {
+        let rankerShouldRethrow = false;
+        let pendingRankerFallbackError;
+        let usedKeywordFallback = false;
+        let scoredAlreadySortedAndLimited = false;
         try {
           const queryVec = await embedFn(trimmedQuery);
           if (queryVec.length === 0 || !queryVec.every((v) => typeof v === "number" && isFinite(v))) {
@@ -832,6 +956,7 @@ var _WikiMemory = class _WikiMemory {
               `Some facts have embeddings that do not match the current model dimension. Call runReembed() to rebuild all embeddings consistently.`
             );
           }
+          const useRanker = Boolean(this.options.vectorRanker);
           let candidateRows;
           let populateCache = true;
           let miniSearchScores;
@@ -850,15 +975,30 @@ var _WikiMemory = class _WikiMemory {
               } else {
                 const topKIds = topKResults.map((r) => r.id);
                 const inClauseChunkSize = 500;
-                candidateRows = [];
-                for (let i = 0; i < topKIds.length; i += inClauseChunkSize) {
-                  const idChunk = topKIds.slice(i, i + inClauseChunkSize);
-                  const placeholders = idChunk.map(() => "?").join(",");
-                  const chunkRows = await this.db.getAllAsync(
-                    `SELECT id, embedding_blob, embedding, updated_at, access_count FROM ${this.prefix}entries WHERE id IN (${placeholders}) AND deleted_at IS NULL`,
-                    idChunk
-                  );
-                  candidateRows.push(...chunkRows);
+                if (useRanker) {
+                  const rows = [];
+                  for (let i = 0; i < topKIds.length; i += inClauseChunkSize) {
+                    const idChunk = topKIds.slice(i, i + inClauseChunkSize);
+                    const placeholders = idChunk.map(() => "?").join(",");
+                    const chunkRows = await this.db.getAllAsync(
+                      `SELECT id, updated_at, access_count FROM ${this.prefix}entries WHERE id IN (${placeholders}) AND deleted_at IS NULL`,
+                      idChunk
+                    );
+                    rows.push(...chunkRows);
+                  }
+                  candidateRows = rows;
+                } else {
+                  const rows = [];
+                  for (let i = 0; i < topKIds.length; i += inClauseChunkSize) {
+                    const idChunk = topKIds.slice(i, i + inClauseChunkSize);
+                    const placeholders = idChunk.map(() => "?").join(",");
+                    const chunkRows = await this.db.getAllAsync(
+                      `SELECT id, embedding_blob, embedding, updated_at, access_count FROM ${this.prefix}entries WHERE id IN (${placeholders}) AND deleted_at IS NULL`,
+                      idChunk
+                    );
+                    rows.push(...chunkRows);
+                  }
+                  candidateRows = rows;
                 }
                 if (weight !== void 0 && weight < 1) {
                   const maxMsScore = Math.max(1, topKResults[0]?.score ?? 1);
@@ -867,10 +1007,17 @@ var _WikiMemory = class _WikiMemory {
               }
             }
           } else {
-            candidateRows = await this.db.getAllAsync(
-              `SELECT id, embedding_blob, embedding, updated_at, access_count FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL`,
-              [entityId]
-            );
+            if (useRanker) {
+              candidateRows = await this.db.getAllAsync(
+                `SELECT id, updated_at, access_count FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL`,
+                [entityId]
+              );
+            } else {
+              candidateRows = await this.db.getAllAsync(
+                `SELECT id, embedding_blob, embedding, updated_at, access_count FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL`,
+                [entityId]
+              );
+            }
             if (weight !== void 0 && weight < 1) {
               const msResults = this.miniSearch.search(trimmedQuery, {
                 filter: (r) => r.entity_id === entityId,
@@ -883,76 +1030,267 @@ var _WikiMemory = class _WikiMemory {
           if (candidateRows === null) {
             usedEmbed = true;
           } else {
-            let entityCache = this.vectorCache.get(entityId);
-            const tooLarge = populateCache && candidateRows.length > _WikiMemory.MAX_VECTOR_CACHE_FACTS_PER_ENTITY;
-            if (tooLarge && entityCache) {
-              this.vectorCache.delete(entityId);
-              entityCache = void 0;
-            }
-            const canCache = populateCache && !tooLarge;
-            if (canCache && !entityCache) {
-              entityCache = /* @__PURE__ */ new Map();
-            }
-            const scored = candidateRows.map((row) => {
-              let vector = entityCache?.get(row.id) ?? parseEmbedding(row.embedding_blob, row.embedding);
-              if (vector && canCache && entityCache && !entityCache.has(row.id)) {
-                entityCache.set(row.id, vector);
-              }
-              let score = 0;
-              if (vector && vector.length === queryVec.length) {
-                const cosSim = cosineSimilarity(queryVec, vector);
-                if (weight !== void 0) {
-                  const kwScore = miniSearchScores?.get(row.id) ?? 0;
-                  score = weight * Math.max(0, cosSim) + (1 - weight) * kwScore;
+            let scored;
+            if (useRanker) {
+              const candidateIds = effectivePreFilterLimit !== void 0 ? candidateRows.map((r) => r.id) : void 0;
+              try {
+                const oversampledLimit = Math.max(maxResults * 2, maxResults + 50);
+                scored = await this._rankWithVectorRanker({
+                  entityId,
+                  queryVec,
+                  candidateIds,
+                  weight,
+                  miniSearchScores,
+                  limit: oversampledLimit
+                });
+                if (scored.length > 0) {
+                  const scoredIds2 = new Set(scored.map((s) => s.id));
+                  const metaMap = /* @__PURE__ */ new Map();
+                  for (const r of candidateRows) {
+                    if (scoredIds2.has(r.id)) {
+                      metaMap.set(r.id, { updated_at: r.updated_at, access_count: r.access_count });
+                    }
+                  }
+                  scored = scored.map((s) => {
+                    const meta = metaMap.get(s.id);
+                    return { ...s, updated_at: meta?.updated_at ?? null, access_count: meta?.access_count ?? null };
+                  });
+                }
+                const scoredIds = new Set(scored.map((s) => s.id));
+                const isHybrid = weight !== void 0 && weight < 1;
+                const maxBackfill = isHybrid ? maxResults : Math.max(0, maxResults - scored.length);
+                if (maxBackfill > 0) {
+                  if (isHybrid) {
+                    const topK = [];
+                    for (const row of candidateRows) {
+                      if (scoredIds.has(row.id)) continue;
+                      const kwScore = miniSearchScores?.get(row.id) ?? 0;
+                      const candidate = { row, kwScore };
+                      if (topK.length < maxBackfill) {
+                        let insertIdx = topK.length;
+                        for (let i = 0; i < topK.length; i++) {
+                          const cmp = this._compareScoredRows(
+                            {
+                              id: candidate.row.id,
+                              score: candidate.kwScore,
+                              updated_at: candidate.row.updated_at,
+                              access_count: candidate.row.access_count
+                            },
+                            {
+                              id: topK[i].row.id,
+                              score: topK[i].kwScore,
+                              updated_at: topK[i].row.updated_at,
+                              access_count: topK[i].row.access_count
+                            }
+                          );
+                          if (cmp < 0) {
+                            insertIdx = i;
+                            break;
+                          }
+                        }
+                        topK.splice(insertIdx, 0, candidate);
+                      } else {
+                        const cmpWorst = this._compareScoredRows(
+                          {
+                            id: candidate.row.id,
+                            score: candidate.kwScore,
+                            updated_at: candidate.row.updated_at,
+                            access_count: candidate.row.access_count
+                          },
+                          {
+                            id: topK[maxBackfill - 1].row.id,
+                            score: topK[maxBackfill - 1].kwScore,
+                            updated_at: topK[maxBackfill - 1].row.updated_at,
+                            access_count: topK[maxBackfill - 1].row.access_count
+                          }
+                        );
+                        if (cmpWorst < 0) {
+                          let insertIdx = maxBackfill - 1;
+                          for (let i = 0; i < topK.length; i++) {
+                            const cmp = this._compareScoredRows(
+                              {
+                                id: candidate.row.id,
+                                score: candidate.kwScore,
+                                updated_at: candidate.row.updated_at,
+                                access_count: candidate.row.access_count
+                              },
+                              {
+                                id: topK[i].row.id,
+                                score: topK[i].kwScore,
+                                updated_at: topK[i].row.updated_at,
+                                access_count: topK[i].row.access_count
+                              }
+                            );
+                            if (cmp < 0) {
+                              insertIdx = i;
+                              break;
+                            }
+                          }
+                          topK.splice(insertIdx, 0, candidate);
+                          topK.pop();
+                        }
+                      }
+                    }
+                    for (const { row, kwScore } of topK) {
+                      scored.push({
+                        id: row.id,
+                        score: (1 - weight) * kwScore,
+                        updated_at: row.updated_at,
+                        access_count: row.access_count
+                      });
+                    }
+                  } else {
+                    const omitted = [];
+                    for (const row of candidateRows) {
+                      if (scoredIds.has(row.id)) continue;
+                      omitted.push({ id: row.id, score: -2, updated_at: row.updated_at, access_count: row.access_count });
+                    }
+                    if (omitted.length > 0) {
+                      this._tieBreakSort(omitted);
+                      scored.push(...omitted.slice(0, maxBackfill));
+                    }
+                  }
+                }
+              } catch (rankerErr) {
+                const rankerError = rankerErr instanceof Error ? rankerErr : new Error(String(rankerErr));
+                const policy = this.options.vectorRankerFallback ?? "js-cosine";
+                this.options.onVectorRankerFallback?.({
+                  error: this._sanitizeRankerError(rankerError),
+                  policy
+                });
+                if (policy === "throw") {
+                  rankerShouldRethrow = true;
+                  throw rankerError;
+                } else if (policy === "js-cosine") {
+                  let fallbackRows = candidateRows;
+                  if (fallbackRows && fallbackRows.length > 0 && !("embedding_blob" in fallbackRows[0])) {
+                    const rowIds = fallbackRows.map((r) => r.id);
+                    const embeddingsMap = /* @__PURE__ */ new Map();
+                    const chunkSize = 500;
+                    for (let i = 0; i < rowIds.length; i += chunkSize) {
+                      const idChunk = rowIds.slice(i, i + chunkSize);
+                      const placeholders = idChunk.map(() => "?").join(",");
+                      const embeddingRows = await this.db.getAllAsync(
+                        `SELECT id, embedding_blob, embedding FROM ${this.prefix}entries WHERE id IN (${placeholders}) AND entity_id = ? AND deleted_at IS NULL`,
+                        [...idChunk, entityId]
+                      );
+                      for (const row of embeddingRows) {
+                        embeddingsMap.set(row.id, { embedding_blob: row.embedding_blob, embedding: row.embedding });
+                      }
+                    }
+                    fallbackRows = fallbackRows.map((r) => ({
+                      ...r,
+                      embedding_blob: embeddingsMap.get(r.id)?.embedding_blob ?? null,
+                      embedding: embeddingsMap.get(r.id)?.embedding ?? null
+                    }));
+                  }
+                  scored = await this._rankWithJsCosine({
+                    entityId,
+                    queryVec,
+                    candidateRows: fallbackRows,
+                    weight,
+                    miniSearchScores,
+                    populateCache,
+                    limit: maxResults
+                  });
+                  scoredAlreadySortedAndLimited = true;
+                } else if (policy === "keyword") {
+                  const msResults = this.miniSearch.search(trimmedQuery, {
+                    filter: (r) => r.entity_id === entityId,
+                    combineWith: "OR"
+                  });
+                  const topResults = msResults.slice(0, maxResults);
+                  const resultIds = new Set(topResults.map((r) => r.id));
+                  const candidateMap = /* @__PURE__ */ new Map();
+                  for (const r of candidateRows) {
+                    if (resultIds.has(r.id)) {
+                      candidateMap.set(r.id, { updated_at: r.updated_at, access_count: r.access_count });
+                    }
+                  }
+                  scored = topResults.map((r) => {
+                    const meta = candidateMap.get(r.id);
+                    return {
+                      id: r.id,
+                      score: r.score ?? 0,
+                      access_count: meta?.access_count ?? null,
+                      updated_at: meta?.updated_at ?? null
+                    };
+                  });
+                  usedKeywordFallback = true;
                 } else {
-                  score = cosSim;
+                  scored = [];
                 }
-              } else if (weight !== void 0 && weight < 1) {
-                const kwScore = miniSearchScores?.get(row.id) ?? 0;
-                score = (1 - weight) * kwScore;
-              } else {
-                score = -2;
-              }
-              return { row, score };
-            });
-            if (canCache && entityCache && entityCache.size > 0) {
-              if (!this.vectorCache.has(entityId)) {
-                if (this.vectorCache.size >= _WikiMemory.MAX_VECTOR_CACHE_ENTITIES) {
-                  const oldestKey = this.vectorCache.keys().next().value;
-                  if (oldestKey !== void 0) this.vectorCache.delete(oldestKey);
+                if (this.options.propagateRankerFailureToRetrievalFallback) {
+                  const mirrored = new Error("Vector ranker failed, falling back", {
+                    cause: this._sanitizeRankerError(rankerErr)
+                  });
+                  pendingRankerFallbackError = mirrored;
                 }
-                this.vectorCache.set(entityId, entityCache);
               }
+            } else {
+              scored = await this._rankWithJsCosine({
+                entityId,
+                queryVec,
+                candidateRows,
+                weight,
+                miniSearchScores,
+                populateCache,
+                limit: maxResults
+              });
+              scoredAlreadySortedAndLimited = true;
             }
-            scored.sort((a, b) => {
-              const scoreDiff = b.score - a.score;
-              if (scoreDiff !== 0) return scoreDiff;
-              const accessCountDiff = (b.row.access_count ?? 0) - (a.row.access_count ?? 0);
-              if (accessCountDiff !== 0) return accessCountDiff;
-              const updatedAtDiff = (b.row.updated_at ?? 0) - (a.row.updated_at ?? 0);
-              if (updatedAtDiff !== 0) return updatedAtDiff;
-              return a.row.id.localeCompare(b.row.id);
-            });
-            const topIds = scored.slice(0, maxResults).map((s) => s.row.id);
-            if (topIds.length > 0) {
-              const fullRows = [];
-              const phase2ChunkSize = 500;
-              for (let i = 0; i < topIds.length; i += phase2ChunkSize) {
-                const idChunk = topIds.slice(i, i + phase2ChunkSize);
-                const placeholders = idChunk.map(() => "?").join(",");
-                const chunkRows = await this.db.getAllAsync(
-                  `SELECT * FROM ${this.prefix}entries WHERE id IN (${placeholders}) AND deleted_at IS NULL`,
-                  idChunk
-                );
-                fullRows.push(...chunkRows);
+            if (scored.length > 0) {
+              if (!usedKeywordFallback && !scoredAlreadySortedAndLimited) {
+                this._tieBreakSort(scored);
               }
-              const byId = new Map(fullRows.map((r) => [r.id, r]));
-              facts = topIds.map((id) => byId.get(id)).filter((f) => f !== void 0);
+              const topIds = (scoredAlreadySortedAndLimited ? scored : scored.slice(0, maxResults)).map((s) => s.id);
+              if (topIds.length > 0) {
+                const fullRows = [];
+                const phase2ChunkSize = 500;
+                for (let i = 0; i < topIds.length; i += phase2ChunkSize) {
+                  const idChunk = topIds.slice(i, i + phase2ChunkSize);
+                  const placeholders = idChunk.map(() => "?").join(",");
+                  const chunkRows = await this.db.getAllAsync(
+                    `SELECT * FROM ${this.prefix}entries WHERE id IN (${placeholders}) AND entity_id = ? AND deleted_at IS NULL`,
+                    [...idChunk, entityId]
+                  );
+                  fullRows.push(...chunkRows);
+                }
+                const byId = new Map(fullRows.map((r) => [r.id, r]));
+                facts = topIds.map((id) => byId.get(id)).filter((f) => f !== void 0);
+                if (facts.length < topIds.length) {
+                  const missingIds = topIds.filter((id) => !byId.has(id));
+                  const missingCount = missingIds.length;
+                  const sample = missingIds.slice(0, 5);
+                  const sampleSuffix = sample.length > 0 ? ` Missing ID sample: ${sample.join(", ")}${missingIds.length > sample.length ? ", ..." : ""}.` : "";
+                  const error = new Error(
+                    `Phase 2 fact hydration returned ${missingCount} fewer row(s) than ranked IDs for entity ${entityId}. Rows may have been concurrently soft-deleted or filtered by deleted_at during hydration, or vector ranker output may include IDs that do not exist for this entity.` + sampleSuffix
+                  );
+                  this.options.onRetrievalFallback?.(error);
+                }
+              }
+              if (pendingRankerFallbackError) {
+                this.options.onRetrievalFallback?.(pendingRankerFallbackError);
+                pendingRankerFallbackError = void 0;
+              }
+              usedEmbed = true;
+            } else {
+              if (pendingRankerFallbackError) {
+                this.options.onRetrievalFallback?.(pendingRankerFallbackError);
+                pendingRankerFallbackError = void 0;
+              }
+              usedEmbed = true;
             }
-            usedEmbed = true;
           }
         } catch (err) {
           const error = err instanceof Error ? err : new Error(String(err));
+          if (rankerShouldRethrow) {
+            throw error;
+          }
+          if (pendingRankerFallbackError) {
+            error.cause = pendingRankerFallbackError;
+            pendingRankerFallbackError = void 0;
+          }
           this.options.onRetrievalFallback?.(error);
         }
       }
@@ -969,8 +1307,8 @@ var _WikiMemory = class _WikiMemory {
             const idChunk = topIds.slice(i, i + kwChunkSize);
             const placeholders = idChunk.map(() => "?").join(",");
             const chunkRows = await this.db.getAllAsync(
-              `SELECT * FROM ${this.prefix}entries WHERE id IN (${placeholders}) AND deleted_at IS NULL`,
-              idChunk
+              `SELECT * FROM ${this.prefix}entries WHERE id IN (${placeholders}) AND entity_id = ? AND deleted_at IS NULL`,
+              [...idChunk, entityId]
             );
             kwRows.push(...chunkRows);
           }
@@ -1026,6 +1364,133 @@ var _WikiMemory = class _WikiMemory {
     });
     return { facts: parsedFacts, tasks, events: events.reverse() };
   }
+  /**
+   * Stable tie-break sort: score desc → access_count desc → updated_at desc → id asc.
+   */
+  _tieBreakSort(items) {
+    items.sort((a, b) => this._compareScoredRows(a, b));
+  }
+  /**
+   * Comparator for score + deterministic tie-break fields.
+   * Negative return means "a ranks ahead of b" for descending score order.
+   */
+  _compareScoredRows(a, b) {
+    const scoreDiff = b.score - a.score;
+    if (scoreDiff !== 0) return scoreDiff;
+    const accessCountDiff = (b.access_count ?? 0) - (a.access_count ?? 0);
+    if (accessCountDiff !== 0) return accessCountDiff;
+    const updatedAtDiff = (b.updated_at ?? 0) - (a.updated_at ?? 0);
+    if (updatedAtDiff !== 0) return updatedAtDiff;
+    return a.id.localeCompare(b.id);
+  }
+  /**
+   * Strip potentially sensitive data from ranker errors before exposing to host callbacks.
+   * Preserves error type for debugging but removes message/stack that may contain credentials.
+   * Recursively sanitizes one level of .cause; deeper chains collapse to type only.
+   */
+  _sanitizeRankerError(err) {
+    if (this.options.sanitizeRankerErrors === false) {
+      return err instanceof Error ? err : new Error(String(err));
+    }
+    const typeName = err instanceof Error ? err.constructor?.name ?? "Error" : typeof err;
+    const innerCause = err instanceof Error && err.cause !== void 0 ? new Error(`Caused by: ${err.cause?.constructor?.name ?? typeof err.cause}`) : void 0;
+    const sanitized = new Error(
+      `VectorRanker ${typeName} (message scrubbed for security)`,
+      innerCause ? { cause: innerCause } : void 0
+    );
+    sanitized.name = typeName;
+    return sanitized;
+  }
+  /**
+   * Score candidate rows using in-process JS cosine similarity.
+   * Applies hybrid blending (if weight set) and tie-break sorting before returning.
+   */
+  async _rankWithJsCosine(args) {
+    const queryVec = args.queryVec instanceof Float32Array ? args.queryVec.slice() : Array.from(args.queryVec);
+    const { entityId, candidateRows, weight, miniSearchScores, populateCache, limit } = args;
+    let entityCache = this.vectorCache.get(entityId);
+    const tooLarge = populateCache && candidateRows.length > _WikiMemory.MAX_VECTOR_CACHE_FACTS_PER_ENTITY;
+    if (tooLarge && entityCache) {
+      this.vectorCache.delete(entityId);
+      entityCache = void 0;
+    }
+    const canCache = populateCache && !tooLarge;
+    if (canCache && !entityCache) {
+      entityCache = /* @__PURE__ */ new Map();
+    }
+    const scored = candidateRows.map((row) => {
+      let vector = entityCache?.get(row.id) ?? parseEmbedding(row.embedding_blob, row.embedding);
+      if (vector && canCache && entityCache && !entityCache.has(row.id)) {
+        entityCache.set(row.id, vector);
+      }
+      let score = 0;
+      if (vector && vector.length === queryVec.length) {
+        const cosSim = cosineSimilarity(queryVec, vector);
+        if (weight !== void 0) {
+          const kwScore = miniSearchScores?.get(row.id) ?? 0;
+          score = weight * Math.max(0, cosSim) + (1 - weight) * kwScore;
+        } else {
+          score = cosSim;
+        }
+      } else if (weight !== void 0 && weight < 1) {
+        const kwScore = miniSearchScores?.get(row.id) ?? 0;
+        score = (1 - weight) * kwScore;
+      } else {
+        score = -2;
+      }
+      return { id: row.id, score, updated_at: row.updated_at, access_count: row.access_count };
+    });
+    if (canCache && entityCache && entityCache.size > 0) {
+      if (!this.vectorCache.has(entityId)) {
+        if (this.vectorCache.size >= _WikiMemory.MAX_VECTOR_CACHE_ENTITIES) {
+          const oldestKey = this.vectorCache.keys().next().value;
+          if (oldestKey !== void 0) this.vectorCache.delete(oldestKey);
+        }
+        this.vectorCache.set(entityId, entityCache);
+      }
+    }
+    this._tieBreakSort(scored);
+    return scored.slice(0, limit);
+  }
+  /**
+   * Delegate semantic ranking to the injected VectorRanker.
+   * Caller should pass an oversampledLimit to preserve recall after re-ranking.
+   * Returns scored results ready for hybrid blending and tie-break sorting.
+   */
+  async _rankWithVectorRanker(args) {
+    const { entityId, candidateIds, weight, miniSearchScores, limit } = args;
+    const ranker = this.options.vectorRanker;
+    if (!ranker) {
+      throw new Error("vectorRanker not configured");
+    }
+    const queryVecCopy = args.queryVec instanceof Float32Array ? args.queryVec.slice() : Array.from(args.queryVec);
+    const rankerResults = await ranker.rankBySimilarity({
+      entityId,
+      queryVec: queryVecCopy,
+      candidateIds,
+      limit
+    });
+    const allowedIds = candidateIds ? new Set(candidateIds) : void 0;
+    const seen = /* @__PURE__ */ new Set();
+    const normalized = [];
+    for (const r of rankerResults) {
+      if (normalized.length >= limit) break;
+      if (seen.has(r.id)) continue;
+      if (allowedIds && !allowedIds.has(r.id)) continue;
+      if (!Number.isFinite(r.semanticScore)) continue;
+      seen.add(r.id);
+      normalized.push(r);
+    }
+    const scored = normalized.map((r) => {
+      let score = r.semanticScore;
+      if (weight !== void 0) {
+        const kwScore = miniSearchScores?.get(r.id) ?? 0;
+        score = weight * Math.max(0, r.semanticScore) + (1 - weight) * kwScore;
+      }
+      return { id: r.id, score };
+    });
+    return scored;
+  }
   async getMemoryBundle(entityId) {
     return this._getFullBundle(entityId, { maxEvents: 10 });
   }
@@ -1142,7 +1607,7 @@ ${JSON.stringify(currentFacts, null, 2)}`;
           INSERT INTO ${this.prefix}entries (id, entity_id, title, body, tags, confidence, source_type, created_at, updated_at)
           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
         `, [id, entityId, fact.title, fact.body, JSON.stringify(fact.tags), fact.confidence, "agent_inferred", now, now]);
-        insertedFacts.push({ id, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
+        insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
       }
       for (const task of validTasks) {
         const id = generateId("task_");
@@ -1222,6 +1687,7 @@ The following document anchors are provided for contradiction detection only. Do
     const safeDeleted = deleted.filter((id) => mutableIds.has(id));
     const validNewFacts = newFacts.map(validateFact).filter((f) => f !== null);
     const insertedFacts = [];
+    const uniqueDeletedFactIds = Array.from(new Set(safeDeleted));
     await this.db.withTransactionAsync(async () => {
       for (const id of safeDowngraded) {
         await this.db.runAsync(`UPDATE ${this.prefix}entries SET confidence = 'tentative', updated_at = ? WHERE id = ? AND entity_id = ?`, [now, id, entityId]);
@@ -1235,11 +1701,18 @@ The following document anchors are provided for contradiction detection only. Do
           INSERT INTO ${this.prefix}entries (id, entity_id, title, body, tags, confidence, source_type, created_at, updated_at)
           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
         `, [id, entityId, fact.title, fact.body, JSON.stringify(fact.tags), fact.confidence, "agent_inferred", now, now]);
-        insertedFacts.push({ id, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
+        insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
       }
     });
     this.vectorCache.delete(entityId);
     await this.rebuildMiniSearchIndex(entityId);
+    for (const factId of uniqueDeletedFactIds) {
+      try {
+        await this._notifyEmbeddingPersisted(entityId, factId, null);
+      } catch (hookErr) {
+        console.warn(`[WikiMemory] onEmbeddingPersisted hook failed during heal for ${factId}:`, hookErr);
+      }
+    }
     for (const fact of insertedFacts) {
       await this.embedFact(fact);
     }
@@ -1539,10 +2012,17 @@ The following document anchors are provided for contradiction detection only. Do
   }
   async _doImportEntity(entityId, bundle, merge) {
     const upsertedFactIds = /* @__PURE__ */ new Set();
-    const factsWithPreservedBlob = /* @__PURE__ */ new Set();
+    const upsertedDeletedFactIds = /* @__PURE__ */ new Set();
+    const factsWithPreservedBlob = /* @__PURE__ */ new Map();
     const preservedBlobDims = /* @__PURE__ */ new Set();
+    const softDeletedFactIds = [];
     await this.db.withTransactionAsync(async () => {
       if (!merge) {
+        const toDelete = await this.db.getAllAsync(
+          `SELECT id FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL`,
+          [entityId]
+        );
+        softDeletedFactIds.push(...toDelete.map((r) => r.id));
         const now = Date.now();
         await this.db.runAsync(
           `UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE entity_id = ? AND deleted_at IS NULL`,
@@ -1596,7 +2076,8 @@ The following document anchors are provided for contradiction detection only. Do
         let blobData = null;
         if (rawBlob !== null && rawBlob.byteLength > 0 && rawBlob.byteLength % 4 === 0) {
           const copy = new ArrayBuffer(rawBlob.byteLength);
-          new Uint8Array(copy).set(rawBlob);
+          const alignedBlob = new Uint8Array(copy);
+          alignedBlob.set(rawBlob);
           const floats = new Float32Array(copy, 0, rawBlob.byteLength / 4);
           let allFinite = true;
           for (let i = 0; i < floats.length; i++) {
@@ -1606,7 +2087,7 @@ The following document anchors are provided for contradiction detection only. Do
             }
           }
           if (allFinite) {
-            blobData = rawBlob;
+            blobData = alignedBlob;
           }
         }
         if (existing) {
@@ -1622,7 +2103,7 @@ The following document anchors are provided for contradiction detection only. Do
               `UPDATE ${this.prefix}entries SET entity_id = ?, title = ?, body = ?, tags = ?, confidence = ?, source_type = ?, source_hash = ?, source_ref = ?, created_at = ?, updated_at = ?, last_accessed_at = ?, access_count = ?, deleted_at = ?, embedding_blob = ?, embedding = NULL WHERE id = ?`,
               [entityId, fact.title, fact.body, tagsJson, fact.confidence, fact.source_type, fact.source_hash, fact.source_ref, fact.created_at, safeUpdatedAt, fact.last_accessed_at, fact.access_count, fact.deleted_at, blobData, fact.id]
             );
-            factsWithPreservedBlob.add(fact.id);
+            factsWithPreservedBlob.set(fact.id, blobData);
             if (!fact.deleted_at) preservedBlobDims.add(blobData.byteLength / 4);
           } else {
             await this.db.runAsync(
@@ -1632,13 +2113,14 @@ The following document anchors are provided for contradiction detection only. Do
           }
           existingFactsById.set(fact.id, { id: fact.id, entity_id: entityId, updated_at: safeUpdatedAt });
           upsertedFactIds.add(fact.id);
+          if (fact.deleted_at) upsertedDeletedFactIds.add(fact.id);
         } else {
           if (blobData != null) {
             await this.db.runAsync(
               `INSERT INTO ${this.prefix}entries (id, entity_id, title, body, tags, confidence, source_type, source_hash, source_ref, created_at, updated_at, last_accessed_at, access_count, deleted_at, embedding_blob) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
               [fact.id, entityId, fact.title, fact.body, tagsJson, fact.confidence, fact.source_type, fact.source_hash, fact.source_ref, fact.created_at, safeUpdatedAt, fact.last_accessed_at, fact.access_count, fact.deleted_at, blobData]
             );
-            factsWithPreservedBlob.add(fact.id);
+            factsWithPreservedBlob.set(fact.id, blobData);
             if (!fact.deleted_at) preservedBlobDims.add(blobData.byteLength / 4);
           } else {
             await this.db.runAsync(
@@ -1648,6 +2130,7 @@ The following document anchors are provided for contradiction detection only. Do
           }
           existingFactsById.set(fact.id, { id: fact.id, entity_id: entityId, updated_at: safeUpdatedAt });
           upsertedFactIds.add(fact.id);
+          if (fact.deleted_at) upsertedDeletedFactIds.add(fact.id);
         }
       }
       const taskIds = bundle.tasks.map((task) => task.id);
@@ -1703,12 +2186,34 @@ The following document anchors are provided for contradiction detection only. Do
       if (!fact.deleted_at && upsertedFactIds.has(fact.id) && !factsWithPreservedBlob.has(fact.id)) {
         await this.embedFact({
           id: fact.id,
+          entity_id: entityId,
+          // Use authoritative entityId from dump key, not fact.entity_id
           title: fact.title,
           body: fact.body,
           tags: Array.isArray(fact.tags) || typeof fact.tags === "string" ? fact.tags : []
         });
       }
     }
+    for (const fact of bundle.facts) {
+      const blobData = factsWithPreservedBlob.get(fact.id);
+      if (blobData && !fact.deleted_at && upsertedFactIds.has(fact.id)) {
+        try {
+          const float32Vector = new Float32Array(blobData.buffer, blobData.byteOffset, blobData.byteLength / 4);
+          await this._notifyEmbeddingPersisted(entityId, fact.id, float32Vector);
+        } catch (hookErr) {
+          console.warn(`[WikiMemory] onEmbeddingPersisted hook failed for preserved-blob fact ${fact.id}:`, hookErr);
+        }
+      }
+    }
+    for (const factId of softDeletedFactIds) {
+      if (!upsertedFactIds.has(factId) || upsertedDeletedFactIds.has(factId)) {
+        try {
+          await this._notifyEmbeddingPersisted(entityId, factId, null);
+        } catch (hookErr) {
+          console.warn(`[WikiMemory] onEmbeddingPersisted(vector=null) hook failed for soft-deleted fact ${factId}:`, hookErr);
+        }
+      }
+    }
     try {
       const canonicalRow = await this.db.getFirstAsync(
         `SELECT value FROM ${this.prefix}meta WHERE key = 'embedding_dimension'`
@@ -1779,7 +2284,17 @@ The following document anchors are provided for contradiction detection only. Do
       const now = Date.now();
       let deletedEntries = 0;
       let deletedTasks = 0;
+      const deletedEntryIds = [];
       if (params.clearAll) {
+        const newDeletions = await this.db.getAllAsync(
+          `SELECT id FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL`,
+          [entityId]
+        );
+        const alreadySoftDeleted = await this.db.getAllAsync(
+          `SELECT id FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NOT NULL`,
+          [entityId]
+        );
+        deletedEntryIds.push(...newDeletions.map((e) => e.id), ...alreadySoftDeleted.map((e) => e.id));
         const [entriesRes, tasksRes] = await Promise.all([
           this.db.runAsync(`UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE entity_id = ? AND deleted_at IS NULL`, [now, now, entityId]),
           this.db.runAsync(`UPDATE ${this.prefix}tasks SET deleted_at = ?, updated_at = ? WHERE entity_id = ? AND deleted_at IS NULL`, [now, now, entityId])
@@ -1797,6 +2312,27 @@ The following document anchors are provided for contradiction detection only. Do
         if (params.sourceRef !== void 0 && !sourceRef) throw new Error("Invalid sourceRef");
         const sourceHash = params.sourceHash !== void 0 ? normalizeSourceHash(params.sourceHash) : null;
         if (params.sourceHash !== void 0 && !sourceHash) throw new Error("Invalid sourceHash (must be 64-char hex string)");
+        if (params.entryId) {
+          const entry = await this.db.getFirstAsync(
+            `SELECT id FROM ${this.prefix}entries WHERE id = ? AND entity_id = ?`,
+            [params.entryId, entityId]
+          );
+          if (entry) deletedEntryIds.push(entry.id);
+        }
+        if (sourceRef || sourceHash) {
+          let q = `SELECT id FROM ${this.prefix}entries WHERE entity_id = ?`;
+          const args = [entityId];
+          if (sourceRef) {
+            q += ` AND source_ref = ?`;
+            args.push(sourceRef);
+          }
+          if (sourceHash) {
+            q += ` AND source_hash = ?`;
+            args.push(sourceHash);
+          }
+          const entriesToDelete = await this.db.getAllAsync(q, args);
+          deletedEntryIds.push(...entriesToDelete.map((e) => e.id));
+        }
         const entryPromise = params.entryId ? this.db.runAsync(`UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE id = ? AND entity_id = ? AND deleted_at IS NULL`, [now, now, params.entryId, entityId]) : null;
         const taskPromise = params.taskId ? this.db.runAsync(`UPDATE ${this.prefix}tasks SET deleted_at = ?, updated_at = ? WHERE id = ? AND entity_id = ? AND deleted_at IS NULL`, [now, now, params.taskId, entityId]) : null;
         let refPromise = null;
@@ -1824,6 +2360,31 @@ The following document anchors are provided for contradiction detection only. Do
       }
       await this.rebuildMiniSearchIndex(entityId);
       this.vectorCache.delete(entityId);
+      const uniqueDeletedIds = Array.from(new Set(deletedEntryIds));
+      for (const factId of uniqueDeletedIds) {
+        try {
+          await this._notifyEmbeddingPersistedOrThrow(entityId, factId, null);
+        } catch (hookErr) {
+          const isTimeout = hookErr?.[HOOK_TIMEOUT_MARKER] === true;
+          if (isTimeout) {
+            throw new Error(
+              `forget(${entityId}/${factId}) failed: ${hookErr.message}`
+            );
+          }
+          const errMsg = hookErr?.message ?? "";
+          const isValidationError = errMsg.startsWith("Invalid deletionHookTimeoutMs");
+          if (isValidationError) {
+            throw new Error(
+              `forget(${entityId}/${factId}) failed: ${errMsg}`,
+              { cause: hookErr }
+            );
+          }
+          throw new Error(
+            `forget(${entityId}/${factId}) failed: ANN cleanup hook rejected`,
+            { cause: this._sanitizeRankerError(hookErr) }
+          );
+        }
+      }
       return { deleted: { entries: deletedEntries, tasks: deletedTasks } };
     } finally {
       this.activeMaintenanceJobs.delete(forgetKey);
@@ -1893,7 +2454,15 @@ ${chunk}`;
       }
       const now = Date.now();
       const insertedFacts = [];
+      const deletedSourceFactIds = [];
       await this.db.withTransactionAsync(async () => {
+        const existingSourceFacts = await this.db.getAllAsync(
+          `SELECT id FROM ${this.prefix}entries WHERE source_ref = ? AND entity_id = ? AND deleted_at IS NULL`,
+          [sourceRef, entityId]
+        );
+        for (const row of existingSourceFacts) {
+          deletedSourceFactIds.push(row.id);
+        }
         await this.db.runAsync(
           `UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE source_ref = ? AND entity_id = ? AND deleted_at IS NULL`,
           [now, now, sourceRef, entityId]
@@ -1905,11 +2474,19 @@ ${chunk}`;
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
             [id, entityId, fact.title, fact.body, JSON.stringify(fact.tags), fact.confidence, "user_document", sourceHash, sourceRef, now, now]
           );
-          insertedFacts.push({ id, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
+          insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
         }
       });
       await this.rebuildMiniSearchIndex(entityId);
       this.vectorCache.delete(entityId);
+      const uniqueDeletedSourceFactIds = Array.from(new Set(deletedSourceFactIds));
+      for (const factId of uniqueDeletedSourceFactIds) {
+        try {
+          await this._notifyEmbeddingPersisted(entityId, factId, null);
+        } catch (hookErr) {
+          console.warn(`[WikiMemory] onEmbeddingPersisted hook failed during ingest for ${factId}:`, hookErr);
+        }
+      }
       for (const fact of insertedFacts) {
         await this.embedFact(fact);
       }
@@ -2154,6 +2731,6 @@ function createWiki(db, options) {
   return new WikiMemory(db, options);
 }
 
-export { WikiBusyError, WikiMemory, createWiki, formatContext, formatMemoryDump };
+export { PrunePartialFailureError, WikiBusyError, WikiMemory, createWiki, formatContext, formatMemoryDump };
 //# sourceMappingURL=index.mjs.map
 //# sourceMappingURL=index.mjs.map