@equationalapplications/core-llm-wiki 6.4.0 → 6.5.1

package/README.md

@@ -454,6 +454,21 @@ wikiMemory.clearVectorCache();
 
 The cache is also automatically invalidated on any mutation (`runLibrarian`, `runHeal`, `runPrune`, `runReembed`, `ingestDocument`, `importDump`, `forget`).
 
+## Re-Embedding & Retry Behavior
+
+`runReembed(entityId?, opts?)` returns `{ embedded, skipped, failed, deferred, permanentlyFailed }`.
+
+- `failed` — attempted this sweep and failed.
+- `deferred` — previously failed and still inside its exponential backoff window (60s doubling, capped at 24h). Not an error; a later sweep will retry.
+- `permanentlyFailed` — excluded for good: a `float32_overflow` failure, or 5 failed attempts. Pass `{ force: true }` to retry these anyway.
+
+Convergence loops should test `failed`, never `deferred` — deferred rows clear themselves once their backoff elapses.
+
+Two caveats on that guidance:
+
+- A `storage_error` failure counts in `failed` but is never marked for backoff — a failure marker is itself a DB write, which is exactly what is broken. While DB writes fail, every sweep re-attempts every row, so a loop testing `failed` can spin without converging. Treat repeated `storage_error` failures as an infrastructure signal to fix, not a retry window to wait out.
+- `{ force: true }` bypasses more than `permanentlyFailed`: it also skips the backoff window, so rows that would otherwise be `deferred` are re-attempted immediately.
+
 ## Entity Status
 
 `WikiMemory` exposes the in-flight job state for a single entity through two complementary APIs.

package/dist/{chunk-DPFM33WV.mjs → chunk-YNFYSLR6.mjs}

@@ -2514,6 +2514,23 @@ var HEAL_ANCHOR_SEARCH_OVERFETCH = 4;
 var HEAL_MAX_PROMPT_CHARS = 4e4;
 var HEAL_BATCH_SIZE = 25;
 var HEAL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
+var MAX_EMBED_ATTEMPTS = 5;
+var EMBED_RETRY_BASE_MS = 6e4;
+var EMBED_RETRY_CAP_MS = 24 * 60 * 60 * 1e3;
+function embedRetryDelayMs(attempts) {
+  const n = Math.max(1, Math.trunc(attempts || 0));
+  const delay = EMBED_RETRY_BASE_MS * Math.pow(2, n - 1);
+  return Math.min(delay, EMBED_RETRY_CAP_MS);
+}
+function classifyReembedRow(row, now, force) {
+  if (force) return "attempt";
+  const failedAt = row.embedding_failed_at;
+  if (failedAt === null || failedAt === void 0) return "attempt";
+  if (row.embedding_failure_kind === "float32_overflow") return "permanent";
+  const attempts = row.embedding_attempts ?? 0;
+  if (attempts >= MAX_EMBED_ATTEMPTS) return "permanent";
+  return now - failedAt >= embedRetryDelayMs(attempts) ? "attempt" : "defer";
+}
 var SKIP_ERROR_LOG_CHARS = 4096;
 var formatSkipError = (err) => {
   let base;
@@ -2652,9 +2669,24 @@ var MaintenanceService = class {
       this.jobManager.releaseLock("ontologyBackfill", entityId);
     }
   }
+  /**
+   * Re-embed facts, honouring per-row failure markers and exponential backoff.
+   *
+   * Marker lifecycle (spec §2): markers are cleared by a successful embed, by
+   * an upsert carrying a valid blob, and by an embedding-dimension promotion.
+   * A promotion revives rows for the NEXT sweep, not this one — reconciliation
+   * runs after candidates were already classified.
+   *
+   * Residual, by design: swapping to a provider with the SAME dimension never
+   * sets embedding_dimension_mismatch, so promotion never fires and markers
+   * survive the swap. Use `runReembed({ force: true })` to clear that state —
+   * `force` bypasses classification entirely and retries every row.
+   */
   async runReembed(entityId, opts) {
     const embedFn = this.options.llmProvider.embed;
-    if (!embedFn) return { embedded: 0, skipped: 0, failed: 0 };
+    if (typeof embedFn !== "function") {
+      return { embedded: 0, skipped: 0, failed: 0, deferred: 0, permanentlyFailed: 0 };
+    }
     const op = entityId ? "reembed" : "global_reembed";
     this.jobManager.acquireLock(op, entityId ?? "*");
     try {
@@ -2677,6 +2709,10 @@ var MaintenanceService = class {
       let embedded = 0;
       let skipped = 0;
       let failed = 0;
+      let deferred = 0;
+      let permanentlyFailed = 0;
+      const force = opts?.force ?? false;
+      const now = Date.now();
       try {
         for (const row of rows) {
           const existingBlob = row.embedding_blob;
@@ -2688,8 +2724,17 @@ var MaintenanceService = class {
               continue;
             }
           }
-          const success = await this.embeddingService.embedFact(row);
-          if (success) embedded++;
+          const disposition = classifyReembedRow(row, now, force);
+          if (disposition === "defer") {
+            deferred++;
+            continue;
+          }
+          if (disposition === "permanent") {
+            permanentlyFailed++;
+            continue;
+          }
+          const result = await this.embeddingService.tryEmbedFact(row);
+          if (result.ok) embedded++;
           else failed++;
         }
         if (embedded > 0) {
@@ -2698,7 +2743,7 @@ var MaintenanceService = class {
       } finally {
         this.searchService.evictCache(entityId);
       }
-      return { embedded, skipped, failed };
+      return { embedded, skipped, failed, deferred, permanentlyFailed };
     } finally {
       this.jobManager.releaseLock(op, entityId ?? "*");
     }
@@ -3876,20 +3921,37 @@ var EmbeddingService = class {
       await this.metadataRepo.setMeta("embedding_dimension", String(dim), this.db);
     }
   }
-  /** Promotes embedding_dimension_mismatch to canonical embedding_dimension when safe. */
+  /**
+   * Promotes embedding_dimension_mismatch to canonical embedding_dimension when
+   * safe, and clears embedding failure markers as part of the same event.
+   *
+   * Marker reset (spec §2.3): a promoted dimension means a different model is
+   * producing the vectors, so prior failures no longer predict future ones.
+   * `float32_overflow` is cleared too — it is terminal only because retrying is
+   * the same arithmetic on the same vector, and after a model change it is not.
+   *
+   * All three writes commit atomically; see the transaction note inline.
+   *
+   * Revived rows are NOT embedded here. They become eligible and are picked up
+   * by the NEXT sweep, because runReembed calls this after its candidates were
+   * already classified; same-sweep revival would be re-entrant.
+   */
   async reconcileEmbeddingDimension() {
     const mismatchValue = await this.metadataRepo.getMeta("embedding_dimension_mismatch");
     if (!mismatchValue) return;
     const newDim = parseInt(mismatchValue, 10);
     const residualCount = await this.entryRepo.countStaleEmbeddings(newDim);
     if (residualCount === 0) {
-      await this.metadataRepo.setMeta("embedding_dimension", mismatchValue, this.db);
-      await this.metadataRepo.clearDimensionMismatch(this.db);
+      await this.db.withTransactionAsync(async (tx) => {
+        await this.metadataRepo.setMeta("embedding_dimension", mismatchValue, tx);
+        await this.metadataRepo.clearDimensionMismatch(tx);
+        await this.entryRepo.clearEmbeddingFailureMarkers(tx);
+      });
     }
   }
-  async embedFact(fact) {
+  async tryEmbedFact(fact) {
     const embedFn = this.options.llmProvider.embed;
-    if (!embedFn) return false;
+    if (typeof embedFn !== "function") return { ok: false, kind: "no_provider" };
     let tagsStr;
     if (Array.isArray(fact.tags)) {
       tagsStr = fact.tags.join(" ");
@@ -3904,13 +3966,15 @@ var EmbeddingService = class {
     const configuredMaxEmbedChars = this.options.config?.maxEmbedChars;
     const maxEmbedChars = Number.isFinite(configuredMaxEmbedChars) ? Math.min(Math.max(0, Math.trunc(configuredMaxEmbedChars)), EMBED_CHARS_CEILING) : DEFAULT_MAX_EMBED_CHARS;
     const text = clip(`${fact.title} ${fact.body} ${tagsStr}`.trim(), maxEmbedChars);
+    let float32Vector;
     try {
       const vector = await embedFn(text);
       if (vector.length === 0 || !vector.every((v) => typeof v === "number" && isFinite(v))) {
         console.warn(`[WikiMemory] embedFact: embed() returned an invalid vector for ${fact.id}; skipping.`);
-        return false;
+        await this.markFailure(fact.id, "invalid_vector");
+        return { ok: false, kind: "invalid_vector" };
       }
-      const float32Vector = new Float32Array(vector);
+      float32Vector = new Float32Array(vector);
       let hasNonFinite = false;
       for (let i = 0; i < float32Vector.length; i++) {
         if (!isFinite(float32Vector[i])) {
@@ -3920,21 +3984,45 @@ var EmbeddingService = class {
       }
       if (hasNonFinite) {
         console.warn(`[WikiMemory] embedFact: embed() returned values that overflow float32 for ${fact.id}; skipping.`);
-        return false;
+        await this.markFailure(fact.id, "float32_overflow");
+        return { ok: false, kind: "float32_overflow" };
       }
+    } catch (err) {
+      console.warn(`[WikiMemory] embedFact failed for ${fact.id}:`, err);
+      await this.markFailure(fact.id, "provider_error");
+      return { ok: false, kind: "provider_error" };
+    }
+    try {
       await this.storeEmbeddingDimension(float32Vector.length);
       const blob = new Uint8Array(float32Vector.buffer);
       await this.entryRepo.updateEmbeddingBlob(fact.id, blob);
-      try {
-        await this.notifyEmbeddingPersisted(fact.entity_id, fact.id, float32Vector);
-      } catch (hookErr) {
-        console.warn(`[WikiMemory] onEmbeddingPersisted hook failed for ${fact.id}:`, hookErr);
-      }
-      return true;
     } catch (err) {
-      console.warn(`[WikiMemory] embedFact failed for ${fact.id}:`, err);
-      return false;
+      console.warn(`[WikiMemory] embedFact: persisting embedding failed for ${fact.id}:`, err);
+      return { ok: false, kind: "storage_error" };
     }
+    try {
+      await this.notifyEmbeddingPersisted(fact.entity_id, fact.id, float32Vector);
+    } catch (hookErr) {
+      console.warn(`[WikiMemory] onEmbeddingPersisted hook failed for ${fact.id}:`, hookErr);
+    }
+    return { ok: true, dimension: float32Vector.length };
+  }
+  /** Marker writes must never fail the caller. Only marker-eligible kinds reach here. */
+  async markFailure(id, kind) {
+    try {
+      await this.entryRepo.markEmbeddingFailure(id, kind, Date.now());
+    } catch (err) {
+      console.warn(`[WikiMemory] failed to record embedding failure for ${id}:`, err);
+    }
+  }
+  /**
+   * Back-compatible boolean form. Kept because callers such as
+   * ImportExportService branch on `if (!embedded)`; a discriminated result
+   * object is always truthy and would silently disable that branch.
+   */
+  async embedFact(fact) {
+    const result = await this.tryEmbedFact(fact);
+    return result.ok;
   }
   async notifyEmbeddingPersisted(entityId, factId, vector) {
     if (!this.options.vectorRanker?.onEmbeddingPersisted) return;
@@ -4779,5 +4867,5 @@ var WriteService = class {
 };
 
 export { BaseRepository, DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, DEFAULT_MAX_EMBED_CHARS, EMBED_CHARS_CEILING, EmbeddingService, HEAL_ANCHORS_PER_CANDIDATE, HEAL_BATCH_SIZE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ImportExportService, IngestionService, JobManager, MaintenanceService, MetadataRepository, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, RetrievalService, SearchService, WikiBusyError, WikiDuplicateHashError, WikiIngestEmptyError, WikiInvalidReadOptions, WikiParseError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiTransactionError, WriteService, __privateAdd, __privateGet, __privateSet, chunkText, configureRandomSource, emptyManifest, entitySummaryMetaKey, extractSqliteCode, generateId, normalizeSourceHash, normalizeSourceRef, normalizeTitleKey, parseEmbedding, resolveEdgeDefinitions, resolveNodeType, safeSlice, typeSatisfies, validateInlineEdges, validateManifest };
-//# sourceMappingURL=chunk-DPFM33WV.mjs.map
-//# sourceMappingURL=chunk-DPFM33WV.mjs.map
+//# sourceMappingURL=chunk-YNFYSLR6.mjs.map
+//# sourceMappingURL=chunk-YNFYSLR6.mjs.map