@equationalapplications/core-llm-wiki 6.4.0 → 6.5.1

package/dist/index.d.mts

@@ -1,5 +1,5 @@
-import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, O as OntologyManifest, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-DWpAgNLt.mjs';
-export { C as ChunkFailure, D as DegradedRecord, E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HEAL_BATCH_SIZE, i as HEAL_RECHECK_MS, j as HOOK_TIMEOUT_MARKER, k as HealResult, I as IngestDocumentResult, L as LLMProvider, l as ONTOLOGY_BACKFILL_BATCH_SIZE, m as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, n as ONTOLOGY_BACKFILL_RECHECK_MS, o as OntologyBackfillResult, p as OntologyConfig, q as OntologyEdgeType, r as OntologyMode, s as OntologyNodeType, t as OntologyPromptContext, u as OntologyUpdates, P as PromptOverrides, v as PromptService, w as PrunePartialFailureError, V as VectorRanker, x as VectorRankerFallback, y as VectorRankerRankArgs, z as VectorRankerSemanticResult, A as WikiBusyError, B as WikiBusyOperation, J as WikiCheckpoint, K as WikiConfig, N as WikiDuplicateHashError, Q as WikiEdge, T as WikiEvent, U as WikiFact, X as WikiIngestEmptyError, Y as WikiInvalidReadOptions, Z as WikiMemoryTestAccess, _ as WikiOutboxEvent, $ as WikiParseError, a0 as WikiSourceRefHashCollision, a1 as WikiStrictOntologyViolation, a2 as WikiTask, a3 as WikiTransactionError } from './testing-DWpAgNLt.mjs';
+import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, O as OntologyManifest, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-BtsN1pTK.mjs';
+export { C as ChunkFailure, D as DegradedRecord, E as EmbedFactResult, d as EmbedFailureKind, e as EmbeddingMarkerKind, f as EntityStatus, g as ExtractedFact, h as ExtractedFactEdge, i as ExtractedFactWithOntology, j as ExtractedTask, k as GraphTraversalOptions, H as HEAL_BATCH_SIZE, l as HEAL_RECHECK_MS, m as HOOK_TIMEOUT_MARKER, n as HealResult, I as IngestDocumentResult, L as LLMProvider, o as ONTOLOGY_BACKFILL_BATCH_SIZE, p as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, q as ONTOLOGY_BACKFILL_RECHECK_MS, r as OntologyBackfillResult, s as OntologyConfig, t as OntologyEdgeType, u as OntologyMode, v as OntologyNodeType, w as OntologyPromptContext, x as OntologyUpdates, P as PromptOverrides, y as PromptService, z as PrunePartialFailureError, A as ReembedResult, V as VectorRanker, B as VectorRankerFallback, J as VectorRankerRankArgs, K as VectorRankerSemanticResult, N as WikiBusyError, Q as WikiBusyOperation, T as WikiCheckpoint, U as WikiConfig, X as WikiDuplicateHashError, Y as WikiEdge, Z as WikiEvent, _ as WikiFact, $ as WikiIngestEmptyError, a0 as WikiInvalidReadOptions, a1 as WikiMemoryTestAccess, a2 as WikiOutboxEvent, a3 as WikiParseError, a4 as WikiSourceRefHashCollision, a5 as WikiStrictOntologyViolation, a6 as WikiTask, a7 as WikiTransactionError } from './testing-BtsN1pTK.mjs';
 import { OkfFile } from '@equationalapplications/core-okf';
 import 'minisearch';
 

package/dist/index.d.ts

@@ -1,5 +1,5 @@
-import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, O as OntologyManifest, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-DWpAgNLt.js';
-export { C as ChunkFailure, D as DegradedRecord, E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HEAL_BATCH_SIZE, i as HEAL_RECHECK_MS, j as HOOK_TIMEOUT_MARKER, k as HealResult, I as IngestDocumentResult, L as LLMProvider, l as ONTOLOGY_BACKFILL_BATCH_SIZE, m as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, n as ONTOLOGY_BACKFILL_RECHECK_MS, o as OntologyBackfillResult, p as OntologyConfig, q as OntologyEdgeType, r as OntologyMode, s as OntologyNodeType, t as OntologyPromptContext, u as OntologyUpdates, P as PromptOverrides, v as PromptService, w as PrunePartialFailureError, V as VectorRanker, x as VectorRankerFallback, y as VectorRankerRankArgs, z as VectorRankerSemanticResult, A as WikiBusyError, B as WikiBusyOperation, J as WikiCheckpoint, K as WikiConfig, N as WikiDuplicateHashError, Q as WikiEdge, T as WikiEvent, U as WikiFact, X as WikiIngestEmptyError, Y as WikiInvalidReadOptions, Z as WikiMemoryTestAccess, _ as WikiOutboxEvent, $ as WikiParseError, a0 as WikiSourceRefHashCollision, a1 as WikiStrictOntologyViolation, a2 as WikiTask, a3 as WikiTransactionError } from './testing-DWpAgNLt.js';
+import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, O as OntologyManifest, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-BtsN1pTK.js';
+export { C as ChunkFailure, D as DegradedRecord, E as EmbedFactResult, d as EmbedFailureKind, e as EmbeddingMarkerKind, f as EntityStatus, g as ExtractedFact, h as ExtractedFactEdge, i as ExtractedFactWithOntology, j as ExtractedTask, k as GraphTraversalOptions, H as HEAL_BATCH_SIZE, l as HEAL_RECHECK_MS, m as HOOK_TIMEOUT_MARKER, n as HealResult, I as IngestDocumentResult, L as LLMProvider, o as ONTOLOGY_BACKFILL_BATCH_SIZE, p as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, q as ONTOLOGY_BACKFILL_RECHECK_MS, r as OntologyBackfillResult, s as OntologyConfig, t as OntologyEdgeType, u as OntologyMode, v as OntologyNodeType, w as OntologyPromptContext, x as OntologyUpdates, P as PromptOverrides, y as PromptService, z as PrunePartialFailureError, A as ReembedResult, V as VectorRanker, B as VectorRankerFallback, J as VectorRankerRankArgs, K as VectorRankerSemanticResult, N as WikiBusyError, Q as WikiBusyOperation, T as WikiCheckpoint, U as WikiConfig, X as WikiDuplicateHashError, Y as WikiEdge, Z as WikiEvent, _ as WikiFact, $ as WikiIngestEmptyError, a0 as WikiInvalidReadOptions, a1 as WikiMemoryTestAccess, a2 as WikiOutboxEvent, a3 as WikiParseError, a4 as WikiSourceRefHashCollision, a5 as WikiStrictOntologyViolation, a6 as WikiTask, a7 as WikiTransactionError } from './testing-BtsN1pTK.js';
 import { OkfFile } from '@equationalapplications/core-okf';
 import 'minisearch';
 

package/dist/index.js

@@ -45,7 +45,10 @@ async function setupDatabase(db, prefix) {
       last_verified_by TEXT,
       okf_sources TEXT,
       okf_verified TEXT,
-      okf_usage_window TEXT
+      okf_usage_window TEXT,
+      embedding_failed_at INTEGER,
+      embedding_failure_kind TEXT,
+      embedding_attempts INTEGER NOT NULL DEFAULT 0
     );
 
     CREATE INDEX IF NOT EXISTS ${prefix}entries_entity_idx ON ${prefix}entries(entity_id);
@@ -576,6 +579,24 @@ var MIGRATIONS = [
         CREATE INDEX IF NOT EXISTS ${prefix}tasks_last_verified_at_idx ON ${prefix}tasks(last_verified_at);
       `);
     }
+  },
+  {
+    version: 11,
+    description: "Add embedding_failed_at, embedding_failure_kind, embedding_attempts to entries for embed retry state",
+    run: async (db, prefix) => {
+      const cols = await db.getAllAsync(
+        `PRAGMA table_info(${prefix}entries)`
+      );
+      const existing = new Set(cols.map((c) => c.name));
+      const adds = [
+        ["embedding_failed_at", `ALTER TABLE ${prefix}entries ADD COLUMN embedding_failed_at INTEGER`],
+        ["embedding_failure_kind", `ALTER TABLE ${prefix}entries ADD COLUMN embedding_failure_kind TEXT`],
+        ["embedding_attempts", `ALTER TABLE ${prefix}entries ADD COLUMN embedding_attempts INTEGER NOT NULL DEFAULT 0`]
+      ];
+      for (const [name, sql] of adds) {
+        if (!existing.has(name)) await db.execAsync(sql);
+      }
+    }
   }
 ];
 for (let i = 1; i < MIGRATIONS.length; i++) {
@@ -736,6 +757,13 @@ var EntryRepository = class extends BaseRepository {
    * New-row INSERTs still bind a value so the column is populated; if the
    * caller supplied nothing, the schema DEFAULT 'stable' applies when the
    * bind is bound as 'stable' explicitly here.
+   *
+   * Marker discipline (spec §4.1): a conflict carrying a real
+   * `embedding_blob` clears the embedding failure markers, because the row now
+   * HAS a valid embedding and a surviving marker would be a self-contradictory
+   * diagnostic that inflates runReembed's permanentlyFailed counter. A conflict
+   * with no blob leaves marker state alone, matching the "absent means don't
+   * touch" semantics used for embedding_blob itself.
    */
   async upsert(fact, tx) {
     const executor = this.getExecutor(tx);
@@ -773,6 +801,9 @@ var EntryRepository = class extends BaseRepository {
         deleted_at = excluded.deleted_at,
         embedding_blob = CASE WHEN excluded.embedding_blob IS NULL THEN embedding_blob ELSE excluded.embedding_blob END,
         embedding = NULL,
+        embedding_failed_at = CASE WHEN excluded.embedding_blob IS NOT NULL THEN NULL ELSE embedding_failed_at END,
+        embedding_failure_kind = CASE WHEN excluded.embedding_blob IS NOT NULL THEN NULL ELSE embedding_failure_kind END,
+        embedding_attempts = CASE WHEN excluded.embedding_blob IS NOT NULL THEN 0 ELSE embedding_attempts END,
         okf_type = excluded.okf_type`,
       [
         fact.id,
@@ -901,6 +932,9 @@ var EntryRepository = class extends BaseRepository {
         deleted_at = excluded.deleted_at,
         embedding_blob = excluded.embedding_blob,
         embedding = NULL,
+        embedding_failed_at = CASE WHEN excluded.embedding_blob IS NOT NULL THEN NULL ELSE embedding_failed_at END,
+        embedding_failure_kind = CASE WHEN excluded.embedding_blob IS NOT NULL THEN NULL ELSE embedding_failure_kind END,
+        embedding_attempts = CASE WHEN excluded.embedding_blob IS NOT NULL THEN 0 ELSE embedding_attempts END,
         okf_type = excluded.okf_type,
         lifecycle_status = excluded.lifecycle_status,
         stale_after = excluded.stale_after,
@@ -1337,10 +1371,58 @@ var EntryRepository = class extends BaseRepository {
   async updateEmbeddingBlob(id, blob, tx) {
     const executor = this.getExecutor(tx);
     await executor.runAsync(
-      `UPDATE ${this.prefix}entries SET embedding_blob = ?, embedding = NULL WHERE id = ?`,
+      `UPDATE ${this.prefix}entries
+         SET embedding_blob = ?,
+             embedding = NULL,
+             embedding_failed_at = NULL,
+             embedding_failure_kind = NULL,
+             embedding_attempts = 0
+       WHERE id = ?`,
       [blob, id]
     );
   }
+  /**
+   * Record a failed embedding attempt. Deliberately does NOT touch updated_at
+   * (import merge is last-write-wins on it) and pushes no outbox event —
+   * embedding lifecycle is local state, not replicated. Same discipline as
+   * updateEmbeddingBlob. See spec §3.5.
+   *
+   * Only marker-eligible kinds (spec §3.3) are accepted: `no_provider` never
+   * marks and `storage_error` never marks, so retry/permanent-failure policy
+   * built on these rows only ever sees real embed failures.
+   */
+  async markEmbeddingFailure(id, kind, now, tx) {
+    const executor = this.getExecutor(tx);
+    await executor.runAsync(
+      `UPDATE ${this.prefix}entries
+         SET embedding_failed_at = ?,
+             embedding_failure_kind = ?,
+             embedding_attempts = embedding_attempts + 1
+       WHERE id = ?`,
+      [now, kind, id]
+    );
+  }
+  /**
+   * Clear marker state for every marked row. Called when the embedding
+   * dimension is promoted: a new model produces different vectors, so past
+   * failures — including `float32_overflow`, which is otherwise terminal —
+   * no longer predict future ones (spec §2.3).
+   *
+   * Same discipline as markEmbeddingFailure: no `updated_at` touch and no
+   * outbox event, because embedding lifecycle is local state, not replicated.
+   * Returns the number of rows cleared.
+   */
+  async clearEmbeddingFailureMarkers(tx) {
+    const executor = this.getExecutor(tx);
+    const result = await executor.runAsync(
+      `UPDATE ${this.prefix}entries
+          SET embedding_failed_at = NULL,
+              embedding_failure_kind = NULL,
+              embedding_attempts = 0
+        WHERE embedding_failed_at IS NOT NULL`
+    );
+    return result.changes;
+  }
   async hasLegacySourceTypes(tx) {
     const executor = this.getExecutor(tx);
     const row = await executor.getFirstAsync(
@@ -4814,6 +4896,23 @@ var HEAL_ANCHOR_SEARCH_OVERFETCH = 4;
 var HEAL_MAX_PROMPT_CHARS = 4e4;
 var HEAL_BATCH_SIZE = 25;
 var HEAL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
+var MAX_EMBED_ATTEMPTS = 5;
+var EMBED_RETRY_BASE_MS = 6e4;
+var EMBED_RETRY_CAP_MS = 24 * 60 * 60 * 1e3;
+function embedRetryDelayMs(attempts) {
+  const n = Math.max(1, Math.trunc(attempts || 0));
+  const delay = EMBED_RETRY_BASE_MS * Math.pow(2, n - 1);
+  return Math.min(delay, EMBED_RETRY_CAP_MS);
+}
+function classifyReembedRow(row, now, force) {
+  if (force) return "attempt";
+  const failedAt = row.embedding_failed_at;
+  if (failedAt === null || failedAt === void 0) return "attempt";
+  if (row.embedding_failure_kind === "float32_overflow") return "permanent";
+  const attempts = row.embedding_attempts ?? 0;
+  if (attempts >= MAX_EMBED_ATTEMPTS) return "permanent";
+  return now - failedAt >= embedRetryDelayMs(attempts) ? "attempt" : "defer";
+}
 var SKIP_ERROR_LOG_CHARS = 4096;
 var formatSkipError = (err) => {
   let base;
@@ -4952,9 +5051,24 @@ var MaintenanceService = class {
       this.jobManager.releaseLock("ontologyBackfill", entityId);
     }
   }
+  /**
+   * Re-embed facts, honouring per-row failure markers and exponential backoff.
+   *
+   * Marker lifecycle (spec §2): markers are cleared by a successful embed, by
+   * an upsert carrying a valid blob, and by an embedding-dimension promotion.
+   * A promotion revives rows for the NEXT sweep, not this one — reconciliation
+   * runs after candidates were already classified.
+   *
+   * Residual, by design: swapping to a provider with the SAME dimension never
+   * sets embedding_dimension_mismatch, so promotion never fires and markers
+   * survive the swap. Use `runReembed({ force: true })` to clear that state —
+   * `force` bypasses classification entirely and retries every row.
+   */
   async runReembed(entityId, opts) {
     const embedFn = this.options.llmProvider.embed;
-    if (!embedFn) return { embedded: 0, skipped: 0, failed: 0 };
+    if (typeof embedFn !== "function") {
+      return { embedded: 0, skipped: 0, failed: 0, deferred: 0, permanentlyFailed: 0 };
+    }
     const op = entityId ? "reembed" : "global_reembed";
     this.jobManager.acquireLock(op, entityId ?? "*");
     try {
@@ -4977,6 +5091,10 @@ var MaintenanceService = class {
       let embedded = 0;
       let skipped = 0;
       let failed = 0;
+      let deferred = 0;
+      let permanentlyFailed = 0;
+      const force = opts?.force ?? false;
+      const now = Date.now();
       try {
         for (const row of rows) {
           const existingBlob = row.embedding_blob;
@@ -4988,8 +5106,17 @@ var MaintenanceService = class {
               continue;
             }
           }
-          const success = await this.embeddingService.embedFact(row);
-          if (success) embedded++;
+          const disposition = classifyReembedRow(row, now, force);
+          if (disposition === "defer") {
+            deferred++;
+            continue;
+          }
+          if (disposition === "permanent") {
+            permanentlyFailed++;
+            continue;
+          }
+          const result = await this.embeddingService.tryEmbedFact(row);
+          if (result.ok) embedded++;
           else failed++;
         }
         if (embedded > 0) {
@@ -4998,7 +5125,7 @@ var MaintenanceService = class {
       } finally {
         this.searchService.evictCache(entityId);
       }
-      return { embedded, skipped, failed };
+      return { embedded, skipped, failed, deferred, permanentlyFailed };
     } finally {
       this.jobManager.releaseLock(op, entityId ?? "*");
     }
@@ -6176,20 +6303,37 @@ var EmbeddingService = class {
       await this.metadataRepo.setMeta("embedding_dimension", String(dim), this.db);
     }
   }
-  /** Promotes embedding_dimension_mismatch to canonical embedding_dimension when safe. */
+  /**
+   * Promotes embedding_dimension_mismatch to canonical embedding_dimension when
+   * safe, and clears embedding failure markers as part of the same event.
+   *
+   * Marker reset (spec §2.3): a promoted dimension means a different model is
+   * producing the vectors, so prior failures no longer predict future ones.
+   * `float32_overflow` is cleared too — it is terminal only because retrying is
+   * the same arithmetic on the same vector, and after a model change it is not.
+   *
+   * All three writes commit atomically; see the transaction note inline.
+   *
+   * Revived rows are NOT embedded here. They become eligible and are picked up
+   * by the NEXT sweep, because runReembed calls this after its candidates were
+   * already classified; same-sweep revival would be re-entrant.
+   */
   async reconcileEmbeddingDimension() {
     const mismatchValue = await this.metadataRepo.getMeta("embedding_dimension_mismatch");
     if (!mismatchValue) return;
     const newDim = parseInt(mismatchValue, 10);
     const residualCount = await this.entryRepo.countStaleEmbeddings(newDim);
     if (residualCount === 0) {
-      await this.metadataRepo.setMeta("embedding_dimension", mismatchValue, this.db);
-      await this.metadataRepo.clearDimensionMismatch(this.db);
+      await this.db.withTransactionAsync(async (tx) => {
+        await this.metadataRepo.setMeta("embedding_dimension", mismatchValue, tx);
+        await this.metadataRepo.clearDimensionMismatch(tx);
+        await this.entryRepo.clearEmbeddingFailureMarkers(tx);
+      });
     }
   }
-  async embedFact(fact) {
+  async tryEmbedFact(fact) {
     const embedFn = this.options.llmProvider.embed;
-    if (!embedFn) return false;
+    if (typeof embedFn !== "function") return { ok: false, kind: "no_provider" };
     let tagsStr;
     if (Array.isArray(fact.tags)) {
       tagsStr = fact.tags.join(" ");
@@ -6204,13 +6348,15 @@ var EmbeddingService = class {
     const configuredMaxEmbedChars = this.options.config?.maxEmbedChars;
     const maxEmbedChars = Number.isFinite(configuredMaxEmbedChars) ? Math.min(Math.max(0, Math.trunc(configuredMaxEmbedChars)), EMBED_CHARS_CEILING) : DEFAULT_MAX_EMBED_CHARS;
     const text = clip(`${fact.title} ${fact.body} ${tagsStr}`.trim(), maxEmbedChars);
+    let float32Vector;
     try {
       const vector = await embedFn(text);
       if (vector.length === 0 || !vector.every((v) => typeof v === "number" && isFinite(v))) {
         console.warn(`[WikiMemory] embedFact: embed() returned an invalid vector for ${fact.id}; skipping.`);
-        return false;
+        await this.markFailure(fact.id, "invalid_vector");
+        return { ok: false, kind: "invalid_vector" };
       }
-      const float32Vector = new Float32Array(vector);
+      float32Vector = new Float32Array(vector);
       let hasNonFinite = false;
       for (let i = 0; i < float32Vector.length; i++) {
         if (!isFinite(float32Vector[i])) {
@@ -6220,21 +6366,45 @@ var EmbeddingService = class {
       }
       if (hasNonFinite) {
         console.warn(`[WikiMemory] embedFact: embed() returned values that overflow float32 for ${fact.id}; skipping.`);
-        return false;
+        await this.markFailure(fact.id, "float32_overflow");
+        return { ok: false, kind: "float32_overflow" };
       }
+    } catch (err) {
+      console.warn(`[WikiMemory] embedFact failed for ${fact.id}:`, err);
+      await this.markFailure(fact.id, "provider_error");
+      return { ok: false, kind: "provider_error" };
+    }
+    try {
       await this.storeEmbeddingDimension(float32Vector.length);
       const blob = new Uint8Array(float32Vector.buffer);
       await this.entryRepo.updateEmbeddingBlob(fact.id, blob);
-      try {
-        await this.notifyEmbeddingPersisted(fact.entity_id, fact.id, float32Vector);
-      } catch (hookErr) {
-        console.warn(`[WikiMemory] onEmbeddingPersisted hook failed for ${fact.id}:`, hookErr);
-      }
-      return true;
     } catch (err) {
-      console.warn(`[WikiMemory] embedFact failed for ${fact.id}:`, err);
-      return false;
+      console.warn(`[WikiMemory] embedFact: persisting embedding failed for ${fact.id}:`, err);
+      return { ok: false, kind: "storage_error" };
+    }
+    try {
+      await this.notifyEmbeddingPersisted(fact.entity_id, fact.id, float32Vector);
+    } catch (hookErr) {
+      console.warn(`[WikiMemory] onEmbeddingPersisted hook failed for ${fact.id}:`, hookErr);
     }
+    return { ok: true, dimension: float32Vector.length };
+  }
+  /** Marker writes must never fail the caller. Only marker-eligible kinds reach here. */
+  async markFailure(id, kind) {
+    try {
+      await this.entryRepo.markEmbeddingFailure(id, kind, Date.now());
+    } catch (err) {
+      console.warn(`[WikiMemory] failed to record embedding failure for ${id}:`, err);
+    }
+  }
+  /**
+   * Back-compatible boolean form. Kept because callers such as
+   * ImportExportService branch on `if (!embedded)`; a discriminated result
+   * object is always truthy and would silently disable that branch.
+   */
+  async embedFact(fact) {
+    const result = await this.tryEmbedFact(fact);
+    return result.ok;
   }
   async notifyEmbeddingPersisted(entityId, factId, vector) {
     if (!this.options.vectorRanker?.onEmbeddingPersisted) return;