memorix 1.2.7 → 1.2.8

package/dist/maintenance-runner.d.ts

@@ -25,6 +25,12 @@ type MaintenanceJobRunResult = {
     delayMs: number;
     resetAttempts?: boolean;
     payload?: Record<string, unknown>;
+    /** Keep recoverable work out of the immediate pending queue during a cooldown. */
+    status?: 'pending' | 'retry';
+    /** Persist a sanitized diagnostic without consuming the job's retry budget. */
+    lastError?: string;
+    /** Clear a stale transient diagnostic after the job makes progress. */
+    clearLastError?: boolean;
 };
 
 interface IsolatedMaintenanceRequest {

package/dist/maintenance-runner.js

@@ -5918,6 +5918,10 @@ var init_maintenance_jobs = __esm({
         LIMIT 1
       `).get(input.projectId, input.kind, dedupeKey);
           if (existing) {
+            if (input.kind === "vector-backfill" && existing.status === "retry") {
+              this.commit();
+              return rowToJob(existing);
+            }
             const nextRunAfter = Math.min(Number(existing.run_after), runAfter);
             this.db.prepare(`
           UPDATE maintenance_jobs
@@ -5928,6 +5932,26 @@ var init_maintenance_jobs = __esm({
             this.commit();
             return rowToJob(updated);
           }
+          if (input.kind === "vector-backfill") {
+            const failed = this.db.prepare(`
+          SELECT * FROM maintenance_jobs
+          WHERE project_id = ? AND kind = ? AND dedupe_key = ? AND status = 'failed'
+          ORDER BY updated_at DESC
+          LIMIT 1
+        `).get(input.projectId, input.kind, dedupeKey);
+            if (failed) {
+              this.db.prepare(`
+            UPDATE maintenance_jobs
+            SET status = 'pending', attempts = 0, max_attempts = ?, run_after = ?,
+                payload_json = ?, lease_owner = NULL, lease_expires_at = NULL,
+                last_error = NULL, completed_at = NULL, updated_at = ?
+            WHERE id = ?
+          `).run(maxAttempts, runAfter, payloadJson, now3, failed.id);
+              const revived = this.getRow(failed.id);
+              this.commit();
+              return rowToJob(revived);
+            }
+          }
           const id = randomUUID2();
           this.db.prepare(`
         INSERT INTO maintenance_jobs (
@@ -6099,15 +6123,42 @@ var init_maintenance_jobs = __esm({
         const now3 = options.now ?? Date.now();
         const delayMs = Number.isFinite(options.delayMs) ? Math.max(0, Math.floor(options.delayMs)) : 0;
         const payloadJson = options.payload === void 0 ? null : JSON.stringify(options.payload);
+        const status = options.status === "retry" ? "retry" : "pending";
+        const hasLastError = options.lastError !== void 0;
+        const lastError = hasLastError ? errorText(options.lastError) : null;
         this.db.prepare(`
       UPDATE maintenance_jobs
-      SET status = 'pending', run_after = ?, attempts = CASE WHEN ? THEN 0 ELSE attempts END,
+      SET status = ?, run_after = ?, attempts = CASE WHEN ? THEN 0 ELSE attempts END,
           payload_json = COALESCE(?, payload_json),
+          last_error = CASE
+            WHEN ? THEN NULL
+            WHEN ? THEN ?
+            ELSE last_error
+          END,
           lease_owner = NULL, lease_expires_at = NULL, updated_at = ?
       WHERE id = ? AND status = 'running' AND lease_owner = ?
-    `).run(now3 + delayMs, options.resetAttempts ? 1 : 0, payloadJson, now3, id, workerId);
+    `).run(
+          status,
+          now3 + delayMs,
+          options.resetAttempts ? 1 : 0,
+          payloadJson,
+          options.clearLastError ? 1 : 0,
+          hasLastError ? 1 : 0,
+          lastError,
+          now3,
+          id,
+          workerId
+        );
         return this.get(id);
       }
+      resolveFailedVectorBackfills(projectId, dedupeKey = "vector-backfill", now3 = Date.now()) {
+        const result = this.db.prepare(`
+      UPDATE maintenance_jobs
+      SET status = 'completed', completed_at = ?, updated_at = ?
+      WHERE project_id = ? AND kind = 'vector-backfill' AND dedupe_key = ? AND status = 'failed'
+    `).run(now3, now3, projectId, dedupeKey);
+        return Number(result.changes ?? 0);
+      }
       fail(id, workerId, error, now3 = Date.now()) {
         this.begin();
         try {
@@ -6203,6 +6254,9 @@ var init_maintenance_jobs = __esm({
                 delayMs: result.delayMs,
                 resetAttempts: result.resetAttempts,
                 payload: result.payload,
+                status: result.status,
+                lastError: result.lastError,
+                clearLastError: result.clearLastError,
                 now: now3
               });
               return { state: "rescheduled", job: updated2 };
@@ -7162,6 +7216,9 @@ function normalizeEmbeddingFailure(error) {
     message: raw
   };
 }
+function vectorBackfillError(error) {
+  return sanitizeCredentials(normalizeEmbeddingFailure(error).message).slice(0, 1e3);
+}
 function queueVectorBackfill(projectId) {
   const dataDir = projectDir;
   if (!dataDir) return;
@@ -7917,17 +7974,53 @@ async function backfillVectorEmbeddings(options = {}) {
   let succeeded = 0;
   let failed = 0;
   let lastFailure;
+  const recordFailure = (message) => {
+    if (!lastFailure) lastFailure = message;
+  };
+  const candidates = [];
+  for (const id of ids) {
+    const observation = observationById.get(id);
+    if (!observation) {
+      vectorMissingIds.delete(id);
+      continue;
+    }
+    candidates.push({
+      id,
+      observation,
+      text: [observation.title, observation.narrative, ...observation.facts].join(" ")
+    });
+  }
   try {
-    for (const id of ids) {
-      const obs = observationById.get(id);
-      if (!obs) {
-        vectorMissingIds.delete(id);
-        continue;
+    if (candidates.length === 0) {
+      return { attempted: ids.length, succeeded, failed };
+    }
+    let embeddings = null;
+    try {
+      const provider2 = await getEmbeddingProvider();
+      if (!provider2) {
+        if (isEmbeddingExplicitlyDisabled()) {
+          for (const candidate of candidates) vectorMissingIds.delete(candidate.id);
+        } else {
+          recordFailure("embedding provider unavailable");
+          failed += candidates.length;
+        }
+      } else {
+        embeddings = await provider2.embedBatch(candidates.map((candidate) => candidate.text));
       }
-      const text = [obs.title, obs.narrative, ...obs.facts].join(" ");
-      try {
-        const embedding = await generateEmbedding(text);
-        if (embedding) {
+    } catch (error) {
+      recordFailure(vectorBackfillError(error));
+      failed += candidates.length;
+    }
+    if (embeddings) {
+      for (let index = 0; index < candidates.length; index++) {
+        const { id, observation: obs } = candidates[index];
+        const embedding = embeddings[index];
+        if (!embedding || embedding.length === 0) {
+          recordFailure("embedding provider returned no vector");
+          failed++;
+          continue;
+        }
+        try {
           if (!isVectorCompatibleWithCurrentIndex(embedding)) {
             await upgradeVectorSchemaAfterFirstEmbedding(embedding);
           }
@@ -7936,7 +8029,7 @@ async function backfillVectorEmbeddings(options = {}) {
             console.error(
               `[memorix] Backfill embedding mismatch for obs-${id}: provider returned ${embedding.length}d, index expects ${vectorDimensions ?? "unknown"}d (kept in queue)`
             );
-            lastFailure = `dimension mismatch: provider returned ${embedding.length}d, index expects ${vectorDimensions ?? "unknown"}d`;
+            recordFailure(`dimension mismatch: provider returned ${embedding.length}d, index expects ${vectorDimensions ?? "unknown"}d`);
             failed++;
             continue;
           }
@@ -7975,15 +8068,10 @@ async function backfillVectorEmbeddings(options = {}) {
           await insertObservation(doc);
           vectorMissingIds.delete(id);
           succeeded++;
-        } else if (isEmbeddingExplicitlyDisabled()) {
-          vectorMissingIds.delete(id);
-        } else {
-          lastFailure = "embedding provider unavailable";
+        } catch (error) {
+          recordFailure(vectorBackfillError(error));
           failed++;
         }
-      } catch (err) {
-        lastFailure = err instanceof Error ? err.message : String(err);
-        failed++;
       }
     }
   } finally {
@@ -7996,7 +8084,12 @@ async function backfillVectorEmbeddings(options = {}) {
       finishedAt: (/* @__PURE__ */ new Date()).toISOString()
     };
   }
-  return { attempted: ids.length, succeeded, failed };
+  return {
+    attempted: ids.length,
+    succeeded,
+    failed,
+    ...lastFailure ? { lastError: lastFailure } : {}
+  };
 }
 var observations, nextId, projectDir, searchIndexPrepared, vectorMissingIds, vectorBackfillRunning, vectorSchemaUpgradePromise, lastVectorBackfill, embeddingFailureLogTimestamps, EMBEDDING_FAILURE_LOG_COOLDOWN_MS;
 var init_observations = __esm({
@@ -11915,6 +12008,7 @@ var DEFAULT_TIMEOUT_MS = 5 * 6e4;
 
 // src/runtime/project-maintenance.ts
 init_esm_shims();
+init_maintenance_jobs();
 init_lifecycle();
 init_timeout();
 var DEFAULT_VECTOR_BATCH_SIZE = 12;
@@ -11924,12 +12018,34 @@ var DEFAULT_CODEGRAPH_MAX_FILES = 5e3;
 var DEFAULT_CLAIM_DERIVATION_BATCH_SIZE = 100;
 var DEFAULT_OBSERVATION_QUALIFICATION_BATCH_SIZE = 100;
 var VECTOR_RETRY_DELAY_MS = 5e3;
+var VECTOR_FAILURE_BASE_RETRY_DELAY_MS = 6e4;
+var VECTOR_FAILURE_MAX_RETRY_DELAY_MS = 15 * 6e4;
 var DEDUP_PER_PAIR_TIMEOUT_MS = 5e3;
 function vectorBatchSize(payload) {
   const value = payload.limit;
   if (typeof value !== "number" || !Number.isFinite(value)) return DEFAULT_VECTOR_BATCH_SIZE;
   return Math.min(100, Math.max(1, Math.floor(value)));
 }
+function vectorBackfillFailureStreak(payload) {
+  const value = payload.vectorBackfillFailureStreak;
+  return typeof value === "number" && Number.isFinite(value) ? Math.min(16, Math.max(0, Math.floor(value))) : 0;
+}
+function vectorBackfillRetryDelayMs(streak) {
+  return Math.min(
+    VECTOR_FAILURE_MAX_RETRY_DELAY_MS,
+    VECTOR_FAILURE_BASE_RETRY_DELAY_MS * 2 ** Math.max(0, streak - 1)
+  );
+}
+function withoutVectorBackfillFailureStreak(payload) {
+  const { vectorBackfillFailureStreak: _streak, ...rest } = payload;
+  return rest;
+}
+function resolveSupersededVectorFailures(projectDir2, projectId) {
+  try {
+    new MaintenanceJobStore(projectDir2).resolveFailedVectorBackfills(projectId);
+  } catch {
+  }
+}
 function retentionBatchSize(payload) {
   const value = payload.limit;
   if (typeof value !== "number" || !Number.isFinite(value)) return DEFAULT_RETENTION_BATCH_SIZE;
@@ -12338,20 +12454,45 @@ function createProjectMaintenanceHandler(projectId, projectDir2, projectRoot, op
     if (isEmbeddingExplicitlyDisabled2()) return { action: "complete" };
     const { backfillVectorEmbeddings: backfillVectorEmbeddings2, getVectorStatus: getVectorStatus2 } = await Promise.resolve().then(() => (init_observations(), observations_exports));
     const before = getVectorStatus2(projectId);
-    if (before.missing === 0) return { action: "complete" };
+    if (before.missing === 0) {
+      resolveSupersededVectorFailures(projectDir2, projectId);
+      return { action: "complete" };
+    }
     const result = await backfillVectorEmbeddings2({
       projectId,
       limit: vectorBatchSize(job.payload)
     });
     const after = getVectorStatus2(projectId);
-    if (after.missing === 0) return { action: "complete" };
+    if (after.missing === 0) {
+      resolveSupersededVectorFailures(projectDir2, projectId);
+      return { action: "complete" };
+    }
     if (result.failed > 0 && result.succeeded === 0) {
-      throw new Error(`vector backfill made no progress (${result.failed}/${result.attempted} failed)`);
+      const streak = vectorBackfillFailureStreak(job.payload) + 1;
+      return {
+        action: "reschedule",
+        status: "retry",
+        delayMs: vectorBackfillRetryDelayMs(streak),
+        // This is a recoverable provider/index state, not a terminal job error.
+        // Keeping one retry row prevents new MCP processes from creating a storm.
+        resetAttempts: true,
+        payload: {
+          ...job.payload,
+          vectorBackfillFailureStreak: streak
+        },
+        lastError: result.lastError ?? `vector backfill made no progress (${result.failed}/${result.attempted} failed)`
+      };
     }
+    const priorFailureStreak = vectorBackfillFailureStreak(job.payload);
+    const payload = withoutVectorBackfillFailureStreak(job.payload);
     return {
       action: "reschedule",
       delayMs: result.attempted === 0 ? VECTOR_RETRY_DELAY_MS : 0,
-      resetAttempts: result.succeeded > 0
+      resetAttempts: result.succeeded > 0,
+      ...result.succeeded > 0 ? {
+        ...priorFailureStreak > 0 ? { payload } : {},
+        ...result.failed > 0 && result.lastError ? { lastError: result.lastError } : priorFailureStreak > 0 ? { clearLastError: true } : {}
+      } : {}
     };
   };
 }