memorix 1.2.6 → 1.2.8

package/src/memory/observations.ts

@@ -102,6 +102,10 @@ function normalizeEmbeddingFailure(error: unknown): { key: string; message: stri
   };
 }
 
+function vectorBackfillError(error: unknown): string {
+  return sanitizeCredentials(normalizeEmbeddingFailure(error).message).slice(0, 1_000);
+}
+
 function queueVectorBackfill(projectId: string): void {
   const dataDir = projectDir;
   if (!dataDir) return;
@@ -1202,6 +1206,7 @@ export async function backfillVectorEmbeddings(options: {
   attempted: number;
   succeeded: number;
   failed: number;
+  lastError?: string;
 }> {
   if (vectorBackfillRunning) {
     return { attempted: 0, succeeded: 0, failed: 0 };
@@ -1218,19 +1223,63 @@ export async function backfillVectorEmbeddings(options: {
   let succeeded = 0;
   let failed = 0;
   let lastFailure: string | undefined;
+  const recordFailure = (message: string): void => {
+    // A single batch may have several symptoms after one root cause. Preserve
+    // the first one so an informative provider/index error is not overwritten
+    // by later missing-result fallout.
+    if (!lastFailure) lastFailure = message;
+  };
+
+  const candidates: Array<{ id: number; observation: Observation; text: string }> = [];
+  for (const id of ids) {
+    const observation = observationById.get(id);
+    if (!observation) {
+      vectorMissingIds.delete(id);
+      continue;
+    }
+    candidates.push({
+      id,
+      observation,
+      text: [observation.title, observation.narrative, ...observation.facts].join(' '),
+    });
+  }
 
   try {
-    for (const id of ids) {
-      const obs = observationById.get(id);
-      if (!obs) {
-        vectorMissingIds.delete(id);
-        continue;
+    if (candidates.length === 0) {
+      return { attempted: ids.length, succeeded, failed };
+    }
+
+    let embeddings: number[][] | null = null;
+    try {
+      const provider = await getEmbeddingProvider();
+      if (!provider) {
+        if (isEmbeddingExplicitlyDisabled()) {
+          for (const candidate of candidates) vectorMissingIds.delete(candidate.id);
+        } else {
+          recordFailure('embedding provider unavailable');
+          failed += candidates.length;
+        }
+      } else {
+        // API providers batch and cache this efficiently; local providers can use
+        // their native batch path. Index writes stay sequential because Orama is
+        // process-local mutable state.
+        embeddings = await provider.embedBatch(candidates.map((candidate) => candidate.text));
       }
+    } catch (error) {
+      recordFailure(vectorBackfillError(error));
+      failed += candidates.length;
+    }
 
-      const text = [obs.title, obs.narrative, ...obs.facts].join(' ');
-      try {
-        const embedding = await generateEmbedding(text);
-        if (embedding) {
+    if (embeddings) {
+      for (let index = 0; index < candidates.length; index++) {
+        const { id, observation: obs } = candidates[index];
+        const embedding = embeddings[index];
+        if (!embedding || embedding.length === 0) {
+          recordFailure('embedding provider returned no vector');
+          failed++;
+          continue;
+        }
+        try {
           if (!isVectorCompatibleWithCurrentIndex(embedding)) {
             await upgradeVectorSchemaAfterFirstEmbedding(embedding);
           }
@@ -1239,7 +1288,7 @@ export async function backfillVectorEmbeddings(options: {
             console.error(
               `[memorix] Backfill embedding mismatch for obs-${id}: provider returned ${embedding.length}d, index expects ${vectorDimensions ?? 'unknown'}d (kept in queue)`,
             );
-            lastFailure = `dimension mismatch: provider returned ${embedding.length}d, index expects ${vectorDimensions ?? 'unknown'}d`;
+            recordFailure(`dimension mismatch: provider returned ${embedding.length}d, index expects ${vectorDimensions ?? 'unknown'}d`);
             failed++;
             continue;
           }
@@ -1277,17 +1326,10 @@ export async function backfillVectorEmbeddings(options: {
           await insertObservation(doc);
           vectorMissingIds.delete(id);
           succeeded++;
-        } else if (isEmbeddingExplicitlyDisabled()) {
-          // Embedding explicitly off — nothing to backfill from
-          vectorMissingIds.delete(id);
-        } else {
-          // Provider temporarily unavailable — keep in queue for next backfill cycle
-          lastFailure = 'embedding provider unavailable';
+        } catch (error) {
+          recordFailure(vectorBackfillError(error));
           failed++;
         }
-      } catch (err) {
-        lastFailure = err instanceof Error ? err.message : String(err);
-        failed++;
       }
     }
   } finally {
@@ -1301,5 +1343,10 @@ export async function backfillVectorEmbeddings(options: {
     };
   }
 
-  return { attempted: ids.length, succeeded, failed };
+  return {
+    attempted: ids.length,
+    succeeded,
+    failed,
+    ...(lastFailure ? { lastError: lastFailure } : {}),
+  };
 }

package/src/runtime/maintenance-jobs.ts

@@ -67,7 +67,18 @@ export interface MaintenanceJobSummary {
 
 export type MaintenanceJobRunResult =
   | { action: 'complete' }
-  | { action: 'reschedule'; delayMs: number; resetAttempts?: boolean; payload?: Record<string, unknown> };
+  | {
+    action: 'reschedule';
+    delayMs: number;
+    resetAttempts?: boolean;
+    payload?: Record<string, unknown>;
+    /** Keep recoverable work out of the immediate pending queue during a cooldown. */
+    status?: 'pending' | 'retry';
+    /** Persist a sanitized diagnostic without consuming the job's retry budget. */
+    lastError?: string;
+    /** Clear a stale transient diagnostic after the job makes progress. */
+    clearLastError?: boolean;
+  };
 
 export type MaintenanceJobHandler = (
   job: MaintenanceJob,
@@ -160,6 +171,12 @@ export class MaintenanceJobStore {
       `).get(input.projectId, input.kind, dedupeKey);
 
       if (existing) {
+        // A vector backfill in cooldown is a circuit-breaker state. A new MCP
+        // process must not pull it forward and recreate the original retry storm.
+        if (input.kind === 'vector-backfill' && existing.status === 'retry') {
+          this.commit();
+          return rowToJob(existing);
+        }
         const nextRunAfter = Math.min(Number(existing.run_after), runAfter);
         this.db.prepare(`
           UPDATE maintenance_jobs
@@ -171,6 +188,30 @@ export class MaintenanceJobStore {
         return rowToJob(updated);
       }
 
+      // Older releases exhausted vector jobs on transient provider/index errors.
+      // Reuse one failed record so the next healthy session recovers work instead
+      // of adding another permanent failure row for the same project.
+      if (input.kind === 'vector-backfill') {
+        const failed = this.db.prepare(`
+          SELECT * FROM maintenance_jobs
+          WHERE project_id = ? AND kind = ? AND dedupe_key = ? AND status = 'failed'
+          ORDER BY updated_at DESC
+          LIMIT 1
+        `).get(input.projectId, input.kind, dedupeKey);
+        if (failed) {
+          this.db.prepare(`
+            UPDATE maintenance_jobs
+            SET status = 'pending', attempts = 0, max_attempts = ?, run_after = ?,
+                payload_json = ?, lease_owner = NULL, lease_expires_at = NULL,
+                last_error = NULL, completed_at = NULL, updated_at = ?
+            WHERE id = ?
+          `).run(maxAttempts, runAfter, payloadJson, now, failed.id);
+          const revived = this.getRow(failed.id)!;
+          this.commit();
+          return rowToJob(revived);
+        }
+      }
+
       const id = randomUUID();
       this.db.prepare(`
         INSERT INTO maintenance_jobs (
@@ -361,21 +402,57 @@ export class MaintenanceJobStore {
   reschedule(
     id: string,
     workerId: string,
-    options: { delayMs: number; resetAttempts?: boolean; payload?: Record<string, unknown>; now?: number },
+    options: {
+      delayMs: number;
+      resetAttempts?: boolean;
+      payload?: Record<string, unknown>;
+      status?: 'pending' | 'retry';
+      lastError?: string;
+      clearLastError?: boolean;
+      now?: number;
+    },
   ): MaintenanceJob | undefined {
     const now = options.now ?? Date.now();
     const delayMs = Number.isFinite(options.delayMs) ? Math.max(0, Math.floor(options.delayMs)) : 0;
     const payloadJson = options.payload === undefined ? null : JSON.stringify(options.payload);
+    const status = options.status === 'retry' ? 'retry' : 'pending';
+    const hasLastError = options.lastError !== undefined;
+    const lastError = hasLastError ? errorText(options.lastError) : null;
     this.db.prepare(`
       UPDATE maintenance_jobs
-      SET status = 'pending', run_after = ?, attempts = CASE WHEN ? THEN 0 ELSE attempts END,
+      SET status = ?, run_after = ?, attempts = CASE WHEN ? THEN 0 ELSE attempts END,
           payload_json = COALESCE(?, payload_json),
+          last_error = CASE
+            WHEN ? THEN NULL
+            WHEN ? THEN ?
+            ELSE last_error
+          END,
           lease_owner = NULL, lease_expires_at = NULL, updated_at = ?
       WHERE id = ? AND status = 'running' AND lease_owner = ?
-    `).run(now + delayMs, options.resetAttempts ? 1 : 0, payloadJson, now, id, workerId);
+    `).run(
+      status,
+      now + delayMs,
+      options.resetAttempts ? 1 : 0,
+      payloadJson,
+      options.clearLastError ? 1 : 0,
+      hasLastError ? 1 : 0,
+      lastError,
+      now,
+      id,
+      workerId,
+    );
     return this.get(id);
   }
 
+  resolveFailedVectorBackfills(projectId: string, dedupeKey = 'vector-backfill', now = Date.now()): number {
+    const result = this.db.prepare(`
+      UPDATE maintenance_jobs
+      SET status = 'completed', completed_at = ?, updated_at = ?
+      WHERE project_id = ? AND kind = 'vector-backfill' AND dedupe_key = ? AND status = 'failed'
+    `).run(now, now, projectId, dedupeKey);
+    return Number(result.changes ?? 0);
+  }
+
   fail(id: string, workerId: string, error: unknown, now = Date.now()): MaintenanceJob | undefined {
     this.begin();
     try {
@@ -486,6 +563,9 @@ export class MaintenanceJobWorker {
             delayMs: result.delayMs,
             resetAttempts: result.resetAttempts,
             payload: result.payload,
+            status: result.status,
+            lastError: result.lastError,
+            clearLastError: result.clearLastError,
             now,
           });
           return { state: 'rescheduled', job: updated };

package/src/runtime/project-maintenance.ts

@@ -1,6 +1,7 @@
-import type {
-  MaintenanceJobHandler,
-  MaintenanceJobRunResult,
+import {
+  MaintenanceJobStore,
+  type MaintenanceJobHandler,
+  type MaintenanceJobRunResult,
 } from './maintenance-jobs.js';
 import { runMaintenanceInChildProcess } from './isolated-maintenance.js';
 import {
@@ -18,6 +19,8 @@ const DEFAULT_CODEGRAPH_MAX_FILES = 5_000;
 const DEFAULT_CLAIM_DERIVATION_BATCH_SIZE = 100;
 const DEFAULT_OBSERVATION_QUALIFICATION_BATCH_SIZE = 100;
 const VECTOR_RETRY_DELAY_MS = 5_000;
+const VECTOR_FAILURE_BASE_RETRY_DELAY_MS = 60_000;
+const VECTOR_FAILURE_MAX_RETRY_DELAY_MS = 15 * 60_000;
 const DEDUP_PER_PAIR_TIMEOUT_MS = 5_000;
 
 function vectorBatchSize(payload: Record<string, unknown>): number {
@@ -26,6 +29,33 @@ function vectorBatchSize(payload: Record<string, unknown>): number {
   return Math.min(100, Math.max(1, Math.floor(value)));
 }
 
+function vectorBackfillFailureStreak(payload: Record<string, unknown>): number {
+  const value = payload.vectorBackfillFailureStreak;
+  return typeof value === 'number' && Number.isFinite(value)
+    ? Math.min(16, Math.max(0, Math.floor(value)))
+    : 0;
+}
+
+function vectorBackfillRetryDelayMs(streak: number): number {
+  return Math.min(
+    VECTOR_FAILURE_MAX_RETRY_DELAY_MS,
+    VECTOR_FAILURE_BASE_RETRY_DELAY_MS * (2 ** Math.max(0, streak - 1)),
+  );
+}
+
+function withoutVectorBackfillFailureStreak(payload: Record<string, unknown>): Record<string, unknown> {
+  const { vectorBackfillFailureStreak: _streak, ...rest } = payload;
+  return rest;
+}
+
+function resolveSupersededVectorFailures(projectDir: string, projectId: string): void {
+  try {
+    new MaintenanceJobStore(projectDir).resolveFailedVectorBackfills(projectId);
+  } catch {
+    // Health cleanup must never block the live memory path.
+  }
+}
+
 function retentionBatchSize(payload: Record<string, unknown>): number {
   const value = payload.limit;
   if (typeof value !== 'number' || !Number.isFinite(value)) return DEFAULT_RETENTION_BATCH_SIZE;
@@ -498,23 +528,50 @@ export function createProjectMaintenanceHandler(
 
     const { backfillVectorEmbeddings, getVectorStatus } = await import('../memory/observations.js');
     const before = getVectorStatus(projectId);
-    if (before.missing === 0) return { action: 'complete' };
+    if (before.missing === 0) {
+      resolveSupersededVectorFailures(projectDir, projectId);
+      return { action: 'complete' };
+    }
 
     const result = await backfillVectorEmbeddings({
       projectId,
       limit: vectorBatchSize(job.payload),
     });
     const after = getVectorStatus(projectId);
-    if (after.missing === 0) return { action: 'complete' };
+    if (after.missing === 0) {
+      resolveSupersededVectorFailures(projectDir, projectId);
+      return { action: 'complete' };
+    }
 
     if (result.failed > 0 && result.succeeded === 0) {
-      throw new Error(`vector backfill made no progress (${result.failed}/${result.attempted} failed)`);
+      const streak = vectorBackfillFailureStreak(job.payload) + 1;
+      return {
+        action: 'reschedule',
+        status: 'retry',
+        delayMs: vectorBackfillRetryDelayMs(streak),
+        // This is a recoverable provider/index state, not a terminal job error.
+        // Keeping one retry row prevents new MCP processes from creating a storm.
+        resetAttempts: true,
+        payload: {
+          ...job.payload,
+          vectorBackfillFailureStreak: streak,
+        },
+        lastError: result.lastError ?? `vector backfill made no progress (${result.failed}/${result.attempted} failed)`,
+      };
     }
 
+    const priorFailureStreak = vectorBackfillFailureStreak(job.payload);
+    const payload = withoutVectorBackfillFailureStreak(job.payload);
     return {
       action: 'reschedule',
       delayMs: result.attempted === 0 ? VECTOR_RETRY_DELAY_MS : 0,
       resetAttempts: result.succeeded > 0,
+      ...(result.succeeded > 0 ? {
+        ...(priorFailureStreak > 0 ? { payload } : {}),
+        ...(result.failed > 0 && result.lastError
+          ? { lastError: result.lastError }
+          : priorFailureStreak > 0 ? { clearLastError: true } : {}),
+      } : {}),
     };
   };
 }

package/src/server/tool-profile.ts

@@ -68,6 +68,7 @@ export const TOOL_PROFILES: Record<string, ReadonlyArray<ToolProfile>> = Object.
   memorix_rules_sync:         ['full'],
   memorix_workspace_sync:     ['full'],
   memorix_ingest_image:       ['full'],
+  memorix_compaction_checkpoint: ['full'],
 
   // ── MCP Official Memory Server compatibility (KG tools) ──────────
   // These are only useful to users specifically migrating from the

package/src/server.ts

@@ -3974,6 +3974,100 @@ export async function createMemorixServer(
     },
   );
 
+  /**
+   * memorix_compaction_checkpoint — Advanced inspection of native host
+   * compaction checkpoints. Automatic recovery does not need this tool; it is
+   * deliberately full-profile only so normal MCP startup stays compact.
+   */
+  server.registerTool(
+    'memorix_compaction_checkpoint',
+    {
+      title: 'Compact Continuity Checkpoints',
+      description:
+        'Inspect, preview, or archive bounded checkpoints recorded around a host-native context compaction. ' +
+        'Use only to debug or explicitly review compaction continuity. These are lifecycle records, not durable memories or transcript backups. ' +
+        'CLI equivalent: memorix checkpoint list|show|context|archive.',
+      inputSchema: {
+        action: z.enum(['list', 'show', 'context', 'archive']).default('list'),
+        id: z.string().optional().describe('Checkpoint ID for show, context, or archive'),
+        sessionId: z.string().optional().describe('Optional host session ID filter'),
+        agent: z.string().optional().describe('Optional host agent filter'),
+        task: z.string().optional().describe('Current task for a bounded context preview'),
+        maxTokens: z.number().optional().describe('Maximum tokens for action=context (default: 420)'),
+        limit: z.number().optional().describe('Maximum records for action=list (default: 20)'),
+        includeArchived: z.boolean().optional().default(false).describe('Include archived records in action=list'),
+      },
+    },
+    async ({ action, id, sessionId, agent, task, maxTokens, limit, includeArchived }) => {
+      const unresolved = requireResolvedProject('inspect compact continuity checkpoints');
+      if (unresolved) return unresolved;
+
+      const [{ CompactionCheckpointStore }, { buildCompactionWorkset }] = await Promise.all([
+        import('./store/compaction-checkpoint-store.js'),
+        import('./memory/compaction.js'),
+      ]);
+      const store = new CompactionCheckpointStore(projectDir);
+      const assertCheckpoint = (checkpointId: string) => {
+        const checkpoint = store.get(checkpointId);
+        if (!checkpoint || checkpoint.projectId !== project.id) return null;
+        return checkpoint;
+      };
+
+      if (action === 'list') {
+        const checkpoints = store.list({
+          projectId: project.id,
+          sessionId: sessionId?.trim() || undefined,
+          agent: agent?.trim() || undefined,
+          includeArchived: Boolean(includeArchived),
+          limit: limit != null ? Math.max(1, Math.min(100, coerceNumber(limit, 20))) : 20,
+        });
+        return {
+          content: [{ type: 'text' as const, text: JSON.stringify({ projectId: project.id, checkpoints }, null, 2) }],
+        };
+      }
+
+      if (!id?.trim()) {
+        return {
+          content: [{ type: 'text' as const, text: 'id is required for this checkpoint action.' }],
+          isError: true,
+        };
+      }
+      const checkpoint = assertCheckpoint(id.trim());
+      if (!checkpoint) {
+        return {
+          content: [{ type: 'text' as const, text: `Checkpoint "${id.trim()}" was not found for the current project.` }],
+          isError: true,
+        };
+      }
+
+      if (action === 'show') {
+        return {
+          content: [{ type: 'text' as const, text: JSON.stringify({ projectId: project.id, checkpoint }, null, 2) }],
+        };
+      }
+      if (action === 'context') {
+        const workset = buildCompactionWorkset(checkpoint, {
+          task: task?.trim(),
+          maxTokens: maxTokens != null ? coerceNumber(maxTokens, 420) : 420,
+        });
+        return {
+          content: [{ type: 'text' as const, text: workset.text }],
+        };
+      }
+
+      const archived = store.archive(checkpoint.id);
+      if (!archived) {
+        return {
+          content: [{ type: 'text' as const, text: `Checkpoint "${checkpoint.id}" is already archived.` }],
+          isError: true,
+        };
+      }
+      return {
+        content: [{ type: 'text' as const, text: `Archived compact checkpoint ${archived.id}.` }],
+      };
+    },
+  );
+
   // ============================================================
   // Export / Import
   // ============================================================