memorix 1.2.7 → 1.2.9

package/src/dashboard/server.ts

@@ -23,6 +23,7 @@ import { resetDotenv } from '../config/dotenv-loader.js';
 import { initProjectRoot } from '../config/yaml-loader.js';
 import { clearProjectRoot } from '../config/yaml-loader.js';
 import { scopeKnowledgeGraphToProject } from '../memory/graph-scope.js';
+import { projectObservationRetention, summarizeRetentionProjections } from '../memory/retention.js';
 import { canManageObservation, filterReadableObservations } from '../memory/visibility.js';
 import type { Observation } from '../types.js';
 
@@ -249,7 +250,7 @@ async function handleApi(
                 const observations = filterDashboardObservations(
                     await getObservationStore().loadByProject(effectiveProjectId, { status: 'active' }),
                     effectiveProjectId,
-                ) as Array<{ type?: string; id?: number; createdAt?: string; title?: string; entityName?: string; relatedEntities?: string[]; status?: string }>;
+                );
                 const nextId = await getObservationStore().loadIdCounter();
 
                 // Project-scoped graph counts (must match /api/graph and /api/export)
@@ -290,21 +291,18 @@ async function handleApi(
                     filesModified: (o as any).filesModified,
                 }));
 
-                // Retention summary
-                let retentionSummary = { active: 0, stale: 0, archive: 0, immune: 0 };
-                for (const obs of observations) {
-                    const age = now - new Date((obs as any).createdAt || now).getTime();
-                    const ageHours = age / (1000 * 60 * 60);
-                    const importance = (obs as any).importance ?? 5;
-                    const accessCount = (obs as any).accessCount ?? 0;
-                    const lambda = 0.01;
-                    const score = Math.min(importance * Math.exp(-lambda * ageHours) + Math.min(accessCount * 0.5, 3), 10);
-                    const isImmune = importance >= 8 || obs.type === 'gotcha' || obs.type === 'decision';
-                    if (isImmune) retentionSummary.immune++;
-                    if (score >= 3) retentionSummary.active++;
-                    else if (score >= 1) retentionSummary.stale++;
-                    else retentionSummary.archive++;
-                }
+                const retentionReferenceTime = new Date(now);
+                const retention = summarizeRetentionProjections(
+                    observations
+                        .filter((observation) => observation.type !== 'probe')
+                        .map((observation) => projectObservationRetention(observation, { referenceTime: retentionReferenceTime })),
+                );
+                const retentionSummary = {
+                    active: retention.active,
+                    stale: retention.stale,
+                    archive: retention.archiveCandidates,
+                    immune: retention.immune,
+                };
 
                 // Recent observations (last 10, exclude probe)
                 const sorted = [...observations].filter(o => o.type !== 'probe')
@@ -351,6 +349,17 @@ async function handleApi(
                     sourceCounts,
                     recentObservations: sorted,
                     embedding: embeddingStatus,
+                    // A standalone dashboard has no access to an active MCP
+                    // process's in-memory Orama index. Keep this explicit so the
+                    // UI never presents an empty local singleton as a healthy
+                    // all-vectors-indexed result.
+                    vectorStatus: {
+                        available: false,
+                        total: 0,
+                        missing: 0,
+                        missingIds: [],
+                        backfillRunning: false,
+                    },
                     storage: storageInfo,
                     maintenance,
                     ...(lifecycle ? { lifecycle } : {}),
@@ -388,56 +397,38 @@ async function handleApi(
                 const observations = filterDashboardObservations(
                     await getObservationStore().loadByProject(effectiveProjectId, { status: 'active' }),
                     effectiveProjectId,
-                ) as Array<{
-                    id?: number;
-                    title?: string;
-                    type?: string;
-                    importance?: number;
-                    accessCount?: number;
-                    lastAccessedAt?: string;
-                    createdAt?: string;
-                    entityName?: string;
-                }>;
-
-                const now = Date.now();
-                // Exclude probe from retention display -- not durable knowledge
-                const scored = observations.filter(obs => obs.type !== 'probe').map((obs) => {
-                    const age = now - new Date(obs.createdAt || now).getTime();
-                    const ageHours = age / (1000 * 60 * 60);
-                    const importance = obs.importance ?? 5;
-                    const accessCount = obs.accessCount ?? 0;
-
-                    // Exponential decay: score = importance * e^(-λt) + access_bonus
-                    const lambda = 0.01;
-                    const decayScore = importance * Math.exp(-lambda * ageHours);
-                    const accessBonus = Math.min(accessCount * 0.5, 3);
-                    const score = Math.min(decayScore + accessBonus, 10);
-
-                    // Immune if importance >= 8 or type is 'gotcha' or 'decision'
-                    const isImmune = importance >= 8 || obs.type === 'gotcha' || obs.type === 'decision';
-
-                    return {
-                        id: obs.id,
-                        title: obs.title,
-                        type: obs.type,
-                        entityName: obs.entityName,
-                        score: Math.round(score * 100) / 100,
-                        isImmune,
-                        ageHours: Math.round(ageHours * 10) / 10,
-                        accessCount,
-                    };
-                });
-
-                // Sort by score descending
-                scored.sort((a, b) => b.score - a.score);
+                );
 
-                const activeCount = scored.filter((s) => s.score >= 3).length;
-                const staleCount = scored.filter((s) => s.score < 3 && s.score >= 1).length;
-                const archiveCount = scored.filter((s) => s.score < 1).length;
-                const immuneCount = scored.filter((s) => s.isImmune).length;
+                const referenceTime = new Date();
+                const rows = observations
+                    .filter((observation) => observation.type !== 'probe')
+                    .map((observation) => ({
+                        observation,
+                        retention: projectObservationRetention(observation, { referenceTime }),
+                    }))
+                    .sort((a, b) => b.retention.displayScore - a.retention.displayScore);
+                const summary = summarizeRetentionProjections(rows.map((row) => row.retention));
+                const scored = rows.map(({ observation, retention }) => ({
+                    id: observation.id,
+                    title: observation.title,
+                    type: observation.type,
+                    entityName: observation.entityName,
+                    score: retention.displayScore,
+                    isImmune: retention.immune,
+                    zone: retention.zone,
+                    ageHours: retention.ageHours,
+                    accessCount: retention.accessCount,
+                    effectiveRetentionDays: retention.effectiveRetentionDays,
+                    immunityReason: retention.immunityReason,
+                }));
 
                 sendJson(res, {
-                    summary: { active: activeCount, stale: staleCount, archive: archiveCount, immune: immuneCount },
+                    summary: {
+                        active: summary.active,
+                        stale: summary.stale,
+                        archive: summary.archiveCandidates,
+                        immune: summary.immune,
+                    },
                     items: scored,
                 });
                 break;

package/src/memory/observations.ts

@@ -49,6 +49,17 @@ let observations: Observation[] = [];
 let nextId = 1;
 let projectDir: string | null = null;
 let searchIndexPrepared = false;
+export type ObservationEmbeddingWriteMode = 'background' | 'deferred';
+
+export interface ObservationRuntimeOptions {
+  /** Defer remote/local embedding to a detached worker for short-lived CLI writes. */
+  embeddingWriteMode?: ObservationEmbeddingWriteMode;
+  /** Git root used by the detached vector worker. */
+  projectRoot?: string;
+}
+
+let embeddingWriteMode: ObservationEmbeddingWriteMode = 'background';
+let embeddingWorkerProjectRoot: string | undefined;
 
 // ── Vector-missing tracking ──────────────────────────────────────
 // Tracks observation IDs whose async embedding write failed or was skipped.
@@ -102,7 +113,14 @@ function normalizeEmbeddingFailure(error: unknown): { key: string; message: stri
   };
 }
 
-function queueVectorBackfill(projectId: string): void {
+function vectorBackfillError(error: unknown): string {
+  return sanitizeCredentials(normalizeEmbeddingFailure(error).message).slice(0, 1_000);
+}
+
+function queueVectorBackfill(
+  projectId: string,
+  options: { detachedWorker?: boolean; projectRoot?: string } = {},
+): void {
   const dataDir = projectDir;
   if (!dataDir) return;
   void import('../runtime/maintenance-jobs.js')
@@ -113,6 +131,19 @@ function queueVectorBackfill(projectId: string): void {
         dedupeKey: 'vector-backfill',
         payload: { limit: 12 },
       });
+      if (options.detachedWorker && options.projectRoot) {
+        void import('../runtime/vector-backfill-runner.js')
+          .then(({ launchDetachedVectorBackfill }) => {
+            launchDetachedVectorBackfill({
+              projectId,
+              projectRoot: options.projectRoot!,
+              dataDir,
+            });
+          })
+          .catch(() => {
+            // The durable queue remains available to a later MCP session.
+          });
+      }
     })
     .catch(() => {
       // Memory writes remain durable even when the optional maintenance queue is unavailable.
@@ -195,7 +226,12 @@ async function upgradeVectorSchemaAfterFirstEmbedding(embedding: number[]): Prom
  * Initialize the observations manager with a project directory.
  * Auto-initializes the ObservationStore if not already set.
  */
-export async function initObservations(dir: string): Promise<void> {
+export async function initObservations(
+  dir: string,
+  options: ObservationRuntimeOptions = {},
+): Promise<void> {
+  embeddingWriteMode = options.embeddingWriteMode ?? 'background';
+  embeddingWorkerProjectRoot = options.projectRoot;
   if (projectDir === dir) return;
   await initObservationStore(dir);
   const store = getObservationStore();
@@ -206,6 +242,70 @@ export async function initObservations(dir: string): Promise<void> {
   vectorSchemaUpgradePromise = null;
 }
 
+function scheduleObservationEmbedding(input: {
+  observationId: number;
+  projectId: string;
+  searchableText: string;
+  doc: MemorixDocument;
+}): void {
+  const { observationId, projectId, searchableText, doc } = input;
+  vectorMissingIds.add(observationId);
+
+  // A network fetch keeps Node's event loop alive even when its Promise is not
+  // awaited. One-shot CLI commands therefore persist first and let a detached
+  // worker consume the same durable queue.
+  if (embeddingWriteMode === 'deferred') {
+    if (isEmbeddingExplicitlyDisabled()) {
+      vectorMissingIds.delete(observationId);
+      return;
+    }
+    queueVectorBackfill(projectId, {
+      detachedWorker: true,
+      projectRoot: embeddingWorkerProjectRoot,
+    });
+    return;
+  }
+
+  void generateEmbedding(searchableText).then(async (embedding) => {
+    if (embedding) {
+      if (!isVectorCompatibleWithCurrentIndex(embedding)) {
+        await upgradeVectorSchemaAfterFirstEmbedding(embedding);
+      }
+      if (!isVectorCompatibleWithCurrentIndex(embedding)) {
+        const vectorDimensions = getVectorDimensions();
+        console.error(
+          `[memorix] Embedding dimension mismatch for obs-${observationId}: provider returned ${embedding.length}d, index expects ${vectorDimensions ?? 'unknown'}d (kept in backfill queue)`,
+        );
+        queueVectorBackfill(projectId);
+        return;
+      }
+      try {
+        await removeObservation(makeOramaObservationId(projectId, observationId));
+        await insertObservation(Object.assign({}, doc, { embedding }));
+        vectorMissingIds.delete(observationId);
+      } catch {
+        console.error(`[memorix] Embedding index update failed for obs-${observationId} (kept in backfill queue)`);
+        queueVectorBackfill(projectId);
+      }
+    } else if (isEmbeddingExplicitlyDisabled()) {
+      vectorMissingIds.delete(observationId);
+    } else {
+      queueVectorBackfill(projectId);
+      logEmbeddingFailureOnce(
+        'provider-unavailable',
+        `[memorix] Embedding provider unavailable (using BM25 until embedding recovers; queued obs-${observationId} for retry)`,
+      );
+    }
+  }).catch((err) => {
+    queueVectorBackfill(projectId);
+    const failure = normalizeEmbeddingFailure(err);
+    logEmbeddingFailureOnce(
+      failure.key,
+      `[memorix] Async embedding failed (using BM25 until embedding recovers; queued obs-${observationId} for retry): ${failure.message}`,
+    );
+  });
+}
+
 /**
  * Check cross-process freshness and reload if another process has written.
  *
@@ -503,49 +603,11 @@ export async function storeObservation(input: {
     queueClaimDerivation(observation);
   }
 
-  // Generate embedding async (fire-and-forget) — never blocks MCP response
-  // Track in vectorMissingIds until embedding is successfully written.
-  const obsId = observation.id;
-  vectorMissingIds.add(obsId);
-  const searchableText = [input.title, input.narrative, ...(input.facts ?? [])].join(' ');
-  generateEmbedding(searchableText).then(async (embedding) => {
-    if (embedding) {
-      if (!isVectorCompatibleWithCurrentIndex(embedding)) {
-        await upgradeVectorSchemaAfterFirstEmbedding(embedding);
-      }
-      if (!isVectorCompatibleWithCurrentIndex(embedding)) {
-        const vectorDimensions = getVectorDimensions();
-        console.error(
-          `[memorix] Embedding dimension mismatch for obs-${obsId}: provider returned ${embedding.length}d, index expects ${vectorDimensions ?? 'unknown'}d (kept in backfill queue)`,
-        );
-        queueVectorBackfill(input.projectId);
-        return;
-      }
-      try {
-        const { removeObservation: removeObs } = await import('../store/orama-store.js');
-        await removeObs(makeOramaObservationId(input.projectId, obsId));
-        await insertObservation(Object.assign({}, doc, { embedding }));
-        vectorMissingIds.delete(obsId);
-      } catch {
-        console.error(`[memorix] Embedding index update failed for obs-${obsId} (kept in backfill queue)`);
-        queueVectorBackfill(input.projectId);
-      }
-    } else if (isEmbeddingExplicitlyDisabled()) {
-      vectorMissingIds.delete(obsId);
-    } else {
-      queueVectorBackfill(input.projectId);
-      logEmbeddingFailureOnce(
-        'provider-unavailable',
-        `[memorix] Embedding provider unavailable (using BM25 until embedding recovers; queued obs-${obsId} for retry)`,
-      );
-    }
-  }).catch((err) => {
-    queueVectorBackfill(input.projectId);
-    const failure = normalizeEmbeddingFailure(err);
-    logEmbeddingFailureOnce(
-      failure.key,
-      `[memorix] Async embedding failed (using BM25 until embedding recovers; queued obs-${obsId} for retry): ${failure.message}`,
-    );
+  scheduleObservationEmbedding({
+    observationId: observation.id,
+    projectId: input.projectId,
+    searchableText: [input.title, input.narrative, ...(input.facts ?? [])].join(' '),
+    doc,
   });
 
   return { observation, upserted: false };
@@ -674,40 +736,11 @@ async function upsertObservation(
     queueClaimDerivation(existing);
   }
 
-  // Generate embedding async (fire-and-forget) — never blocks MCP response
-  const searchableText = [input.title, input.narrative, ...(input.facts ?? [])].join(' ');
-  const obsId = existing.id;
-  vectorMissingIds.add(obsId);
-  generateEmbedding(searchableText).then(async (embedding) => {
-    if (embedding) {
-      if (!isVectorCompatibleWithCurrentIndex(embedding)) {
-        await upgradeVectorSchemaAfterFirstEmbedding(embedding);
-      }
-      if (!isVectorCompatibleWithCurrentIndex(embedding)) {
-        queueVectorBackfill(existing.projectId);
-        return;
-      }
-      try {
-        const { removeObservation: removeObs } = await import('../store/orama-store.js');
-        await removeObs(makeOramaObservationId(existing.projectId, obsId));
-        await insertObservation(Object.assign({}, doc, { embedding }));
-        vectorMissingIds.delete(obsId);
-      } catch {
-        // Embedding index update failed — observation still persisted without vector
-        queueVectorBackfill(existing.projectId);
-      }
-    } else if (isEmbeddingExplicitlyDisabled()) {
-      vectorMissingIds.delete(obsId);
-    } else {
-      queueVectorBackfill(existing.projectId);
-    }
-  }).catch((err) => {
-    queueVectorBackfill(existing.projectId);
-    const failure = normalizeEmbeddingFailure(err);
-    logEmbeddingFailureOnce(
-      failure.key,
-      `[memorix] Async embedding failed (using BM25 until embedding recovers; queued obs-${obsId} for retry): ${failure.message}`,
-    );
+  scheduleObservationEmbedding({
+    observationId: existing.id,
+    projectId: existing.projectId,
+    searchableText: [input.title, input.narrative, ...(input.facts ?? [])].join(' '),
+    doc,
   });
 
   return existing;
@@ -1202,6 +1235,7 @@ export async function backfillVectorEmbeddings(options: {
   attempted: number;
   succeeded: number;
   failed: number;
+  lastError?: string;
 }> {
   if (vectorBackfillRunning) {
     return { attempted: 0, succeeded: 0, failed: 0 };
@@ -1218,19 +1252,63 @@ export async function backfillVectorEmbeddings(options: {
   let succeeded = 0;
   let failed = 0;
   let lastFailure: string | undefined;
+  const recordFailure = (message: string): void => {
+    // A single batch may have several symptoms after one root cause. Preserve
+    // the first one so an informative provider/index error is not overwritten
+    // by later missing-result fallout.
+    if (!lastFailure) lastFailure = message;
+  };
+
+  const candidates: Array<{ id: number; observation: Observation; text: string }> = [];
+  for (const id of ids) {
+    const observation = observationById.get(id);
+    if (!observation) {
+      vectorMissingIds.delete(id);
+      continue;
+    }
+    candidates.push({
+      id,
+      observation,
+      text: [observation.title, observation.narrative, ...observation.facts].join(' '),
+    });
+  }
 
   try {
-    for (const id of ids) {
-      const obs = observationById.get(id);
-      if (!obs) {
-        vectorMissingIds.delete(id);
-        continue;
+    if (candidates.length === 0) {
+      return { attempted: ids.length, succeeded, failed };
+    }
+
+    let embeddings: number[][] | null = null;
+    try {
+      const provider = await getEmbeddingProvider();
+      if (!provider) {
+        if (isEmbeddingExplicitlyDisabled()) {
+          for (const candidate of candidates) vectorMissingIds.delete(candidate.id);
+        } else {
+          recordFailure('embedding provider unavailable');
+          failed += candidates.length;
+        }
+      } else {
+        // API providers batch and cache this efficiently; local providers can use
+        // their native batch path. Index writes stay sequential because Orama is
+        // process-local mutable state.
+        embeddings = await provider.embedBatch(candidates.map((candidate) => candidate.text));
       }
+    } catch (error) {
+      recordFailure(vectorBackfillError(error));
+      failed += candidates.length;
+    }
 
-      const text = [obs.title, obs.narrative, ...obs.facts].join(' ');
-      try {
-        const embedding = await generateEmbedding(text);
-        if (embedding) {
+    if (embeddings) {
+      for (let index = 0; index < candidates.length; index++) {
+        const { id, observation: obs } = candidates[index];
+        const embedding = embeddings[index];
+        if (!embedding || embedding.length === 0) {
+          recordFailure('embedding provider returned no vector');
+          failed++;
+          continue;
+        }
+        try {
           if (!isVectorCompatibleWithCurrentIndex(embedding)) {
             await upgradeVectorSchemaAfterFirstEmbedding(embedding);
           }
@@ -1239,7 +1317,7 @@ export async function backfillVectorEmbeddings(options: {
             console.error(
               `[memorix] Backfill embedding mismatch for obs-${id}: provider returned ${embedding.length}d, index expects ${vectorDimensions ?? 'unknown'}d (kept in queue)`,
             );
-            lastFailure = `dimension mismatch: provider returned ${embedding.length}d, index expects ${vectorDimensions ?? 'unknown'}d`;
+            recordFailure(`dimension mismatch: provider returned ${embedding.length}d, index expects ${vectorDimensions ?? 'unknown'}d`);
             failed++;
             continue;
           }
@@ -1277,17 +1355,10 @@ export async function backfillVectorEmbeddings(options: {
           await insertObservation(doc);
           vectorMissingIds.delete(id);
           succeeded++;
-        } else if (isEmbeddingExplicitlyDisabled()) {
-          // Embedding explicitly off — nothing to backfill from
-          vectorMissingIds.delete(id);
-        } else {
-          // Provider temporarily unavailable — keep in queue for next backfill cycle
-          lastFailure = 'embedding provider unavailable';
+        } catch (error) {
+          recordFailure(vectorBackfillError(error));
           failed++;
         }
-      } catch (err) {
-        lastFailure = err instanceof Error ? err.message : String(err);
-        failed++;
       }
     }
   } finally {
@@ -1301,5 +1372,10 @@ export async function backfillVectorEmbeddings(options: {
     };
   }
 
-  return { attempted: ids.length, succeeded, failed };
+  return {
+    attempted: ids.length,
+    succeeded,
+    failed,
+    ...(lastFailure ? { lastError: lastFailure } : {}),
+  };
 }