memorix 1.2.7 → 1.2.9

package/src/memory/retention.ts

@@ -319,6 +319,40 @@ export interface RetentionExplanation {
   summary: string;
 }
 
+/** Access metadata maintained outside the persisted observation record. */
+export type ObservationAccessMap = ReadonlyMap<number, {
+  accessCount: number;
+  lastAccessedAt: string;
+}>;
+
+/**
+ * One canonical retention result for a persisted observation.
+ *
+ * Consumers should use this projection instead of recreating a local decay
+ * formula. That keeps the UI, HTTP control plane, and archive worker aligned
+ * on the same retention state.
+ */
+export interface RetentionProjection extends RetentionExplanation {
+  relevance: RelevanceScore;
+  /** A backwards-compatible 0-10 display value derived from relevance. */
+  displayScore: number;
+  ageHours: number;
+  accessCount: number;
+  lastAccessedAt: string;
+}
+
+export interface ObservationRetentionProjectionOptions {
+  referenceTime?: Date;
+  accessMap?: ObservationAccessMap;
+}
+
+export interface RetentionProjectionSummary {
+  active: number;
+  stale: number;
+  archiveCandidates: number;
+  immune: number;
+}
+
 /**
  * Produce a structured explanation of why an observation has its current
  * retention posture.  Designed for operator-facing reporting.
@@ -371,27 +405,16 @@ export function explainRetention(
   };
 }
 
-// ── Auto-Archive ────────────────────────────────────────────────────
-
-export interface ArchiveExpiredBatchOptions {
-  projectId: string;
-  afterId?: number;
-  limit?: number;
-  referenceTime?: Date;
-  accessMap?: Map<number, { accessCount: number; lastAccessedAt: string }>;
-  /** Omit only for trusted background maintenance. */
-  reader?: ObservationReader;
-}
-
-export interface ArchiveExpiredBatchResult {
-  archived: number;
-  scanned: number;
-  nextCursor?: number;
-}
-
-function toRetentionDocument(
+/**
+ * Convert a stored observation into the search/retention document shape.
+ *
+ * Observation fields such as concepts are arrays, while the retrieval schema
+ * intentionally stores them as strings. Centralising that conversion prevents
+ * callers from accidentally feeding raw observations into retention helpers.
+ */
+export function toRetentionDocument(
   obs: Observation,
-  accessMap?: Map<number, { accessCount: number; lastAccessedAt: string }>,
+  accessMap?: ObservationAccessMap,
 ): MemorixDocument {
   const access = accessMap?.get(obs.id);
   return {
@@ -418,6 +441,62 @@ function toRetentionDocument(
   };
 }
 
+/** Build the canonical retention result used by every Observation consumer. */
+export function projectObservationRetention(
+  observation: Observation,
+  options: ObservationRetentionProjectionOptions = {},
+): RetentionProjection {
+  const document = toRetentionDocument(observation, options.accessMap);
+  const relevance = calculateRelevance(document, options.referenceTime);
+  const explanation = explainRetention(document, options.referenceTime);
+
+  return {
+    ...explanation,
+    relevance,
+    displayScore: Math.round(Math.min(10, relevance.totalScore * 10) * 100) / 100,
+    ageHours: Math.round(relevance.ageDays * 24 * 10) / 10,
+    accessCount: document.accessCount,
+    lastAccessedAt: document.lastAccessedAt,
+  };
+}
+
+/** Summarise canonical projections without reimplementing retention thresholds. */
+export function summarizeRetentionProjections(
+  projections: readonly Pick<RetentionProjection, 'zone' | 'immune'>[],
+): RetentionProjectionSummary {
+  let active = 0;
+  let stale = 0;
+  let archiveCandidates = 0;
+  let immune = 0;
+
+  for (const projection of projections) {
+    if (projection.zone === 'active') active++;
+    else if (projection.zone === 'stale') stale++;
+    else archiveCandidates++;
+    if (projection.immune) immune++;
+  }
+
+  return { active, stale, archiveCandidates, immune };
+}
+
+// ── Auto-Archive ────────────────────────────────────────────────────
+
+export interface ArchiveExpiredBatchOptions {
+  projectId: string;
+  afterId?: number;
+  limit?: number;
+  referenceTime?: Date;
+  accessMap?: ObservationAccessMap;
+  /** Omit only for trusted background maintenance. */
+  reader?: ObservationReader;
+}
+
+export interface ArchiveExpiredBatchResult {
+  archived: number;
+  scanned: number;
+  nextCursor?: number;
+}
+
 /**
  * Archive one bounded page of a project's active memories. The caller owns the
  * cursor, which makes the operation safe to run from a durable maintenance job.
@@ -438,10 +517,10 @@ export async function archiveExpiredBatch(
   const scanned = hasMore ? page.slice(0, limit) : page;
   const candidateIds = scanned
     .filter((observation) => !options.reader || canManageObservation(observation, options.reader))
-    .filter((observation) => getRetentionZone(
-      toRetentionDocument(observation, options.accessMap),
-      options.referenceTime,
-    ) === 'archive-candidate')
+    .filter((observation) => projectObservationRetention(observation, {
+      accessMap: options.accessMap,
+      referenceTime: options.referenceTime,
+    }).zone === 'archive-candidate')
     .map((observation) => observation.id);
 
   const archived = candidateIds.length === 0
@@ -474,7 +553,7 @@ export async function archiveExpiredBatch(
 export async function archiveExpired(
   projectDir: string,
   referenceTime?: Date,
-  accessMap?: Map<number, { accessCount: number; lastAccessedAt: string }>,
+  accessMap?: ObservationAccessMap,
   projectId?: string,
   reader?: ObservationReader,
 ): Promise<{ archived: number; remaining: number }> {
@@ -512,9 +591,8 @@ export async function archiveExpired(
     const archivedIds: number[] = [];
 
     for (const obs of activeObs) {
-      const doc = toRetentionDocument(obs, accessMap);
-      const zone = getRetentionZone(doc, referenceTime);
-      if (zone === 'archive-candidate') {
+      const retention = projectObservationRetention(obs, { accessMap, referenceTime });
+      if (retention.zone === 'archive-candidate') {
         archivedIds.push(obs.id);
       }
     }

package/src/memory/session.ts

@@ -21,7 +21,18 @@ import { KnowledgeGraphManager } from './graph.js';
 import { redactCredentials, sanitizeCredentials } from './secret-filter.js';
 import { canReadObservation } from './visibility.js';
 
-const PRIORITY_TYPES = new Set(['gotcha', 'decision', 'problem-solution', 'trade-off', 'discovery']);
+// Types eligible for L2 "Key Project Memories" injection. Session requests carry
+// the original goal and change records carry the latest outcome, so both matter
+// when an agent resumes work after a handoff.
+const PRIORITY_TYPES = new Set([
+  'gotcha',
+  'decision',
+  'problem-solution',
+  'trade-off',
+  'discovery',
+  'session-request',
+  'what-changed',
+]);
 const RESUME_TYPES = new Set([
   'gotcha',
   'decision',
@@ -31,7 +42,6 @@ const RESUME_TYPES = new Set([
   'how-it-works',
   'what-changed',
   'reasoning',
-  'session-request',
 ]);
 const TYPE_EMOJI: Record<string, string> = {
   'gotcha': '[DISCOVERY]',
@@ -50,19 +60,24 @@ const TYPE_WEIGHTS: Record<string, number> = {
   'decision': 5.5,
   'problem-solution': 5.25,
   'trade-off': 4.75,
+  'session-request': 4.5,
   'discovery': 4.25,
+  'what-changed': 4,
 };
+// Markers that identify a memory as a demo/scratch artifact rather than real
+// working context. These are TITLE conventions — see isNoiseObservation.
+//
+// Deliberately excluded: 验证 / 兼容 / compat / 交接 / handoff. Those are ordinary
+// engineering vocabulary, not noise signals. Matching them silently dropped real
+// memories: any Chinese note about verification or compatibility, and any handoff
+// note that merely named a handoff tool, disappeared from session context with no
+// diagnostic. A noise filter must not veto the words its users normally write.
 const NOISE_PATTERNS = [
   /\[测试\]/i,
   /\[test\]/i,
-  /验证/i,
-  /兼容/i,
-  /\bcompat(?:ibility)?\b/i,
   /\bdemo\b/i,
   /展示/i,
   /全能力/i,
-  /handoff/i,
-  /交接/i,
   /for_memmcp_test/i,
   /\bbenchmark\b/i,
   /\bsandbox\b/i,
@@ -180,8 +195,13 @@ function isCommandTrace(obs: Observation): boolean {
 }
 
 function isNoiseObservation(obs: Observation): boolean {
-  const text = stringifyObservation(obs, false);
-  return NOISE_PATTERNS.some((pattern) => pattern.test(text)) || isCommandTrace(obs);
+  // Match the title only. Demo/scratch markers are a titling convention, so scanning
+  // the narrative/facts/concepts produced false positives on real memories that merely
+  // *mentioned* one of these words in prose — and because this filter runs before the
+  // PRIORITY_TYPES check and drops the observation outright (not a score penalty), a
+  // single unlucky word in a long narrative made the whole memory unreachable.
+  const title = obs.title ?? '';
+  return NOISE_PATTERNS.some((pattern) => pattern.test(title)) || isCommandTrace(obs);
 }
 
 function isSystemSelfObservation(obs: Observation): boolean {
@@ -321,6 +341,42 @@ export async function getSessionResumeBrief(
   };
 }
 
+const AUTO_RESUME_SUMMARY_LIMIT = 480;
+const AUTO_RESUME_TITLE_LIMIT = 180;
+
+function compactResumeText(text: string, limit: number): string {
+  const normalized = text.replace(/\s+/g, ' ').trim();
+  return normalized.length <= limit ? normalized : `${normalized.slice(0, Math.max(0, limit - 3)).trimEnd()}...`;
+}
+
+/**
+ * Render the bounded automatic delivery form of a resume brief.
+ *
+ * Full session context remains available through getSessionContext() and
+ * individual memory details remain on demand. Automatic session start gets an
+ * index, not a transcript, so it cannot crowd out the new task's context.
+ */
+export function renderSessionResumeCard(brief: SessionResumeBrief): string {
+  if (!brief.previousSession && brief.memories.length === 0) return '';
+
+  const lines = ['## Memorix Resume'];
+  if (brief.previousSession) {
+    const agent = brief.previousSession.agent ? ` (${brief.previousSession.agent})` : '';
+    lines.push(`Previous session: ${brief.previousSession.id}${agent}`);
+    lines.push(compactResumeText(brief.previousSession.summary, AUTO_RESUME_SUMMARY_LIMIT));
+  }
+
+  if (brief.memories.length > 0) {
+    lines.push('', 'Relevant memory references:');
+    for (const memory of brief.memories) {
+      lines.push(`- #${memory.id} [${memory.type}] ${compactResumeText(memory.title, AUTO_RESUME_TITLE_LIMIT)}`);
+    }
+    lines.push('Read a listed memory only when its title is relevant to the current task.');
+  }
+
+  return lines.join('\n');
+}
+
 export function scoreObservationForSessionContext(obs: Observation, projectTokens: string[], now = Date.now()): number {
   let score = TYPE_WEIGHTS[obs.type] ?? 1;
   const text = stringifyObservation(obs);
@@ -384,7 +440,7 @@ export function scoreObservationForSessionContext(obs: Observation, projectToken
  * so the agent can resume work without re-explaining everything.
  */
 export async function startSession(
-  projectDir: string,
+  _projectDir: string,
   projectId: string,
   opts?: { sessionId?: string; agent?: string; reader?: ObservationReader },
 ): Promise<{ session: Session; previousContext: string }> {
@@ -399,8 +455,11 @@ export async function startSession(
     agent: opts?.agent,
   };
 
-  // Load previous context before creating new session
-  const previousContext = await getSessionContext(projectDir, projectId, 3, opts?.reader);
+  // Automatic delivery stays intentionally small. The explicit context tool can
+  // still provide the expanded packet when an agent chooses to inspect it.
+  const previousContext = renderSessionResumeCard(
+    await getSessionResumeBrief(projectId, undefined, opts?.reader),
+  );
 
   // Atomic rollover: complete all active sessions for this project's aliases
   // and insert the new session in a single SQLite transaction.

package/src/runtime/maintenance-jobs.ts

@@ -67,7 +67,18 @@ export interface MaintenanceJobSummary {
 
 export type MaintenanceJobRunResult =
   | { action: 'complete' }
-  | { action: 'reschedule'; delayMs: number; resetAttempts?: boolean; payload?: Record<string, unknown> };
+  | {
+    action: 'reschedule';
+    delayMs: number;
+    resetAttempts?: boolean;
+    payload?: Record<string, unknown>;
+    /** Keep recoverable work out of the immediate pending queue during a cooldown. */
+    status?: 'pending' | 'retry';
+    /** Persist a sanitized diagnostic without consuming the job's retry budget. */
+    lastError?: string;
+    /** Clear a stale transient diagnostic after the job makes progress. */
+    clearLastError?: boolean;
+  };
 
 export type MaintenanceJobHandler = (
   job: MaintenanceJob,
@@ -160,6 +171,12 @@ export class MaintenanceJobStore {
       `).get(input.projectId, input.kind, dedupeKey);
 
       if (existing) {
+        // A vector backfill in cooldown is a circuit-breaker state. A new MCP
+        // process must not pull it forward and recreate the original retry storm.
+        if (input.kind === 'vector-backfill' && existing.status === 'retry') {
+          this.commit();
+          return rowToJob(existing);
+        }
         const nextRunAfter = Math.min(Number(existing.run_after), runAfter);
         this.db.prepare(`
           UPDATE maintenance_jobs
@@ -171,6 +188,30 @@ export class MaintenanceJobStore {
         return rowToJob(updated);
       }
 
+      // Older releases exhausted vector jobs on transient provider/index errors.
+      // Reuse one failed record so the next healthy session recovers work instead
+      // of adding another permanent failure row for the same project.
+      if (input.kind === 'vector-backfill') {
+        const failed = this.db.prepare(`
+          SELECT * FROM maintenance_jobs
+          WHERE project_id = ? AND kind = ? AND dedupe_key = ? AND status = 'failed'
+          ORDER BY updated_at DESC
+          LIMIT 1
+        `).get(input.projectId, input.kind, dedupeKey);
+        if (failed) {
+          this.db.prepare(`
+            UPDATE maintenance_jobs
+            SET status = 'pending', attempts = 0, max_attempts = ?, run_after = ?,
+                payload_json = ?, lease_owner = NULL, lease_expires_at = NULL,
+                last_error = NULL, completed_at = NULL, updated_at = ?
+            WHERE id = ?
+          `).run(maxAttempts, runAfter, payloadJson, now, failed.id);
+          const revived = this.getRow(failed.id)!;
+          this.commit();
+          return rowToJob(revived);
+        }
+      }
+
       const id = randomUUID();
       this.db.prepare(`
         INSERT INTO maintenance_jobs (
@@ -361,21 +402,57 @@ export class MaintenanceJobStore {
   reschedule(
     id: string,
     workerId: string,
-    options: { delayMs: number; resetAttempts?: boolean; payload?: Record<string, unknown>; now?: number },
+    options: {
+      delayMs: number;
+      resetAttempts?: boolean;
+      payload?: Record<string, unknown>;
+      status?: 'pending' | 'retry';
+      lastError?: string;
+      clearLastError?: boolean;
+      now?: number;
+    },
   ): MaintenanceJob | undefined {
     const now = options.now ?? Date.now();
     const delayMs = Number.isFinite(options.delayMs) ? Math.max(0, Math.floor(options.delayMs)) : 0;
     const payloadJson = options.payload === undefined ? null : JSON.stringify(options.payload);
+    const status = options.status === 'retry' ? 'retry' : 'pending';
+    const hasLastError = options.lastError !== undefined;
+    const lastError = hasLastError ? errorText(options.lastError) : null;
     this.db.prepare(`
       UPDATE maintenance_jobs
-      SET status = 'pending', run_after = ?, attempts = CASE WHEN ? THEN 0 ELSE attempts END,
+      SET status = ?, run_after = ?, attempts = CASE WHEN ? THEN 0 ELSE attempts END,
           payload_json = COALESCE(?, payload_json),
+          last_error = CASE
+            WHEN ? THEN NULL
+            WHEN ? THEN ?
+            ELSE last_error
+          END,
           lease_owner = NULL, lease_expires_at = NULL, updated_at = ?
       WHERE id = ? AND status = 'running' AND lease_owner = ?
-    `).run(now + delayMs, options.resetAttempts ? 1 : 0, payloadJson, now, id, workerId);
+    `).run(
+      status,
+      now + delayMs,
+      options.resetAttempts ? 1 : 0,
+      payloadJson,
+      options.clearLastError ? 1 : 0,
+      hasLastError ? 1 : 0,
+      lastError,
+      now,
+      id,
+      workerId,
+    );
     return this.get(id);
   }
 
+  resolveFailedVectorBackfills(projectId: string, dedupeKey = 'vector-backfill', now = Date.now()): number {
+    const result = this.db.prepare(`
+      UPDATE maintenance_jobs
+      SET status = 'completed', completed_at = ?, updated_at = ?
+      WHERE project_id = ? AND kind = 'vector-backfill' AND dedupe_key = ? AND status = 'failed'
+    `).run(now, now, projectId, dedupeKey);
+    return Number(result.changes ?? 0);
+  }
+
   fail(id: string, workerId: string, error: unknown, now = Date.now()): MaintenanceJob | undefined {
     this.begin();
     try {
@@ -486,6 +563,9 @@ export class MaintenanceJobWorker {
             delayMs: result.delayMs,
             resetAttempts: result.resetAttempts,
             payload: result.payload,
+            status: result.status,
+            lastError: result.lastError,
+            clearLastError: result.clearLastError,
             now,
           });
           return { state: 'rescheduled', job: updated };

package/src/runtime/project-maintenance.ts

@@ -1,6 +1,7 @@
-import type {
-  MaintenanceJobHandler,
-  MaintenanceJobRunResult,
+import {
+  MaintenanceJobStore,
+  type MaintenanceJobHandler,
+  type MaintenanceJobRunResult,
 } from './maintenance-jobs.js';
 import { runMaintenanceInChildProcess } from './isolated-maintenance.js';
 import {
@@ -18,6 +19,8 @@ const DEFAULT_CODEGRAPH_MAX_FILES = 5_000;
 const DEFAULT_CLAIM_DERIVATION_BATCH_SIZE = 100;
 const DEFAULT_OBSERVATION_QUALIFICATION_BATCH_SIZE = 100;
 const VECTOR_RETRY_DELAY_MS = 5_000;
+const VECTOR_FAILURE_BASE_RETRY_DELAY_MS = 60_000;
+const VECTOR_FAILURE_MAX_RETRY_DELAY_MS = 15 * 60_000;
 const DEDUP_PER_PAIR_TIMEOUT_MS = 5_000;
 
 function vectorBatchSize(payload: Record<string, unknown>): number {
@@ -26,6 +29,33 @@ function vectorBatchSize(payload: Record<string, unknown>): number {
   return Math.min(100, Math.max(1, Math.floor(value)));
 }
 
+function vectorBackfillFailureStreak(payload: Record<string, unknown>): number {
+  const value = payload.vectorBackfillFailureStreak;
+  return typeof value === 'number' && Number.isFinite(value)
+    ? Math.min(16, Math.max(0, Math.floor(value)))
+    : 0;
+}
+
+function vectorBackfillRetryDelayMs(streak: number): number {
+  return Math.min(
+    VECTOR_FAILURE_MAX_RETRY_DELAY_MS,
+    VECTOR_FAILURE_BASE_RETRY_DELAY_MS * (2 ** Math.max(0, streak - 1)),
+  );
+}
+
+function withoutVectorBackfillFailureStreak(payload: Record<string, unknown>): Record<string, unknown> {
+  const { vectorBackfillFailureStreak: _streak, ...rest } = payload;
+  return rest;
+}
+
+function resolveSupersededVectorFailures(projectDir: string, projectId: string): void {
+  try {
+    new MaintenanceJobStore(projectDir).resolveFailedVectorBackfills(projectId);
+  } catch {
+    // Health cleanup must never block the live memory path.
+  }
+}
+
 function retentionBatchSize(payload: Record<string, unknown>): number {
   const value = payload.limit;
   if (typeof value !== 'number' || !Number.isFinite(value)) return DEFAULT_RETENTION_BATCH_SIZE;
@@ -498,23 +528,50 @@ export function createProjectMaintenanceHandler(
 
     const { backfillVectorEmbeddings, getVectorStatus } = await import('../memory/observations.js');
     const before = getVectorStatus(projectId);
-    if (before.missing === 0) return { action: 'complete' };
+    if (before.missing === 0) {
+      resolveSupersededVectorFailures(projectDir, projectId);
+      return { action: 'complete' };
+    }
 
     const result = await backfillVectorEmbeddings({
       projectId,
       limit: vectorBatchSize(job.payload),
     });
     const after = getVectorStatus(projectId);
-    if (after.missing === 0) return { action: 'complete' };
+    if (after.missing === 0) {
+      resolveSupersededVectorFailures(projectDir, projectId);
+      return { action: 'complete' };
+    }
 
     if (result.failed > 0 && result.succeeded === 0) {
-      throw new Error(`vector backfill made no progress (${result.failed}/${result.attempted} failed)`);
+      const streak = vectorBackfillFailureStreak(job.payload) + 1;
+      return {
+        action: 'reschedule',
+        status: 'retry',
+        delayMs: vectorBackfillRetryDelayMs(streak),
+        // This is a recoverable provider/index state, not a terminal job error.
+        // Keeping one retry row prevents new MCP processes from creating a storm.
+        resetAttempts: true,
+        payload: {
+          ...job.payload,
+          vectorBackfillFailureStreak: streak,
+        },
+        lastError: result.lastError ?? `vector backfill made no progress (${result.failed}/${result.attempted} failed)`,
+      };
     }
 
+    const priorFailureStreak = vectorBackfillFailureStreak(job.payload);
+    const payload = withoutVectorBackfillFailureStreak(job.payload);
     return {
       action: 'reschedule',
       delayMs: result.attempted === 0 ? VECTOR_RETRY_DELAY_MS : 0,
       resetAttempts: result.succeeded > 0,
+      ...(result.succeeded > 0 ? {
+        ...(priorFailureStreak > 0 ? { payload } : {}),
+        ...(result.failed > 0 && result.lastError
+          ? { lastError: result.lastError }
+          : priorFailureStreak > 0 ? { clearLastError: true } : {}),
+      } : {}),
     };
   };
 }