memorix 1.8.9 → 1.9.0

package/src/cli/commands/serve-http.ts

@@ -21,6 +21,7 @@
 
 import { defineCommand } from 'citty';
 import type { IncomingMessage, ServerResponse } from 'node:http';
+import { getHeapStatistics } from 'node:v8';
 import type { ObservationStore } from '../../store/obs-store.js';
 import { resolveToolProfile } from '../../server/tool-profile.js';
 import { scopeKnowledgeGraphToProject } from '../../memory/graph-scope.js';
@@ -1714,6 +1715,8 @@ export default defineCommand({
       // This is the readiness signal for background start / status.
       if (url.pathname === '/health') {
         const { getEmbeddingRuntimeHealth } = await import('../../embedding/provider.js');
+        const memory = process.memoryUsage();
+        const heap = getHeapStatistics();
         res.writeHead(200, { 'Content-Type': 'application/json' });
         res.end(JSON.stringify({
           status: 'ok',
@@ -1725,6 +1728,14 @@ export default defineCommand({
           projectId: defaultProject?.id ?? '__unresolved__',
           projectName: defaultProject?.name ?? null,
           readyAt: httpReadyAt,
+          memory: {
+            rss: memory.rss,
+            heapUsed: memory.heapUsed,
+            heapTotal: memory.heapTotal,
+            external: memory.external,
+            arrayBuffers: memory.arrayBuffers,
+            heapLimit: heap.heap_size_limit,
+          },
           embedding: getEmbeddingRuntimeHealth(),
         }));
         return;

package/src/cli/commands/sync-store.ts

@@ -0,0 +1,191 @@
+/**
+ * `memorix sync store <push|pull|status>` — multi-device store replication.
+ *
+ * Separated from `memorix sync rules|workspace` (cross-agent rules) so the two
+ * meanings of "sync" never collide. Store sync is opt-in and provider-agnostic;
+ * the remote is selected by `MEMORIX_SYNC_PROVIDER`.
+ */
+
+import { defineCommand } from 'citty';
+
+import { getCliProjectContext, emitError, emitResult } from './operator-shared.js';
+import { getObservationStore, initObservationStore } from '../../store/obs-store.js';
+import { runSync } from '../../sync/engine.js';
+import { createSqliteSyncStore } from '../../sync/store-port.js';
+import { resolveSyncConfig, createRemote } from '../../sync/remote-factory.js';
+import type { SyncReport } from '../../sync/types.js';
+
+function renderReport(report: SyncReport): string {
+  const lines = [
+    `Store sync (${report.remote}) — device ${report.deviceId}${report.dryRun ? ' [dry-run]' : ''}`,
+    `- Pushed changes:   ${report.pushed}`,
+    `- Pulled batches:   ${report.pulledBatches}`,
+    `- Applied upserts:  ${report.applied}`,
+    `- Applied deletes:  ${report.tombstoned}`,
+    `- Skipped (older):  ${report.skipped}`,
+    `- Excluded locally:  ${report.excluded}`,
+    `- Conflicts recorded: ${report.conflicts}`,
+    `- Pending outbox:    ${report.pending}`,
+  ];
+  if (report.dryRun && report.decisions.length > 0) {
+    lines.push('', 'Decisions:');
+    for (const d of report.decisions) {
+      lines.push(`  ${d.syncKey} ${d.kind} -> ${d.outcome} (${d.reason})`);
+    }
+  }
+  return lines.join('\n');
+}
+
+export default defineCommand({
+  meta: {
+    name: 'sync-store',
+    description: 'Replicate the local observation store across your devices (opt-in, provider-agnostic)',
+  },
+  args: {
+    dry: { type: 'boolean', description: 'Preview changes without writing anything', default: false },
+    json: { type: 'boolean', description: 'Emit JSON', default: false },
+    'push-only': { type: 'boolean', description: 'Only push local changes', default: false },
+    'pull-only': { type: 'boolean', description: 'Only pull remote changes', default: false },
+    through: { type: 'string', description: 'Compaction cutoff: device=sequence,device=sequence' },
+    yes: { type: 'boolean', description: 'Confirm destructive remote compaction', default: false },
+  },
+  run: async ({ args }) => {
+    const asJson = Boolean(args.json);
+    const action = (args._ as string[])?.[0] || 'status';
+    if (action === 'device') {
+      const { project, dataDir } = await getCliProjectContext();
+      await initObservationStore(dataDir);
+      const syncStore = createSqliteSyncStore(dataDir, getObservationStore(), project.id);
+      const subaction = (args._ as string[])?.[1] || '';
+      if (subaction !== 'rotate') {
+        emitError('expected "memorix sync store device rotate"', asJson);
+        process.exitCode = 2;
+        return;
+      }
+      const deviceId = syncStore.rotateDevice();
+      emitResult({ project: project.id, deviceId, rotated: true }, `Sync device identity rotated: ${deviceId}`, asJson);
+      return;
+    }
+    if (action !== 'push' && action !== 'pull' && action !== 'status' && action !== 'compact') {
+      emitError(`unknown action "${action}" (expected push|pull|status|compact|device rotate)`, asJson);
+      process.exitCode = 2;
+      return;
+    }
+
+    let config;
+    try {
+      config = resolveSyncConfig();
+    } catch (err) {
+      emitError((err as Error).message, asJson);
+      process.exitCode = 2;
+      return;
+    }
+
+    if (!config.enabled || !config.provider) {
+      emitResult(
+        { enabled: false },
+        'Store sync is disabled. Set MEMORIX_SYNC_PROVIDER=fs|github|s3|postgres to enable it.',
+        asJson,
+      );
+      return;
+    }
+
+    const { project, dataDir } = await getCliProjectContext();
+    await initObservationStore(dataDir);
+    const obsStore = getObservationStore();
+    const syncStore = createSqliteSyncStore(dataDir, obsStore, project.id);
+
+    if (action === 'compact') {
+      if (!args.yes && !args.dry) {
+        emitError('remote compaction deletes relay events; repeat with --yes after specifying --through device=sequence', asJson);
+        process.exitCode = 2;
+        return;
+      }
+      const raw = String(args.through ?? '');
+      const through: Record<string, number> = {};
+      for (const item of raw.split(',').map((value) => value.trim()).filter(Boolean)) {
+        const [device, sequence] = item.split('=', 2);
+        const parsed = Number(sequence);
+        if (!device || !Number.isSafeInteger(parsed) || parsed < 0) {
+          emitError('invalid --through; expected device=sequence[,device=sequence]', asJson);
+          process.exitCode = 2;
+          return;
+        }
+        through[device] = parsed;
+      }
+      if (Object.keys(through).length === 0) {
+        emitError('--through is required for compaction', asJson);
+        process.exitCode = 2;
+        return;
+      }
+      let remote;
+      try {
+        remote = await createRemote(config.provider, syncStore.namespace());
+      } catch (err) {
+        emitError((err as Error).message, asJson);
+        process.exitCode = 1;
+        return;
+      }
+      try {
+        await remote.init({ create: false });
+        const result = await remote.compact(through, { dryRun: Boolean(args.dry) });
+        emitResult({ project: project.id, through, ...result, dryRun: Boolean(args.dry) }, `Compaction candidates: ${result.candidates}; deleted: ${result.deleted}`, asJson);
+      } catch (err) {
+        emitError((err as Error).message, asJson);
+        process.exitCode = 1;
+      } finally {
+        await remote.close();
+      }
+      return;
+    }
+
+    if (action === 'status') {
+      let remote;
+      try {
+        remote = await createRemote(config.provider, syncStore.namespace());
+      } catch (err) {
+        emitError((err as Error).message, asJson);
+        process.exitCode = 1;
+        return;
+      }
+      try {
+        const report = await runSync(syncStore, remote, {
+          deviceId: syncStore.deviceId(),
+          dryRun: true,
+          push: true,
+          pull: true,
+        });
+        emitResult({ project: project.id, ...report }, renderReport(report), asJson);
+      } catch (err) {
+        emitError((err as Error).message, asJson);
+        process.exitCode = 1;
+      }
+      return;
+    }
+
+    const doPush = action === 'push' || !args['pull-only'];
+    const doPull = action === 'pull' || !args['push-only'];
+
+    let remote;
+    try {
+      remote = await createRemote(config.provider, syncStore.namespace());
+    } catch (err) {
+      emitError((err as Error).message, asJson);
+      process.exitCode = 1;
+      return;
+    }
+
+    try {
+      const report = await runSync(syncStore, remote, {
+        deviceId: syncStore.deviceId(),
+        dryRun: Boolean(args.dry),
+        push: doPush,
+        pull: doPull,
+      });
+      emitResult({ project: project.id, ...report }, renderReport(report), asJson);
+    } catch (err) {
+      emitError((err as Error).message, asJson);
+      process.exitCode = 1;
+    }
+  },
+});

package/src/cli/commands/sync.ts

@@ -51,6 +51,14 @@ export default defineCommand({
   },
   run: async ({ args }) => {
     const section = (args._ as string[])?.[0] || '';
+
+    // Store replication is a distinct concern from rules/workspace sync.
+    if (section === 'store') {
+      const storeCmd = (await import('./sync-store.js')).default;
+      const rest = (args._ as string[]).slice(1);
+      await storeCmd.run!({ args: { ...args, _: rest } } as never);
+      return;
+    }
     const asJson = !!args.json;
 
     if (section === 'rules' || section === 'workspace') {

package/src/memory/observations.ts

@@ -83,6 +83,7 @@ export function resetObservationRuntime(): void {
   embeddingWorkerProjectRoot = undefined;
   vectorMissingIds.clear();
   vectorSchemaUpgradePromise = null;
+  embeddingFailureLogTimestamps.clear();
 }
 
 // ── Vector-missing tracking ──────────────────────────────────────
@@ -100,11 +101,17 @@ let lastVectorBackfill: {
 } | null = null;
 const embeddingFailureLogTimestamps = new Map<string, number>();
 const EMBEDDING_FAILURE_LOG_COOLDOWN_MS = 30_000;
+const MAX_EMBEDDING_FAILURE_KEYS = 1_024;
 
 function logEmbeddingFailureOnce(key: string, message: string): void {
   const now = Date.now();
   const last = embeddingFailureLogTimestamps.get(key) ?? 0;
   if (now - last < EMBEDDING_FAILURE_LOG_COOLDOWN_MS) return;
+  if (!embeddingFailureLogTimestamps.has(key) && embeddingFailureLogTimestamps.size >= MAX_EMBEDDING_FAILURE_KEYS) {
+    const oldest = embeddingFailureLogTimestamps.keys().next().value;
+    if (oldest !== undefined) embeddingFailureLogTimestamps.delete(oldest);
+  }
+  embeddingFailureLogTimestamps.delete(key);
   embeddingFailureLogTimestamps.set(key, now);
   console.error(message);
 }
@@ -149,13 +156,19 @@ function queueVectorBackfill(
   if (!dataDir) return;
   void import('../runtime/maintenance-jobs.js')
     .then(({ MaintenanceJobStore }) => {
-      new MaintenanceJobStore(dataDir).enqueue({
+      const job = new MaintenanceJobStore(dataDir).enqueue({
         projectId,
         kind: 'vector-backfill',
         dedupeKey: 'vector-backfill',
         payload: { limit: 12 },
       });
-      if (options.detachedWorker && options.projectRoot) {
+      // A retrying job is intentionally in cooldown. Starting a new detached
+      // process for every hook event would turn one provider outage into a
+      // process storm even though the durable queue already deduplicated it.
+      // Keep compatibility with lightweight test/embedding adapters that only
+      // implement enqueue and return no job receipt.
+      const due = !job || (job.status === 'pending' && job.runAfter <= Date.now());
+      if (due && options.detachedWorker && options.projectRoot) {
         void import('../runtime/vector-backfill-runner.js')
           .then(({ launchDetachedVectorBackfill }) => {
             launchDetachedVectorBackfill({

package/src/runtime/maintenance-targets.ts

@@ -56,4 +56,10 @@ export class MaintenanceTargetStore {
     `).get(projectId);
     return row ? rowToTarget(row) : undefined;
   }
+
+  list(): MaintenanceTarget[] {
+    return (this.db.prepare(`
+      SELECT * FROM maintenance_targets ORDER BY updated_at DESC
+    `).all() as any[]).map(rowToTarget);
+  }
 }

package/src/runtime/vector-backfill-runner.ts

@@ -1,5 +1,14 @@
 import { spawn, type ChildProcess } from 'node:child_process';
-import { existsSync } from 'node:fs';
+import {
+  closeSync,
+  existsSync,
+  mkdirSync,
+  openSync,
+  readFileSync,
+  statSync,
+  unlinkSync,
+  writeFileSync,
+} from 'node:fs';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 
@@ -11,8 +20,12 @@ import { initObservationStore } from '../store/obs-store.js';
 import { getDeferredCachedVectorHydration } from '../store/orama-store.js';
 import { closeAllDatabases } from '../store/sqlite-db.js';
 import { MaintenanceJobStore, MaintenanceJobWorker } from './maintenance-jobs.js';
+import { MaintenanceTargetStore, type MaintenanceTarget } from './maintenance-targets.js';
 import { createProjectMaintenanceHandler } from './project-maintenance.js';
 
+const VECTOR_BACKFILL_LOCK_FILE = '.memorix-vector-backfill.lock';
+const STALE_LOCK_RECOVERY_MS = 24 * 60 * 60 * 1_000;
+
 export interface VectorBackfillRequest {
   projectId: string;
   projectRoot: string;
@@ -23,6 +36,20 @@ export interface VectorBackfillLauncherOptions {
   runnerPath?: string;
   exists?: (path: string) => boolean;
   spawn?: typeof spawn;
+  lockPath?: string;
+}
+
+interface VectorBackfillLock {
+  parentPid: number;
+  childPid?: number;
+  startedAt: number;
+  projectId: string;
+}
+
+interface VectorBackfillLockHandle {
+  path: string;
+  setChildPid(pid: number): void;
+  release(): void;
 }
 
 function isNonEmptyString(value: unknown): value is string {
@@ -68,6 +95,111 @@ export function parseVectorBackfillRequest(raw: string): VectorBackfillRequest {
   };
 }
 
+export function vectorBackfillLockPath(dataDir: string): string {
+  return path.join(dataDir, VECTOR_BACKFILL_LOCK_FILE);
+}
+
+function processIsAlive(pid: number | undefined): boolean {
+  if (!Number.isSafeInteger(pid) || pid == null || pid <= 0) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    // EPERM means the process exists but this process cannot signal it.
+    return (error as NodeJS.ErrnoException).code === 'EPERM';
+  }
+}
+
+function readVectorBackfillLock(lockPath: string): VectorBackfillLock | undefined {
+  try {
+    const value = JSON.parse(readFileSync(lockPath, 'utf8')) as Partial<VectorBackfillLock>;
+    if (
+      typeof value.parentPid !== 'number'
+      || !Number.isSafeInteger(value.parentPid)
+      || typeof value.startedAt !== 'number'
+      || !Number.isFinite(value.startedAt)
+      || typeof value.projectId !== 'string'
+    ) return undefined;
+    return {
+      parentPid: value.parentPid,
+      ...(typeof value.childPid === 'number' ? { childPid: value.childPid } : {}),
+      startedAt: value.startedAt,
+      projectId: value.projectId,
+    };
+  } catch {
+    return undefined;
+  }
+}
+
+function writeVectorBackfillLock(lockPath: string, value: VectorBackfillLock): void {
+  writeFileSync(lockPath, `${JSON.stringify(value)}\n`, 'utf8');
+}
+
+function acquireVectorBackfillLock(
+  dataDir: string,
+  projectId: string,
+  preferredPath?: string,
+): VectorBackfillLockHandle | undefined {
+  const lockPath = preferredPath ?? vectorBackfillLockPath(dataDir);
+  mkdirSync(path.dirname(lockPath), { recursive: true });
+  const initial: VectorBackfillLock = {
+    parentPid: process.pid,
+    startedAt: Date.now(),
+    projectId,
+  };
+
+  for (let attempt = 0; attempt < 2; attempt++) {
+    try {
+      const fd = openSync(lockPath, 'wx');
+      try {
+        writeFileSync(fd, `${JSON.stringify(initial)}\n`, 'utf8');
+      } finally {
+        closeSync(fd);
+      }
+      return {
+        path: lockPath,
+        setChildPid(pid: number) {
+          try {
+            writeVectorBackfillLock(lockPath, { ...initial, childPid: pid });
+          } catch {
+            // The child can still release the lock on normal shutdown; a dead
+            // owner is recoverable on the next launch.
+          }
+        },
+        release() {
+          try { unlinkSync(lockPath); } catch { /* already released */ }
+        },
+      };
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== 'EEXIST') return undefined;
+      const existing = readVectorBackfillLock(lockPath);
+      const ownerAlive = existing && (processIsAlive(existing.childPid) || processIsAlive(existing.parentPid));
+      if (ownerAlive) return undefined;
+
+      // A malformed lock is not deleted immediately: another process may be
+      // between create and write. A dead, well-formed owner is safe to reap;
+      // an unreadable lock needs a long recovery window so it cannot cause a
+      // second heavy worker during a slow filesystem operation.
+      let stale = false;
+      try {
+        stale = Date.now() - statSync(lockPath).mtimeMs >= STALE_LOCK_RECOVERY_MS;
+      } catch {
+        stale = false;
+      }
+      if (!existing && !stale) return undefined;
+      try { unlinkSync(lockPath); } catch { return undefined; }
+    }
+  }
+  return undefined;
+}
+
+export function releaseVectorBackfillLock(lockPath = process.env.MEMORIX_VECTOR_BACKFILL_LOCK): void {
+  if (!lockPath) return;
+  const lock = readVectorBackfillLock(lockPath);
+  if (lock && lock.childPid !== process.pid) return;
+  try { unlinkSync(lockPath); } catch { /* already released */ }
+}
+
 /**
  * Start a detached one-shot worker. The caller has already persisted the
  * observation and durable vector job, so failure to start is recoverable by a
@@ -81,6 +213,9 @@ export function launchDetachedVectorBackfill(
   const exists = options.exists ?? existsSync;
   if (!exists(runnerPath)) return false;
 
+  const lock = acquireVectorBackfillLock(request.dataDir, request.projectId, options.lockPath);
+  if (!lock) return false;
+
   try {
     const child = (options.spawn ?? spawn)(process.execPath, [runnerPath], {
       cwd: request.projectRoot,
@@ -90,13 +225,24 @@ export function launchDetachedVectorBackfill(
       stdio: 'ignore',
       // The request contains only local project metadata, never credentials.
       // Environment transport avoids a live stdin pipe keeping the CLI alive.
-      env: { ...process.env, MEMORIX_VECTOR_BACKFILL_REQUEST: JSON.stringify(request) },
+      env: {
+        ...process.env,
+        MEMORIX_VECTOR_BACKFILL_REQUEST: JSON.stringify(request),
+        MEMORIX_VECTOR_BACKFILL_LOCK: lock.path,
+      },
       windowsHide: true,
     }) as ChildProcess;
+    if (child.pid) lock.setChildPid(child.pid);
+    else lock.release();
     child.once?.('error', () => {});
+    child.once?.('close', () => lock.release());
+    // Test doubles and unusual spawn adapters may not expose ChildProcess
+    // events. Do not leave a lock behind when there is no lifecycle to own it.
+    if (typeof child.once !== 'function') lock.release();
     child.unref();
-    return true;
+    return Boolean(child.pid);
   } catch {
+    lock.release();
     return false;
   }
 }
@@ -110,12 +256,43 @@ export async function executeVectorBackfill(request: VectorBackfillRequest) {
   await prepareSearchIndex();
   await getDeferredCachedVectorHydration()?.catch(() => {});
 
+  const queue = new MaintenanceJobStore(request.dataDir);
+  const targets = new MaintenanceTargetStore(request.dataDir);
+  let preparedRoot = request.projectRoot;
+  const prepareTarget = async (target: MaintenanceTarget): Promise<void> => {
+    if (target.projectRoot === preparedRoot) return;
+    initProjectRoot(target.projectRoot);
+    loadDotenv(target.projectRoot);
+    preparedRoot = target.projectRoot;
+    // The data directory is shared across projects. The in-process index is
+    // intentionally global, so switching the target only changes the config
+    // and the job scope; it does not create another corpus/index copy.
+  };
+
+  const handler = async (job: Parameters<ReturnType<typeof createProjectMaintenanceHandler>>[0]) => {
+    const target = targets.get(job.projectId)
+      ?? (job.projectId === request.projectId
+        ? { projectId: request.projectId, projectRoot: request.projectRoot, dataDir: request.dataDir, updatedAt: Date.now() }
+        : undefined);
+    if (!target) return { action: 'reschedule' as const, delayMs: 30_000 };
+    await prepareTarget(target);
+    return createProjectMaintenanceHandler(job.projectId, target.dataDir, target.projectRoot)(job);
+  };
+
   const worker = new MaintenanceJobWorker(
-    new MaintenanceJobStore(request.dataDir),
-    createProjectMaintenanceHandler(request.projectId, request.dataDir, request.projectRoot),
-    { projectId: request.projectId, kinds: ['vector-backfill'] },
+    queue,
+    handler,
+    { kinds: ['vector-backfill'] },
   );
-  return worker.runOnce();
+  // Drain currently due vector jobs while this one process owns the lock. A
+  // hook storm therefore creates at most one heavy process, and pending jobs
+  // for other projects are picked up without requiring another hook event.
+  let result: Awaited<ReturnType<typeof worker.runOnce>> = { state: 'idle' };
+  while (true) {
+    result = await worker.runOnce();
+    if (result.state === 'idle') break;
+  }
+  return result;
 }
 
 async function readStdin(): Promise<string> {
@@ -137,6 +314,7 @@ export async function main(): Promise<void> {
     process.stderr.write(`[memorix] vector backfill worker failed: ${detail}\n`);
     process.exitCode = 1;
   } finally {
+    releaseVectorBackfillLock();
     closeAllDatabases();
   }
 }