@felan-ai/felan 0.21.16 → 0.22.0

package/dist/memory/coordinator.js

@@ -1,12 +1,15 @@
 import { join } from 'node:path';
 import { mkdir, rm } from 'node:fs/promises';
-import { createMemorySnapshot, createMemoryProjectionSnapshot, createSessionCheckpoint, extractSourceIds, hydrateMemoryDirectory, readMemoryDirectory, validateMemoryArtifact, } from '@felan-ai/ext-memory';
+import { createMemorySnapshot, createMemoryProjectionSnapshot, createSessionCheckpoint, extractSourceIds, hydrateMemoryDirectory, readMemoryDirectory, } from '@felan-ai/ext-memory';
 import { ModelRuntime, SessionManager, } from '@felan-ai/agent-core';
 import { acquireLocalMemoryLease, } from './lease.js';
 import { resolveLocalMemoryProject } from './project.js';
-import { createDefaultLocalMemoryDreamRunner, materializeMemoryInput, } from './dreamer.js';
-import { LocalMemoryStore, } from './store.js';
+import { createDefaultLocalMemoryDreamRunner, MemoryModelUnavailableError, materializeMemoryInput, } from './dreamer.js';
+import { LocalMemoryStore, LocalMemoryControlError, } from './store.js';
+import { LocalMemoryRun, sanitizeMemoryDiagnostic } from './run.js';
 const MEMORY_INPUT_BLOCKED_MESSAGE = 'Some memory checkpoints could not be materialized; evidence remains pending';
+const MEMORY_AUTOMATIC_UPDATE_THRESHOLD = 5;
+const MEMORY_AUTOMATIC_INTERVAL_MS = 60 * 60 * 1_000;
 export class LocalMemoryCoordinator {
     #options;
     #contexts = new Map();
@@ -16,11 +19,16 @@ export class LocalMemoryCoordinator {
     #enabled;
     #selectedModel;
     #disposed = false;
+    #monitor;
+    #monitoring;
+    #enableVersion = 0;
     constructor(options) {
         this.#options = options;
         this.#enabled = options.enabled !== false;
         this.#selectedModel = options.selectedModel;
         this.#dreamRunner = options.dreamRunner ?? createDefaultLocalMemoryDreamRunner();
+        if (this.#enabled)
+            this.#startMonitor();
     }
     createSessionHost(options) {
         return new LocalMemorySessionHost(this, options);
@@ -36,19 +44,39 @@ export class LocalMemoryCoordinator {
     }
     setSelectedModel(model) {
         this.#selectedModel = model;
+        for (const context of this.#contexts.values()) {
+            context.blocked.clear();
+            if (context.initialized)
+                void this.#trackScheduleIfPending(context);
+        }
     }
     setEnabled(enabled) {
+        this.#enableVersion += 1;
         this.#enabled = enabled;
+        if (enabled)
+            this.#startMonitor();
+        else if (this.#monitor) {
+            clearInterval(this.#monitor);
+            this.#monitor = undefined;
+        }
         for (const context of this.#contexts.values()) {
-            if (!enabled && context.timer) {
-                clearTimeout(context.timer);
-                context.timer = undefined;
+            if (!enabled)
+                this.#cancelScheduledRun(context);
+            if (!enabled)
+                this.#cancelContext(context, 'Memory processing was disabled', true);
+            if (enabled) {
+                context.storageBlocked = false;
+                context.writerBlocked = false;
+                context.blocked.clear();
+                if (context.initialized)
+                    this.#startRecovery(context);
             }
             context.state = enabled ? 'idle' : 'disabled';
         }
         if (enabled) {
             for (const context of this.#contexts.values()) {
-                void this.#trackScheduleIfPending(context, 0);
+                if (context.initialized)
+                    void this.#trackScheduleIfPending(context);
             }
         }
         this.#emitStatusChange();
@@ -57,26 +85,36 @@ export class LocalMemoryCoordinator {
         try {
             const context = await this.#context(cwd);
             const state = await context.store.status();
+            const control = await context.store.readControl();
             const processed = Object.values(state.processed);
             const lastProcessedAt = processed
                 .map((entry) => entry.processedAt)
                 .sort()
                 .at(-1);
             return {
-                enabled: this.#enabled,
-                state: this.#enabled ? context.state : 'disabled',
+                enabled: this.#enabled && !control.autoDisabled,
+                state: !this.#enabled || control.autoDisabled ? 'disabled'
+                    : control.nextRetryAt && Date.parse(control.nextRetryAt) > this.#now() ? 'scheduled'
+                        : context.state === 'error' ? 'error'
+                            : control.activeAttempt ? 'processing' : context.state,
                 pendingCheckpoints: Object.keys(state.pending).length,
                 memoryFingerprint: state.memoryFingerprint,
                 ...(lastProcessedAt === undefined ? {} : { lastProcessedAt }),
                 ...(context.message === undefined ? {} : { message: context.message }),
+                consecutiveFailures: control.consecutiveFailures,
+                ...(control.nextRetryAt ? { nextRetryAt: control.nextRetryAt } : {}),
+                ...(control.autoDisabled ? { autoDisabled: control.autoDisabled } : {}),
+                ...(control.activeAttempt || control.lastOutcome ? {
+                    lastRunId: control.activeAttempt?.runId ?? control.lastOutcome.runId,
+                } : {}),
             };
         }
-        catch {
+        catch (error) {
             return {
                 enabled: this.#enabled,
                 state: this.#enabled ? 'error' : 'disabled',
                 pendingCheckpoints: 0,
-                message: 'Local memory storage is unavailable',
+                message: error instanceof LocalMemoryControlError ? error.message : 'Local memory storage is unavailable',
             };
         }
     }
@@ -95,9 +133,8 @@ export class LocalMemoryCoordinator {
     async recordCheckpoint(cwd, checkpoint) {
         const context = await this.#context(cwd);
         if (await context.store.recordCheckpoint(checkpoint)) {
-            context.checkpointVersion += 1;
             context.blocked.delete(checkpoint.sessionId);
-            await this.#schedule(context);
+            await this.#trackScheduleIfPending(context);
             this.#emitStatusChange();
         }
     }
@@ -114,8 +151,7 @@ export class LocalMemoryCoordinator {
             this.#cancelScheduledRun(context);
         }
         context.blocked.clear();
-        const checkpointVersion = context.checkpointVersion;
-        const run = this.#run(context);
+        const run = this.#run(context, true);
         context.run = run;
         try {
             await run;
@@ -123,10 +159,31 @@ export class LocalMemoryCoordinator {
         finally {
             if (context.run === run)
                 context.run = undefined;
-            if (context.checkpointVersion !== checkpointVersion) {
-                await this.#trackScheduleIfPending(context);
-            }
+            await this.#trackScheduleIfPending(context);
+        }
+        return this.status(cwd);
+    }
+    async retryProject(cwd) {
+        if (!this.#enabled)
+            return this.status(cwd);
+        const context = await this.#context(cwd);
+        if (!this.#enabled || this.#disposed)
+            return this.status(cwd);
+        const enableVersion = this.#enableVersion;
+        this.#cancelScheduledRun(context);
+        try {
+            await context.store.resetProjectControl(this.#now(), () => this.#enabled && !this.#disposed && this.#enableVersion === enableVersion);
         }
+        catch (error) {
+            throw error;
+        }
+        context.abort?.abort(new Error('Memory processing was explicitly retried'));
+        if (context.run)
+            await context.run;
+        context.blocked.clear();
+        context.storageBlocked = false;
+        context.message = undefined;
+        await this.runNow(cwd);
         return this.status(cwd);
     }
     async canonicalDirectory(cwd) {
@@ -137,17 +194,22 @@ export class LocalMemoryCoordinator {
         if (this.#disposed)
             return;
         this.#disposed = true;
+        if (this.#monitor)
+            clearInterval(this.#monitor);
+        this.#monitor = undefined;
         for (const context of this.#contexts.values()) {
-            if (context.timer)
-                clearTimeout(context.timer);
-            context.timer = undefined;
-            context.abort?.abort();
+            this.#cancelScheduledRun(context);
+            this.#cancelContext(context, 'Memory processing was cancelled during shutdown', false);
         }
         while (true) {
             const work = [...this.#contexts.values()].flatMap((context) => [
                 ...(context.run ? [context.run] : []),
+                ...(context.ready ? [context.ready] : []),
+                ...(context.recovery ? [context.recovery] : []),
                 ...context.followUps,
             ]);
+            if (this.#monitoring)
+                work.push(this.#monitoring);
             if (work.length === 0)
                 break;
             await Promise.allSettled(work);
@@ -158,67 +220,250 @@ export class LocalMemoryCoordinator {
         if (this.#disposed)
             throw new Error('Local memory coordinator is closed');
         const project = await resolveLocalMemoryProject(cwd);
+        if (this.#disposed)
+            throw new Error('Local memory coordinator is closed');
         const existing = this.#contexts.get(project.key);
-        if (existing)
+        if (existing) {
+            await this.#initialize(existing);
             return existing;
-        const store = new LocalMemoryStore(this.#options.agentDir, project);
+        }
+        const store = new LocalMemoryStore(this.#options.agentDir, project, {
+            ...(this.#options.retryDelaysMs ? { retryDelaysMs: this.#options.retryDelaysMs } : {}),
+        });
         const context = {
             project,
             cwd,
             ...(this.#options.sessionDir === undefined ? {} : { sessionDir: this.#options.sessionDir }),
             store,
-            ready: store.initialize(),
-            recoveryStarted: false,
+            ready: undefined,
+            initialized: false,
+            recovery: undefined,
+            recoveryCompleted: false,
+            diagnosticsReconciled: false,
             timer: undefined,
+            timerDueAt: undefined,
             run: undefined,
             followUps: new Set(),
             abort: undefined,
+            attempt: undefined,
+            storageBlocked: false,
+            writerBlocked: false,
+            statusSignature: undefined,
             blocked: new Map(),
-            checkpointVersion: 0,
             state: this.#enabled ? 'idle' : 'disabled',
             message: undefined,
         };
         this.#contexts.set(project.key, context);
-        await context.ready;
+        await this.#initialize(context);
+        return context;
+    }
+    async #initialize(context) {
+        if (context.ready) {
+            await context.ready;
+            return;
+        }
+        if (context.initialized && context.diagnosticsReconciled) {
+            this.#startRecovery(context);
+            return;
+        }
+        const ready = this.#initializeContext(context).finally(() => {
+            if (context.ready === ready)
+                context.ready = undefined;
+        });
+        context.ready = ready;
+        await ready;
+    }
+    async #initializeContext(context) {
+        if (!context.initialized) {
+            await context.store.initialize(this.#now()).then(() => {
+                context.initialized = true;
+                context.state = this.#enabled ? 'idle' : 'disabled';
+                context.message = undefined;
+            }).catch(async (error) => {
+                if (!(error instanceof LocalMemoryControlError))
+                    throw error;
+                const current = await context.store.readCurrent();
+                if (current.fingerprint !== (await context.store.status()).memoryFingerprint)
+                    throw error;
+                context.message = error.message;
+            });
+        }
+        if (!context.initialized)
+            return;
+        if (!context.diagnosticsReconciled) {
+            try {
+                const lease = await acquireLocalMemoryLease(context.store.projectDirectory, this.#options.leaseOptions);
+                if (lease) {
+                    try {
+                        await this.#reconcileRunDiagnostics(context, lease);
+                        context.diagnosticsReconciled = true;
+                    }
+                    finally {
+                        await lease.release();
+                    }
+                }
+            }
+            catch (error) {
+                context.diagnosticsReconciled = true;
+                context.storageBlocked = true;
+                context.message = `Memory diagnostics recovery is blocked: ${sanitizeMemoryDiagnostic(error)}`;
+            }
+        }
+        if (this.#disposed)
+            return;
         if (this.#enabled)
             void this.#trackScheduleIfPending(context);
-        if (this.#options.recover !== false && !context.recoveryStarted) {
-            context.recoveryStarted = true;
-            void this.#recover(context).catch(() => {
-                context.message = 'Memory startup recovery could not complete';
-            });
+        this.#startRecovery(context);
+    }
+    #startRecovery(context) {
+        if (this.#disposed || !this.#enabled || this.#options.recover === false
+            || context.recoveryCompleted || context.recovery)
+            return;
+        const recovery = this.#recover(context).then((completed) => {
+            if (completed)
+                context.recoveryCompleted = true;
+        }).catch(() => {
+            context.message = 'Memory startup recovery could not complete';
+        }).finally(() => {
+            if (context.recovery === recovery)
+                context.recovery = undefined;
+        });
+        context.recovery = recovery;
+    }
+    #now() { return this.#options.now?.() ?? Date.now(); }
+    #startMonitor() {
+        if (this.#disposed || this.#monitor)
+            return;
+        const interval = this.#options.monitorIntervalMs ?? 2_000;
+        if (!Number.isFinite(interval) || interval < 1)
+            throw new Error('Memory monitor interval must be positive');
+        this.#monitor = setInterval(() => {
+            if (this.#monitoring)
+                return;
+            this.#monitoring = this.#refresh().catch(() => { }).finally(() => { this.#monitoring = undefined; });
+        }, interval);
+        this.#monitor.unref?.();
+    }
+    async #refresh() {
+        if (this.#disposed)
+            return;
+        for (const context of this.#contexts.values()) {
+            try {
+                await this.#initialize(context);
+                const observedAttempt = context.attempt;
+                const control = await context.store.readControl();
+                if (observedAttempt && context.attempt === observedAttempt && (!this.#enabled || control.autoDisabled
+                    || control.activeAttempt?.runId !== observedAttempt.runId
+                    || control.activeAttempt?.generation !== observedAttempt.generation)) {
+                    this.#cancelContext(context, 'Memory processing was cancelled by another session', true);
+                }
+                context.writerBlocked = false;
+                if (context.initialized)
+                    await this.#trackScheduleIfPending(context);
+                const signature = JSON.stringify([this.#enabled, context.state, context.message, control.updatedAt]);
+                if (signature !== context.statusSignature) {
+                    context.statusSignature = signature;
+                    this.#emitStatusChange();
+                }
+            }
+            catch (error) {
+                context.state = 'error';
+                context.message = sanitizeMemoryDiagnostic(error);
+                context.abort?.abort(error);
+                this.#emitStatusChange();
+            }
         }
-        return context;
     }
-    async #schedule(context, delay = this.#options.debounceMs ?? 2_000) {
-        if (this.#disposed || !this.#enabled || context.run || context.timer)
+    #cancelContext(context, reason, cancelPersisted) {
+        const alreadyAborted = context.abort?.signal.aborted ?? false;
+        context.abort?.abort(new Error(reason));
+        if (alreadyAborted || !context.initialized || (!cancelPersisted && !context.attempt))
+            return;
+        const cancellation = cancelPersisted
+            ? context.store.cancelActiveAttempt(this.#now())
+            : context.store.finishAttempt(undefined, context.attempt, { status: 'cancelled', reason }, this.#now());
+        const followUp = cancellation
+            .then(() => { })
+            .catch(() => { })
+            .finally(() => { context.followUps.delete(followUp); });
+        context.followUps.add(followUp);
+    }
+    async #schedule(context, dueAt) {
+        if (this.#disposed || !this.#enabled || context.run || context.storageBlocked)
+            return;
+        if (context.timer && context.timerDueAt === dueAt)
             return;
+        this.#cancelScheduledRun(context);
         context.state = 'scheduled';
+        const delay = Math.max(0, dueAt - this.#now());
         const timer = setTimeout(() => {
             if (context.timer !== timer)
                 return;
             context.timer = undefined;
-            const run = this.#run(context).finally(async () => {
+            context.timerDueAt = undefined;
+            const run = this.#run(context).catch((error) => {
+                context.state = 'error';
+                context.storageBlocked = true;
+                context.message = sanitizeMemoryDiagnostic(error);
+                this.#emitStatusChange();
+            }).finally(async () => {
                 context.run = undefined;
                 await this.#trackScheduleIfPending(context);
             });
             context.run = run;
         }, delay);
         context.timer = timer;
+        context.timerDueAt = dueAt;
         timer.unref?.();
     }
-    async #scheduleIfPending(context, delay = this.#options.debounceMs ?? 2_000) {
+    async #scheduleIfPending(context) {
         if (!this.#enabled || this.#disposed)
             return;
         try {
+            const control = await context.store.readControl();
+            if (control.autoDisabled) {
+                this.#cancelScheduledRun(context);
+                context.state = 'disabled';
+                return;
+            }
+            if (context.storageBlocked)
+                return;
+            if (context.writerBlocked)
+                return;
             const state = await context.store.status();
             const pending = Object.values(state.pending);
-            if (pending.some(({ checkpoint }) => !isBlockedCheckpoint(context, checkpoint))) {
-                await this.#schedule(context, delay);
+            if (control.activeAttempt) {
+                this.#cancelScheduledRun(context);
+            }
+            else if (pending.some(({ checkpoint }) => !isBlockedCheckpoint(context, checkpoint))) {
+                const acceptedUpdates = await context.store.pendingAcceptedUpdates((checkpoint) => !isBlockedCheckpoint(context, checkpoint));
+                if (acceptedUpdates < MEMORY_AUTOMATIC_UPDATE_THRESHOLD) {
+                    this.#cancelScheduledRun(context);
+                    if (context.state === 'scheduled')
+                        context.state = 'idle';
+                    return;
+                }
+                const processedAt = Object.values(state.processed)
+                    .map(({ processedAt }) => Date.parse(processedAt))
+                    .filter(Number.isFinite)
+                    .reduce((latest, at) => Math.max(latest, at), 0);
+                const dueAt = control.nextRetryAt
+                    ? Date.parse(control.nextRetryAt)
+                    : processedAt === 0
+                        ? this.#now()
+                        : processedAt + (this.#options.debounceMs ?? MEMORY_AUTOMATIC_INTERVAL_MS);
+                await this.#schedule(context, dueAt);
             }
             else if (pending.length > 0) {
+                this.#cancelScheduledRun(context);
+                const message = context.state === 'blocked' ? context.message : undefined;
                 context.state = 'blocked';
-                context.message = MEMORY_INPUT_BLOCKED_MESSAGE;
+                context.message = message ?? MEMORY_INPUT_BLOCKED_MESSAGE;
+            }
+            else {
+                this.#cancelScheduledRun(context);
+                if (context.state === 'scheduled')
+                    context.state = 'idle';
             }
         }
         catch {
@@ -226,15 +471,15 @@ export class LocalMemoryCoordinator {
             context.message = 'Local memory storage is unavailable';
         }
     }
-    #trackScheduleIfPending(context, delay = this.#options.debounceMs ?? 2_000) {
-        const followUp = this.#scheduleIfPending(context, delay).finally(() => {
+    #trackScheduleIfPending(context) {
+        const followUp = this.#scheduleIfPending(context).finally(() => {
             context.followUps.delete(followUp);
         });
         context.followUps.add(followUp);
         return followUp;
     }
-    async #run(context) {
-        if (this.#disposed || !this.#enabled)
+    async #run(context, bypassRetryAt = false) {
+        if (this.#disposed || !this.#enabled || context.storageBlocked)
             return;
         context.state = 'processing';
         context.message = undefined;
@@ -243,15 +488,18 @@ export class LocalMemoryCoordinator {
             lease = await acquireLocalMemoryLease(context.store.projectDirectory, this.#options.leaseOptions);
         }
         catch (error) {
+            context.writerBlocked = true;
             context.state = 'error';
             context.message = safeProcessingMessage(error);
             return;
         }
         if (!lease) {
+            context.writerBlocked = true;
             context.state = 'blocked';
             context.message = 'Another Felan process owns the memory writer';
             return;
         }
+        context.writerBlocked = false;
         const abort = new AbortController();
         context.abort = abort;
         const abortOnLeaseCompromise = () => {
@@ -259,13 +507,34 @@ export class LocalMemoryCoordinator {
         };
         lease.compromised.addEventListener('abort', abortOnLeaseCompromise, { once: true });
         let staging;
+        let retainedRun;
+        let published = false;
+        let attempt;
         try {
-            const snapshot = await context.store.processingSnapshot(this.#options.batchSize ?? 8);
+            const control = await this.#reconcileRunDiagnostics(context, lease);
+            if (control.autoDisabled) {
+                context.state = 'disabled';
+                return;
+            }
+            if (!bypassRetryAt && control.nextRetryAt && Date.parse(control.nextRetryAt) > this.#now()) {
+                context.state = 'scheduled';
+                return;
+            }
+            const snapshot = await context.store.processingSnapshot(this.#options.batchSize ?? 8, (checkpoint) => !isBlockedCheckpoint(context, checkpoint));
             if (snapshot.checkpoints.length === 0) {
                 context.state = 'idle';
                 return;
             }
-            staging = await context.store.createStagingDirectory();
+            retainedRun = await LocalMemoryRun.create({
+                projectDirectory: context.store.projectDirectory,
+                sessionDirectory: join(this.#options.sessionDir ?? join(this.#options.agentDir, 'sessions'), 'memory'),
+                projectKey: context.project.key,
+                projectRoot: context.project.canonicalRoot,
+                checkpoints: snapshot.checkpoints.map(({ checkpoint }) => checkpoint),
+                baseFingerprint: snapshot.fingerprint,
+                ...(control.lastDisabledRunId ? { protectedRunIds: [control.lastDisabledRunId] } : {}),
+            });
+            staging = retainedRun.workspace;
             const memoryDirectory = join(staging, '.memory');
             await hydrateMemoryDirectory(snapshot.artifact, memoryDirectory, { memoryPath: '.memory', mode: 'read' });
             const manifest = await materializeMemoryInput({
@@ -287,8 +556,24 @@ export class LocalMemoryCoordinator {
             if (processedCheckpoints.length === 0) {
                 context.state = 'blocked';
                 context.message = MEMORY_INPUT_BLOCKED_MESSAGE;
+                await retainedRun.finish('blocked', MEMORY_INPUT_BLOCKED_MESSAGE);
                 return;
             }
+            await retainedRun.record({ phase: 'model', checkpoints: processedCheckpoints.map(({ checkpoint }) => checkpoint) });
+            if (abort.signal.aborted || !this.#enabled || this.#disposed)
+                throw new Error('Memory processing was cancelled');
+            attempt = await context.store.beginAttempt(lease, {
+                runId: retainedRun.metadata.sessionId,
+                baseFingerprint: snapshot.fingerprint,
+                checkpoints: processedCheckpoints,
+                ...(bypassRetryAt ? { bypassRetryAt: true } : {}),
+            }, this.#now());
+            if (!attempt) {
+                await retainedRun.finish('cancelled', 'Memory inputs or processing policy changed before the attempt started');
+                return;
+            }
+            context.attempt = attempt;
+            this.#emitStatusChange();
             const result = await this.#dreamRunner({
                 stagingDirectory: staging,
                 memoryDirectory,
@@ -297,25 +582,28 @@ export class LocalMemoryCoordinator {
                 manifest,
                 modelRuntime: this.#options.modelRuntime,
                 ...(this.#selectedModel === undefined ? {} : { selectedModel: this.#selectedModel }),
+                sessionDirectory: join(this.#options.sessionDir ?? join(this.#options.agentDir, 'sessions'), 'memory'),
+                run: retainedRun,
                 signal: abort.signal,
             });
+            if (abort.signal.aborted || !this.#enabled || this.#disposed)
+                throw new Error('Memory processing was cancelled');
             if (result)
                 await hydrateMemoryDirectory(result, memoryDirectory, { replace: true, memoryPath: '.memory' });
+            await retainedRun.record({ phase: 'validate' });
             const artifact = await readMemoryDirectory(memoryDirectory, {
                 memoryPath: '.memory',
                 sourceSessionIds: allowedSourceIds(snapshot, manifest),
                 requireSources: true,
-                validateNavigation: true,
             });
-            const validation = validateMemoryArtifact(artifact, {
-                memoryPath: '.memory',
-                sourceSessionIds: allowedSourceIds(snapshot, manifest),
-                requireSources: true,
-                validateNavigation: true,
+            await retainedRun.record({
+                phase: 'publish',
+                outputFingerprint: createMemorySnapshot(artifact, '.memory').fingerprint,
             });
-            if (!validation.ok || !validation.artifact)
-                throw new Error('Memory output validation failed');
-            await context.store.commit(lease, snapshot.fingerprint, validation.artifact, processedCheckpoints);
+            await context.store.commit(lease, snapshot.fingerprint, artifact, processedCheckpoints, attempt);
+            published = true;
+            await context.store.finishAttempt(lease, attempt, { status: 'success' }, this.#now());
+            await retainedRun.finish('completed');
             if (manifest.failures.length > 0) {
                 context.state = 'blocked';
                 context.message = MEMORY_INPUT_BLOCKED_MESSAGE;
@@ -326,22 +614,76 @@ export class LocalMemoryCoordinator {
             this.#emitStatusChange();
         }
         catch (error) {
+            const leaseValid = await lease.verify();
+            const explicitlyCancelled = !this.#enabled || this.#disposed
+                || (abort.signal.aborted && !lease.compromised.aborted);
+            const unavailableModel = error instanceof MemoryModelUnavailableError;
+            const blockedBeforeInference = retainedRun !== undefined && attempt === undefined && !explicitlyCancelled;
+            if (attempt && (leaseValid || explicitlyCancelled)) {
+                try {
+                    const control = await context.store.finishAttempt(explicitlyCancelled || unavailableModel ? undefined : lease, attempt, explicitlyCancelled || unavailableModel ? { status: 'cancelled' }
+                        : { status: 'failure', reason: sanitizeMemoryDiagnostic(error) }, this.#now());
+                    published ||= control.lastOutcome?.runId === attempt.runId && control.lastOutcome.status === 'success';
+                }
+                catch {
+                    context.storageBlocked = true;
+                }
+            }
+            if (published) {
+                context.state = 'idle';
+                context.message = 'Memory was published, but its diagnostics could not be fully saved';
+                this.#emitStatusChange();
+                return;
+            }
             context.state = 'error';
-            context.message = !await lease.verify()
+            context.message = !leaseValid
                 ? safeProcessingMessage(new Error('Memory writer lease was lost'))
                 : safeProcessingMessage(error);
+            if (unavailableModel || blockedBeforeInference) {
+                context.state = 'blocked';
+                for (const checkpoint of retainedRun?.metadata.checkpoints ?? []) {
+                    context.blocked.set(checkpoint.sessionId, checkpoint);
+                }
+            }
+            if (!retainedRun) {
+                context.storageBlocked = true;
+                context.message = `Memory processing is paused: ${sanitizeMemoryDiagnostic(error)}`;
+            }
+            await retainedRun?.finish(explicitlyCancelled ? 'cancelled'
+                : unavailableModel || blockedBeforeInference ? 'blocked'
+                    : !leaseValid ? 'interrupted' : 'failed', error).catch(() => {
+                context.storageBlocked = true;
+                context.message = 'Memory diagnostics could not be saved; evidence remains pending';
+            });
+            this.#emitStatusChange();
         }
         finally {
             lease.compromised.removeEventListener('abort', abortOnLeaseCompromise);
             context.abort = undefined;
+            context.attempt = undefined;
             if (staging)
-                await rm(staging, { recursive: true, force: true }).catch(() => { });
+                await rm(join(staging, '.pi-memory-runtime'), { recursive: true, force: true }).catch(() => { });
             await lease.release();
         }
     }
+    async #reconcileRunDiagnostics(context, lease) {
+        let control = await context.store.readControl();
+        control = await context.store.reconcileAbandonedAttempt(lease, this.#now());
+        const options = {
+            projectDirectory: context.store.projectDirectory,
+            projectKey: context.project.key,
+            sessionDirectory: join(this.#options.sessionDir ?? join(this.#options.agentDir, 'sessions'), 'memory'),
+            ...(control.lastDisabledRunId ? { protectedRunIds: [control.lastDisabledRunId] } : {}),
+        };
+        if (control.lastOutcome)
+            await LocalMemoryRun.reconcile(options, control.lastOutcome);
+        await LocalMemoryRun.reconcileAbandoned(options, [
+            ...(control.activeAttempt ? [control.activeAttempt.runId] : []),
+            ...(control.lastOutcome ? [control.lastOutcome.runId] : []),
+        ], this.#now());
+        return control;
+    }
     async #recover(context) {
-        if (!this.#enabled)
-            return;
         const sessionDir = context.sessionDir ?? join(this.#options.agentDir, 'sessions');
         let sessions;
         try {
@@ -349,26 +691,33 @@ export class LocalMemoryCoordinator {
         }
         catch {
             context.message = 'Memory startup recovery could not inspect sessions';
-            return;
+            return true;
         }
+        if (this.#disposed || !this.#enabled)
+            return false;
         let recovered = false;
         const stableAfter = this.#options.recoveryStableMs ?? 30_000;
         for (const info of sessions) {
-            if (info.parentSessionPath || Date.now() - info.modified.getTime() < stableAfter)
+            if (this.#disposed || !this.#enabled)
+                return false;
+            if (info.parentSessionPath || this.#now() - info.modified.getTime() < stableAfter)
                 continue;
             try {
                 const manager = SessionManager.open(info.path, sessionDir, context.cwd);
                 const checkpoint = createSessionCheckpoint(manager);
                 if (!checkpoint)
                     continue;
+                if (this.#disposed || !this.#enabled)
+                    return false;
                 recovered = (await context.store.recordCheckpoint(checkpoint)) || recovered;
             }
             catch { }
         }
-        if (recovered) {
-            await this.#schedule(context);
+        if (recovered && !this.#disposed && this.#enabled) {
+            await this.#trackScheduleIfPending(context);
             this.#emitStatusChange();
         }
+        return true;
     }
     #emitStatusChange() {
         for (const listener of [...this.#statusListeners]) {
@@ -379,10 +728,10 @@ export class LocalMemoryCoordinator {
         }
     }
     #cancelScheduledRun(context) {
-        if (!context.timer)
-            return;
-        clearTimeout(context.timer);
+        if (context.timer)
+            clearTimeout(context.timer);
         context.timer = undefined;
+        context.timerDueAt = undefined;
     }
 }
 class LocalMemorySessionHost {