@felan-ai/ext-memory 0.10.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,3 @@
1
- import { removeIfPresent } from './context.js';
2
1
  import { createDefaultLocalMemoryDreamRunner, materializeMemoryInput } from './dreamer.js';
3
2
  import { LocalMemoryRun } from './run.js';
4
3
  import { readMemorySessionCheckpoint } from '../session-history.js';
@@ -12,48 +11,73 @@ import {} from '@felan-ai/agent-core';
12
11
  import { MemoryModelUnavailableError } from './dreamer.js';
13
12
  import { LocalMemoryControlError } from './store.js';
14
13
  import { sanitizeMemoryDiagnostic } from '../diagnostics.js';
15
- const MEMORY_INPUT_BLOCKED_MESSAGE = 'Some memory checkpoints could not be materialized; evidence remains pending';
14
+ const MEMORY_INPUT_BLOCKED_MESSAGE = 'Memory checkpoints are paused after a failed run; evidence remains pending';
16
15
  const MEMORY_AUTOMATIC_UPDATE_THRESHOLD = 5;
17
16
  const MEMORY_AUTOMATIC_INTERVAL_MS = 24 * 60 * 60 * 1_000;
18
17
  const MEMORY_DREAM_INPUT_MAX_BYTES = 10 * 1024 * 1024;
19
18
  const MEMORY_MAX_BATCH_SESSIONS = 8;
20
19
  export class LocalMemoryCoordinator {
21
- environment;
22
20
  #options;
23
21
  #contexts = new Map();
24
- #projections = new Map();
22
+ #attachments = new Set();
25
23
  #statusListeners = new Set();
26
- #dreamRunner;
27
24
  #batchSize;
28
25
  #enabled;
29
- #selectedModel;
30
- #scopedModels;
31
- #classifier;
32
26
  #disposed = false;
27
+ #disposal;
33
28
  #draining = false;
34
- #monitor;
35
- #monitoring;
36
29
  #enableVersion = 0;
37
- constructor(environment, options) {
38
- this.environment = environment;
30
+ constructor(options) {
39
31
  this.#options = options;
40
32
  this.#batchSize = boundedMemoryBatchSize(options.batchSize);
41
33
  this.#enabled = options.enabled !== false;
42
- this.#selectedModel = options.selectedModel;
43
- this.#scopedModels = options.scopedModels === undefined ? undefined : [...options.scopedModels];
44
- this.#classifier = options.classifier;
45
- this.#dreamRunner = options.dreamRunner ?? createDefaultLocalMemoryDreamRunner(this.environment);
46
- if (this.#enabled)
47
- this.#startMonitor();
48
34
  }
49
- createSessionHost(options) {
50
- return new LocalMemorySessionHost(this, options);
35
+ attach(options) {
36
+ if (this.#disposed)
37
+ throw new Error('Local memory coordinator is closed');
38
+ const state = {
39
+ ...options,
40
+ selectedModel: options.selectedModel,
41
+ scopedModels: options.scopedModels === undefined ? undefined : [...options.scopedModels],
42
+ classifier: options.classifier,
43
+ projections: new Map(),
44
+ listeners: new Set(),
45
+ draining: false,
46
+ closing: false,
47
+ detached: false,
48
+ detaching: undefined,
49
+ };
50
+ this.#attachments.add(state);
51
+ const attachment = {
52
+ createSessionHost: options => new LocalMemorySessionHost(attachment, options),
53
+ readCurrent: (cwd, root) => this.#readCurrent(state, cwd, root),
54
+ recordCheckpoint: (cwd, checkpoint) => this.#recordCheckpoint(state, cwd, checkpoint),
55
+ status: cwd => this.#status(state, cwd),
56
+ runNow: cwd => this.#runNow(state, cwd),
57
+ retryProject: cwd => this.#retryProject(state, cwd),
58
+ canonicalDirectory: async (cwd) => this.#store(await this.#context(state, cwd), state.environment).currentDirectory,
59
+ setSelectedModel: model => this.#setModelSelection(state, model, state.scopedModels, state.classifier),
60
+ setModelSelection: (model, scopedModels, classifier) => this.#setModelSelection(state, model, scopedModels, classifier),
61
+ subscribeStatusChanges: listener => {
62
+ if (state.detached || this.#disposed)
63
+ return () => { };
64
+ state.listeners.add(listener);
65
+ return () => { state.listeners.delete(listener); };
66
+ },
67
+ busy: () => [...this.#contexts.values()].some(context => context.driver === state || context.recoveryDriver === state),
68
+ settle: mode => this.#settleAttachment(state, mode),
69
+ detach: () => {
70
+ state.detaching ??= this.#detach(state);
71
+ return state.detaching;
72
+ },
73
+ };
74
+ return attachment;
51
75
  }
52
76
  isEnabled() {
53
77
  return this.#enabled;
54
78
  }
55
79
  hasActiveWork() {
56
- return !!this.#monitoring || [...this.#contexts.values()].some(context => !!context.run || !!context.ready || !!context.recovery || context.followUps.size > 0);
80
+ return [...this.#contexts.values()].some(context => !!context.run || !!context.ready || !!context.recovery || context.followUps.size > 0);
57
81
  }
58
82
  subscribeStatusChanges(listener) {
59
83
  if (this.#disposed)
@@ -61,34 +85,28 @@ export class LocalMemoryCoordinator {
61
85
  this.#statusListeners.add(listener);
62
86
  return () => this.#statusListeners.delete(listener);
63
87
  }
64
- setModelSelection(model, scopedModels, classifier) {
65
- this.#scopedModels = scopedModels === undefined ? undefined : [...scopedModels];
66
- this.#classifier = classifier;
67
- this.setSelectedModel(model);
68
- }
69
- setSelectedModel(model) {
70
- this.#selectedModel = model;
88
+ #setModelSelection(attachment, model, scopedModels, classifier) {
89
+ if (attachment.detached || this.#disposed)
90
+ return;
91
+ attachment.selectedModel = model;
92
+ attachment.scopedModels = scopedModels === undefined ? undefined : [...scopedModels];
93
+ attachment.classifier = classifier;
71
94
  for (const context of this.#contexts.values()) {
95
+ if (!context.attachments.has(attachment))
96
+ continue;
72
97
  context.blocked.clear();
73
98
  if (context.initialized)
74
- void this.#trackScheduleIfPending(context);
99
+ void this.#trackScheduleIfPending(context, attachment.environment);
75
100
  }
76
101
  }
77
102
  setEnabled(enabled) {
78
103
  this.#enableVersion += 1;
79
- enabled = enabled && !this.#options.unavailableReason;
80
104
  this.#enabled = enabled;
81
- if (enabled)
82
- this.#startMonitor();
83
- else if (this.#monitor) {
84
- clearInterval(this.#monitor);
85
- this.#monitor = undefined;
86
- }
87
105
  for (const context of this.#contexts.values()) {
88
106
  if (!enabled)
89
107
  context.recoveryAbort?.abort();
90
108
  if (!enabled)
91
- this.#cancelScheduledRun(context);
109
+ this.#cancelTimers(context);
92
110
  if (!enabled)
93
111
  this.#cancelContext(context, 'Memory processing was disabled', true);
94
112
  if (enabled) {
@@ -108,26 +126,27 @@ export class LocalMemoryCoordinator {
108
126
  }
109
127
  this.#emitStatusChange();
110
128
  }
111
- async status(cwd) {
129
+ async #status(attachment, cwd) {
112
130
  try {
113
- const context = await this.#context(cwd);
114
- const state = await context.store.status();
115
- const control = await context.store.readControl();
131
+ const context = await this.#context(attachment, cwd);
132
+ const store = this.#store(context, attachment.environment);
133
+ const state = await store.status();
134
+ const control = await store.readControl();
116
135
  const processed = Object.values(state.processed);
117
136
  const lastProcessedAt = processed
118
137
  .map((entry) => entry.processedAt)
119
138
  .sort()
120
139
  .at(-1);
121
140
  return {
122
- enabled: this.#enabled && !control.autoDisabled,
123
- state: !this.#enabled || control.autoDisabled ? 'disabled'
141
+ enabled: this.#enabled && attachment.scheduling && !control.autoDisabled,
142
+ state: !this.#enabled || !attachment.scheduling || control.autoDisabled ? 'disabled'
124
143
  : control.nextRetryAt && Date.parse(control.nextRetryAt) > this.#now() ? 'scheduled'
125
144
  : context.state === 'error' ? 'error'
126
145
  : control.activeAttempt ? 'processing' : context.state,
127
146
  pendingCheckpoints: Object.keys(state.pending).length,
128
147
  memoryFingerprint: state.memoryFingerprint,
129
148
  ...(lastProcessedAt === undefined ? {} : { lastProcessedAt }),
130
- ...(context.message || this.#options.unavailableReason ? { message: context.message ?? this.#options.unavailableReason } : {}),
149
+ ...(context.message || attachment.unavailableReason ? { message: context.message ?? attachment.unavailableReason } : {}),
131
150
  consecutiveFailures: control.consecutiveFailures,
132
151
  ...(control.nextRetryAt ? { nextRetryAt: control.nextRetryAt } : {}),
133
152
  ...(control.autoDisabled ? { autoDisabled: control.autoDisabled } : {}),
@@ -138,37 +157,39 @@ export class LocalMemoryCoordinator {
138
157
  }
139
158
  catch (error) {
140
159
  return {
141
- enabled: this.#enabled,
142
- state: this.#enabled ? 'error' : 'disabled',
160
+ enabled: this.#enabled && attachment.scheduling,
161
+ state: this.#enabled && attachment.scheduling ? 'error' : 'disabled',
143
162
  pendingCheckpoints: 0,
144
163
  message: error instanceof LocalMemoryControlError ? error.message : 'Local memory storage is unavailable',
145
164
  };
146
165
  }
147
166
  }
148
- async readCurrent(cwd, sessionStorageRoot) {
149
- const context = await this.#context(cwd);
150
- const snapshot = await context.store.readCurrent();
167
+ async #readCurrent(attachment, cwd, sessionStorageRoot) {
168
+ const context = await this.#context(attachment, cwd);
169
+ const store = this.#store(context, attachment.environment);
170
+ const snapshot = await store.readCurrent();
151
171
  const projectionKey = `${context.project.key}:${sessionStorageRoot}`;
152
- const memoryPath = join(sessionStorageRoot, context.store.projectionName);
153
- if (this.#projections.get(projectionKey) !== snapshot.fingerprint) {
154
- const projection = await context.store.projectTo(sessionStorageRoot, snapshot);
155
- this.#projections.set(projectionKey, snapshot.fingerprint);
172
+ const memoryPath = join(sessionStorageRoot, store.projectionName);
173
+ if (attachment.projections.get(projectionKey) !== snapshot.fingerprint) {
174
+ const projection = await store.projectTo(sessionStorageRoot, snapshot);
175
+ if (!attachment.detached)
176
+ attachment.projections.set(projectionKey, snapshot.fingerprint);
156
177
  return projection;
157
178
  }
158
179
  return createMemoryProjectionSnapshot(snapshot, memoryPath);
159
180
  }
160
- async recordCheckpoint(cwd, checkpoint) {
161
- const context = await this.#context(cwd);
162
- if (await context.store.recordCheckpoint(checkpoint)) {
181
+ async #recordCheckpoint(attachment, cwd, checkpoint) {
182
+ const context = await this.#context(attachment, cwd);
183
+ if (await this.#store(context, attachment.environment).recordCheckpoint(checkpoint)) {
163
184
  context.blocked.delete(checkpoint.sessionId);
164
- await this.#trackScheduleIfPending(context);
185
+ await this.#trackScheduleIfPending(context, attachment.environment);
165
186
  this.#emitStatusChange();
166
187
  }
167
188
  }
168
- async runNow(cwd) {
169
- if (!this.#enabled || this.#draining)
170
- return this.status(cwd);
171
- const context = await this.#context(cwd);
189
+ async #runNow(attachment, cwd) {
190
+ if (!this.#eligible(attachment))
191
+ return this.#status(attachment, cwd);
192
+ const context = await this.#context(attachment, cwd);
172
193
  this.#cancelScheduledRun(context);
173
194
  while (context.run || context.followUps.size > 0) {
174
195
  await Promise.allSettled([
@@ -178,61 +199,114 @@ export class LocalMemoryCoordinator {
178
199
  this.#cancelScheduledRun(context);
179
200
  }
180
201
  context.blocked.clear();
181
- const run = this.#run(context, true);
182
- context.run = run;
183
- try {
184
- await run;
185
- }
186
- finally {
187
- if (context.run === run)
188
- context.run = undefined;
189
- await this.#trackScheduleIfPending(context);
190
- }
191
- return this.status(cwd);
202
+ context.storageBlocked = false;
203
+ context.message = undefined;
204
+ await this.#launchRun(context, true, attachment);
205
+ return this.#status(attachment, cwd);
192
206
  }
193
- async retryProject(cwd) {
194
- if (!this.#enabled || this.#draining)
195
- return this.status(cwd);
196
- const context = await this.#context(cwd);
197
- if (!this.#enabled || this.#disposed || this.#draining)
198
- return this.status(cwd);
207
+ async #retryProject(attachment, cwd) {
208
+ if (!this.#eligible(attachment))
209
+ return this.#status(attachment, cwd);
210
+ const context = await this.#context(attachment, cwd);
211
+ if (!this.#eligible(attachment))
212
+ return this.#status(attachment, cwd);
199
213
  const enableVersion = this.#enableVersion;
200
214
  this.#cancelScheduledRun(context);
201
- try {
202
- await context.store.resetProjectControl(this.#now(), () => this.#enabled && !this.#disposed && !this.#draining && this.#enableVersion === enableVersion);
203
- }
204
- catch (error) {
205
- throw error;
206
- }
215
+ await this.#store(context, attachment.environment).resetProjectControl(this.#now(), () => this.#eligible(attachment) && this.#enableVersion === enableVersion);
216
+ this.#emitStatusChange();
207
217
  context.abort?.abort(new Error('Memory processing was explicitly retried'));
208
218
  if (context.run)
209
219
  await context.run;
210
220
  context.blocked.clear();
211
221
  context.storageBlocked = false;
212
222
  context.message = undefined;
213
- await this.runNow(cwd);
214
- return this.status(cwd);
223
+ await this.#runNow(attachment, cwd);
224
+ return this.#status(attachment, cwd);
225
+ }
226
+ #eligible(attachment) {
227
+ return this.#enabled && !this.#disposed && !this.#draining
228
+ && attachment.scheduling && !attachment.draining && !attachment.closing && !attachment.detached;
229
+ }
230
+ #driver(context, preferred) {
231
+ if (preferred && context.attachments.has(preferred) && this.#eligible(preferred))
232
+ return preferred;
233
+ return [...this.#attachments].reverse().find(attachment => context.attachments.has(attachment) && this.#eligible(attachment));
234
+ }
235
+ #liveMember(context) {
236
+ return [...this.#attachments].reverse().find(attachment => context.attachments.has(attachment)
237
+ && attachment.scheduling && !attachment.draining && !attachment.closing && !attachment.detached);
238
+ }
239
+ #runEnvironment(context, driver) {
240
+ // The daemon seals closing runtimes outside their own cleanup async context.
241
+ const current = () => (driver.closing || driver.detached ? this.#liveMember(context) ?? driver : driver).environment;
242
+ const environment = {
243
+ get runtime() { return current().runtime; },
244
+ get sessions() { return current().sessions; },
245
+ };
246
+ Object.defineProperty(environment, 'openWorker', { get: () => current().openWorker });
247
+ return environment;
248
+ }
249
+ async #settleAttachment(attachment, mode) {
250
+ attachment.draining = true;
251
+ if (mode === 'force')
252
+ attachment.closing = true;
253
+ for (const context of this.#contexts.values()) {
254
+ if (context.recoveryDriver === attachment)
255
+ context.recoveryAbort?.abort();
256
+ if (mode === 'force' && context.driver === attachment) {
257
+ this.#cancelContext(context, 'Memory processing was cancelled during shutdown', false);
258
+ }
259
+ if (!this.#driver(context))
260
+ this.#cancelTimers(context);
261
+ }
262
+ while (true) {
263
+ const work = [...this.#contexts.values()].flatMap(context => [
264
+ ...(context.driver === attachment ? [...(context.run ? [context.run] : []), ...context.followUps] : []),
265
+ ...(context.recoveryDriver === attachment && context.recovery ? [context.recovery] : []),
266
+ ]);
267
+ if (!work.length)
268
+ break;
269
+ await Promise.allSettled(work);
270
+ }
271
+ for (const context of this.#contexts.values()) {
272
+ if (mode === 'force' && context.failedDriver === attachment) {
273
+ context.storageBlocked = false;
274
+ context.blocked.clear();
275
+ if (context.state === 'error' || context.state === 'blocked')
276
+ context.state = 'idle';
277
+ context.message = undefined;
278
+ context.failedDriver = undefined;
279
+ }
280
+ if (!context.attachments.has(attachment) || !context.initialized)
281
+ continue;
282
+ this.#startRecovery(context);
283
+ void this.#trackScheduleIfPending(context);
284
+ }
215
285
  }
216
- async canonicalDirectory(cwd) {
217
- const context = await this.#context(cwd);
218
- return context.store.currentDirectory;
286
+ async #detach(attachment) {
287
+ attachment.closing = true;
288
+ await this.#settleAttachment(attachment, 'force');
289
+ attachment.detached = true;
290
+ this.#attachments.delete(attachment);
291
+ attachment.listeners.clear();
292
+ attachment.projections.clear();
293
+ for (const context of this.#contexts.values()) {
294
+ context.attachments.delete(attachment);
295
+ if (!this.#driver(context))
296
+ this.#cancelTimers(context);
297
+ }
219
298
  }
220
299
  async drain() {
221
300
  this.#draining = true;
222
- if (this.#monitor)
223
- clearInterval(this.#monitor);
224
- this.#monitor = undefined;
225
301
  for (const context of this.#contexts.values()) {
226
302
  context.recoveryAbort?.abort();
227
- this.#cancelScheduledRun(context);
303
+ this.#cancelTimers(context);
228
304
  }
229
305
  while (true) {
230
306
  const work = [...this.#contexts.values()].flatMap(context => [
231
307
  ...(context.run ? [context.run] : []), ...(context.ready ? [context.ready] : []),
232
308
  ...(context.recovery ? [context.recovery] : []), ...context.followUps,
233
309
  ]);
234
- if (this.#monitoring)
235
- work.push(this.#monitoring);
236
310
  if (!work.length)
237
311
  return;
238
312
  const results = await Promise.allSettled(work);
@@ -241,103 +315,121 @@ export class LocalMemoryCoordinator {
241
315
  throw new AggregateError(failures, 'Memory drain could not settle all work.');
242
316
  }
243
317
  }
244
- async dispose() {
245
- if (this.#disposed)
246
- return;
318
+ dispose() {
319
+ this.#disposal ??= this.#dispose();
320
+ return this.#disposal;
321
+ }
322
+ async #dispose() {
247
323
  this.#disposed = true;
248
- if (this.#monitor)
249
- clearInterval(this.#monitor);
250
- this.#monitor = undefined;
251
324
  for (const context of this.#contexts.values()) {
252
325
  context.recoveryAbort?.abort();
253
- this.#cancelScheduledRun(context);
326
+ this.#cancelTimers(context);
254
327
  this.#cancelContext(context, 'Memory processing was cancelled during shutdown', false);
255
328
  }
256
329
  while (true) {
257
- const work = [...this.#contexts.values()].flatMap((context) => [
330
+ const work = [...this.#contexts.values()].flatMap(context => [
258
331
  ...(context.run ? [context.run] : []),
259
332
  ...(context.ready ? [context.ready] : []),
260
333
  ...(context.recovery ? [context.recovery] : []),
261
334
  ...context.followUps,
262
335
  ]);
263
- if (this.#monitoring)
264
- work.push(this.#monitoring);
265
- if (work.length === 0)
336
+ if (!work.length)
266
337
  break;
267
338
  await Promise.allSettled(work);
268
339
  }
340
+ for (const attachment of this.#attachments) {
341
+ attachment.detached = true;
342
+ attachment.listeners.clear();
343
+ attachment.projections.clear();
344
+ }
345
+ this.#attachments.clear();
269
346
  this.#statusListeners.clear();
270
347
  }
271
- async #context(cwd) {
272
- if (this.#disposed)
348
+ async #context(attachment, cwd) {
349
+ if (this.#disposed || attachment.detached)
273
350
  throw new Error('Local memory coordinator is closed');
274
- const project = await resolveLocalMemoryProject(this.environment, cwd);
275
- if (this.#disposed)
351
+ const project = await resolveLocalMemoryProject(attachment.environment, cwd);
352
+ if (this.#disposed || attachment.detached)
276
353
  throw new Error('Local memory coordinator is closed');
277
354
  const existing = this.#contexts.get(project.key);
278
355
  if (existing) {
279
- await this.#initialize(existing);
356
+ existing.attachments.add(attachment);
357
+ await this.#initialize(existing, attachment);
280
358
  return existing;
281
359
  }
282
- const store = new LocalMemoryStore(this.environment, this.#options.agentDir, project, {
283
- ...(this.#options.retryDelaysMs ? { retryDelaysMs: this.#options.retryDelaysMs } : {}),
284
- });
285
360
  const context = {
286
361
  project,
287
- cwd,
288
- ...(this.#options.sessionDir === undefined ? {} : { sessionDir: this.#options.sessionDir }),
289
- store,
362
+ stores: new WeakMap(),
363
+ attachments: new Set([attachment]),
364
+ driver: undefined,
365
+ failedDriver: undefined,
366
+ recoveryDriver: undefined,
290
367
  ready: undefined,
291
368
  initialized: false,
292
369
  recovery: undefined,
293
370
  recoveryAbort: undefined,
294
371
  recoveryRetryAt: undefined,
372
+ recoveryTimer: undefined,
295
373
  recoveryCompleted: false,
296
374
  diagnosticsReconciled: false,
297
375
  timer: undefined,
298
376
  timerDueAt: undefined,
299
377
  run: undefined,
378
+ runEnvironment: undefined,
300
379
  followUps: new Set(),
301
380
  abort: undefined,
302
381
  attempt: undefined,
303
382
  storageBlocked: false,
304
383
  writerBlocked: false,
305
- statusSignature: undefined,
384
+ writerTimer: undefined,
306
385
  blocked: new Map(),
307
386
  state: this.#enabled ? 'idle' : 'disabled',
308
387
  message: undefined,
309
388
  };
310
389
  this.#contexts.set(project.key, context);
311
- await this.#initialize(context);
390
+ await this.#initialize(context, attachment);
312
391
  return context;
313
392
  }
314
- async #initialize(context) {
393
+ #store(context, environment) {
394
+ let store = context.stores.get(environment);
395
+ if (!store) {
396
+ store = new LocalMemoryStore(environment, this.#options.agentDir, context.project, {
397
+ ...(this.#options.retryDelaysMs ? { retryDelaysMs: this.#options.retryDelaysMs } : {}),
398
+ });
399
+ context.stores.set(environment, store);
400
+ }
401
+ return store;
402
+ }
403
+ async #initialize(context, attachment) {
315
404
  if (context.ready) {
316
405
  await context.ready;
317
406
  return;
318
407
  }
319
408
  if (context.initialized && context.diagnosticsReconciled) {
320
409
  this.#startRecovery(context);
410
+ void this.#trackScheduleIfPending(context, attachment.environment);
321
411
  return;
322
412
  }
323
- const ready = this.#initializeContext(context).finally(() => {
413
+ const ready = this.#initializeContext(context, attachment).finally(() => {
324
414
  if (context.ready === ready)
325
415
  context.ready = undefined;
326
416
  });
327
417
  context.ready = ready;
328
418
  await ready;
329
419
  }
330
- async #initializeContext(context) {
420
+ async #initializeContext(context, attachment) {
421
+ const environment = attachment.environment;
422
+ const store = this.#store(context, environment);
331
423
  if (!context.initialized) {
332
- await context.store.initialize(this.#now()).then(() => {
424
+ await store.initialize(this.#now()).then(() => {
333
425
  context.initialized = true;
334
426
  context.state = this.#enabled ? 'idle' : 'disabled';
335
427
  context.message = undefined;
336
428
  }).catch(async (error) => {
337
429
  if (!(error instanceof LocalMemoryControlError))
338
430
  throw error;
339
- const current = await context.store.readCurrent();
340
- if (current.fingerprint !== (await context.store.status()).memoryFingerprint)
431
+ const current = await store.readCurrent();
432
+ if (current.fingerprint !== (await store.status()).memoryFingerprint)
341
433
  throw error;
342
434
  context.message = error.message;
343
435
  });
@@ -346,11 +438,16 @@ export class LocalMemoryCoordinator {
346
438
  return;
347
439
  if (!context.diagnosticsReconciled) {
348
440
  try {
349
- const lease = await acquireLocalMemoryLease(this.environment, context.store.projectDirectory, this.#options.leaseOptions);
441
+ const lease = await acquireLocalMemoryLease(environment, store.projectDirectory, this.#options.leaseOptions);
350
442
  if (lease) {
351
443
  try {
352
- await this.#reconcileRunDiagnostics(context, lease);
444
+ await this.#reconcileRunDiagnostics(context, environment, lease);
353
445
  context.diagnosticsReconciled = true;
446
+ if (context.storageBlocked) {
447
+ context.storageBlocked = false;
448
+ context.failedDriver = undefined;
449
+ context.message = undefined;
450
+ }
354
451
  }
355
452
  finally {
356
453
  await lease.release();
@@ -358,24 +455,34 @@ export class LocalMemoryCoordinator {
358
455
  }
359
456
  }
360
457
  catch (error) {
361
- context.diagnosticsReconciled = true;
362
458
  context.storageBlocked = true;
459
+ context.failedDriver = attachment;
363
460
  context.message = `Memory diagnostics recovery is blocked: ${sanitizeMemoryDiagnostic(error)}`;
364
461
  }
365
462
  }
366
463
  if (this.#disposed)
367
464
  return;
368
465
  if (this.#enabled)
369
- void this.#trackScheduleIfPending(context);
466
+ void this.#trackScheduleIfPending(context, environment);
370
467
  this.#startRecovery(context);
371
468
  }
372
469
  #startRecovery(context) {
373
- if (this.#disposed || this.#draining || !this.#enabled || this.#options.recover === false
374
- || context.recoveryCompleted || context.recovery
375
- || context.recoveryRetryAt !== undefined && this.#now() < context.recoveryRetryAt)
470
+ if (this.#options.recover === false || context.recoveryCompleted || context.recovery)
471
+ return;
472
+ const driver = this.#driver(context);
473
+ if (!driver)
474
+ return;
475
+ if (context.recoveryRetryAt !== undefined && this.#now() < context.recoveryRetryAt) {
476
+ this.#scheduleRecoveryRetry(context);
376
477
  return;
478
+ }
479
+ if (context.recoveryTimer)
480
+ clearTimeout(context.recoveryTimer);
481
+ context.recoveryTimer = undefined;
482
+ context.recoveryDriver = driver;
377
483
  const abort = new AbortController();
378
- const recovery = this.#recover(context, abort.signal).then((completed) => {
484
+ context.recoveryAbort = abort;
485
+ const recovery = this.#recover(context, driver, abort.signal).then(completed => {
379
486
  if (completed)
380
487
  context.recoveryCompleted = true;
381
488
  }).catch(() => {
@@ -384,71 +491,70 @@ export class LocalMemoryCoordinator {
384
491
  if (context.recovery === recovery) {
385
492
  context.recovery = undefined;
386
493
  context.recoveryAbort = undefined;
494
+ context.recoveryDriver = undefined;
387
495
  }
496
+ if (abort.signal.aborted)
497
+ this.#startRecovery(context);
498
+ else
499
+ this.#scheduleRecoveryRetry(context);
500
+ this.#emitStatusChange();
388
501
  });
389
502
  context.recovery = recovery;
390
- context.recoveryAbort = abort;
391
503
  }
392
- #now() { return this.#options.now?.() ?? Date.now(); }
393
- #startMonitor() {
394
- if (this.#disposed || this.#draining || this.#monitor)
504
+ #scheduleRecoveryRetry(context) {
505
+ if (context.recoveryTimer || context.recoveryCompleted || context.recoveryRetryAt === undefined || !this.#driver(context))
395
506
  return;
396
- const interval = this.#options.monitorIntervalMs ?? 2_000;
397
- if (!Number.isFinite(interval) || interval < 1)
398
- throw new Error('Memory monitor interval must be positive');
399
- this.#monitor = setInterval(() => {
400
- if (this.#monitoring)
507
+ const timer = setTimeout(() => {
508
+ if (context.recoveryTimer !== timer)
401
509
  return;
402
- this.#monitoring = this.#refresh().catch(() => { }).finally(() => { this.#monitoring = undefined; });
403
- }, interval);
404
- this.#monitor.unref?.();
510
+ context.recoveryTimer = undefined;
511
+ this.#startRecovery(context);
512
+ }, Math.max(0, context.recoveryRetryAt - this.#now()));
513
+ context.recoveryTimer = timer;
514
+ timer.unref?.();
405
515
  }
406
- async #refresh() {
407
- if (this.#disposed || this.#draining)
516
+ #scheduleWriterRetry(context) {
517
+ if (context.writerTimer || !this.#driver(context))
408
518
  return;
409
- for (const context of this.#contexts.values()) {
410
- try {
411
- await this.#initialize(context);
412
- const observedAttempt = context.attempt;
413
- const control = await context.store.readControl();
414
- if (observedAttempt && context.attempt === observedAttempt && (!this.#enabled || control.autoDisabled
415
- || control.activeAttempt?.runId !== observedAttempt.runId
416
- || control.activeAttempt?.generation !== observedAttempt.generation)) {
417
- this.#cancelContext(context, 'Memory processing was cancelled by another session', true);
418
- }
419
- context.writerBlocked = false;
420
- if (context.initialized)
421
- await this.#trackScheduleIfPending(context);
422
- const signature = JSON.stringify([this.#enabled, context.state, context.message, control.updatedAt]);
423
- if (signature !== context.statusSignature) {
424
- context.statusSignature = signature;
425
- this.#emitStatusChange();
426
- }
427
- }
428
- catch (error) {
429
- context.state = 'error';
430
- context.message = sanitizeMemoryDiagnostic(error);
431
- context.abort?.abort(error);
432
- this.#emitStatusChange();
519
+ context.writerBlocked = true;
520
+ const timer = setTimeout(() => {
521
+ if (context.writerTimer !== timer)
522
+ return;
523
+ context.writerTimer = undefined;
524
+ context.writerBlocked = false;
525
+ if (context.run) {
526
+ void this.#trackScheduleIfPending(context);
527
+ return;
433
528
  }
434
- }
529
+ context.diagnosticsReconciled = false;
530
+ const driver = this.#driver(context);
531
+ if (driver)
532
+ void this.#initialize(context, driver).catch(() => { });
533
+ }, this.#options.writerRetryMs ?? 30_000);
534
+ context.writerTimer = timer;
535
+ timer.unref?.();
435
536
  }
537
+ #now() { return this.#options.now?.() ?? Date.now(); }
436
538
  #cancelContext(context, reason, cancelPersisted) {
437
539
  const alreadyAborted = context.abort?.signal.aborted ?? false;
438
540
  context.abort?.abort(new Error(reason));
439
541
  if (alreadyAborted || !context.initialized || (!cancelPersisted && !context.attempt))
440
542
  return;
543
+ const environment = context.runEnvironment ?? this.#liveMember(context)?.environment;
544
+ if (!environment)
545
+ return;
546
+ const store = this.#store(context, environment);
441
547
  const cancellation = cancelPersisted
442
- ? context.store.cancelActiveAttempt(this.#now())
443
- : context.store.finishAttempt(undefined, context.attempt, { status: 'cancelled', reason }, this.#now());
548
+ ? store.cancelActiveAttempt(this.#now())
549
+ : store.finishAttempt(undefined, context.attempt, { status: 'cancelled', reason }, this.#now());
444
550
  const followUp = cancellation
445
- .then(() => { })
551
+ .then(() => { this.#emitStatusChange(); })
446
552
  .catch(() => { })
447
553
  .finally(() => { context.followUps.delete(followUp); });
448
554
  context.followUps.add(followUp);
449
555
  }
450
556
  async #schedule(context, dueAt) {
451
- if (this.#disposed || this.#draining || !this.#enabled || context.run || context.storageBlocked)
557
+ if (!this.#driver(context) || context.run || context.storageBlocked)
452
558
  return;
453
559
  if (context.timer && context.timerDueAt === dueAt)
454
560
  return;
@@ -460,28 +566,26 @@ export class LocalMemoryCoordinator {
460
566
  return;
461
567
  context.timer = undefined;
462
568
  context.timerDueAt = undefined;
463
- const run = this.#run(context).catch((error) => {
464
- context.state = 'error';
465
- context.storageBlocked = true;
466
- context.message = sanitizeMemoryDiagnostic(error);
467
- this.#emitStatusChange();
468
- }).finally(async () => {
469
- context.run = undefined;
470
- await this.#trackScheduleIfPending(context);
471
- });
472
- context.run = run;
569
+ void this.#launchRun(context)?.catch(() => { });
473
570
  }, delay);
474
571
  context.timer = timer;
475
572
  context.timerDueAt = dueAt;
476
573
  timer.unref?.();
477
574
  }
478
- async #scheduleIfPending(context) {
479
- if (!this.#enabled || this.#disposed || this.#draining)
575
+ async #scheduleIfPending(context, environment) {
576
+ const driver = this.#driver(context);
577
+ if (!driver) {
578
+ this.#cancelTimers(context);
480
579
  return;
580
+ }
581
+ // Callers capture their environment before awaiting; a member sealed since then cannot read the store.
582
+ const owner = environment && [...context.attachments].find(attachment => attachment.environment === environment);
583
+ const source = owner && (owner.closing || owner.detached) ? (this.#liveMember(context) ?? owner).environment : environment;
481
584
  try {
482
- const control = await context.store.readControl();
585
+ const store = this.#store(context, source ?? driver.environment);
586
+ const control = await store.readControl();
483
587
  if (control.autoDisabled) {
484
- this.#cancelScheduledRun(context);
588
+ this.#cancelTimers(context);
485
589
  context.state = 'disabled';
486
590
  return;
487
591
  }
@@ -489,13 +593,21 @@ export class LocalMemoryCoordinator {
489
593
  return;
490
594
  if (context.writerBlocked)
491
595
  return;
492
- const state = await context.store.status();
596
+ const state = await store.status();
597
+ if (context.state === 'error' && context.message === 'Local memory storage is unavailable') {
598
+ context.state = 'idle';
599
+ context.message = undefined;
600
+ }
493
601
  const pending = Object.values(state.pending);
602
+ if (!this.#driver(context))
603
+ return;
494
604
  if (control.activeAttempt) {
495
605
  this.#cancelScheduledRun(context);
606
+ if (!context.run)
607
+ this.#scheduleWriterRetry(context);
496
608
  }
497
609
  else if (pending.some(({ checkpoint }) => !isBlockedCheckpoint(context, checkpoint))) {
498
- const acceptedUpdates = await context.store.pendingAcceptedUpdates((checkpoint) => !isBlockedCheckpoint(context, checkpoint));
610
+ const acceptedUpdates = await store.pendingAcceptedUpdates((checkpoint) => !isBlockedCheckpoint(context, checkpoint));
499
611
  if (acceptedUpdates < MEMORY_AUTOMATIC_UPDATE_THRESHOLD) {
500
612
  this.#cancelScheduledRun(context);
501
613
  if (context.state === 'scheduled')
@@ -530,37 +642,84 @@ export class LocalMemoryCoordinator {
530
642
  context.message = 'Local memory storage is unavailable';
531
643
  }
532
644
  }
533
- #trackScheduleIfPending(context) {
534
- const followUp = this.#scheduleIfPending(context).finally(() => {
645
+ #trackScheduleIfPending(context, environment) {
646
+ const followUp = this.#scheduleIfPending(context, environment).finally(() => {
535
647
  context.followUps.delete(followUp);
536
648
  });
537
649
  context.followUps.add(followUp);
538
650
  return followUp;
539
651
  }
540
- async #run(context, bypassRetryAt = false) {
652
+ #launchRun(context, bypassRetryAt = false, preferred) {
653
+ if (context.run || context.storageBlocked || preferred && !this.#eligible(preferred))
654
+ return;
655
+ const driver = this.#driver(context, preferred);
656
+ if (!driver)
657
+ return;
658
+ context.driver = driver;
659
+ context.failedDriver = undefined;
660
+ const environment = this.#runEnvironment(context, driver);
661
+ context.runEnvironment = environment;
662
+ const run = this.#run(context, driver, environment, bypassRetryAt).catch((error) => {
663
+ context.state = 'error';
664
+ context.failedDriver = driver;
665
+ if (!bypassRetryAt && !driver.closing && !driver.detached)
666
+ context.storageBlocked = true;
667
+ context.message = sanitizeMemoryDiagnostic(error);
668
+ throw error;
669
+ }).finally(async () => {
670
+ if (context.run === settled) {
671
+ context.run = undefined;
672
+ context.driver = undefined;
673
+ context.runEnvironment = undefined;
674
+ context.abort = undefined;
675
+ }
676
+ await this.#trackScheduleIfPending(context);
677
+ this.#emitStatusChange();
678
+ });
679
+ const settled = run.catch(() => { });
680
+ context.run = settled;
681
+ return run;
682
+ }
683
+ async #run(context, driver, environment, bypassRetryAt) {
541
684
  if (this.#disposed || this.#draining || !this.#enabled || context.storageBlocked)
542
685
  return;
543
686
  context.state = 'processing';
544
687
  context.message = undefined;
688
+ const abort = new AbortController();
689
+ context.abort = abort;
690
+ const store = this.#store(context, environment);
545
691
  let lease;
692
+ const cancelled = () => driver.closing || driver.detached || !this.#enabled || this.#disposed
693
+ || abort.signal.aborted && !lease?.compromised.aborted;
546
694
  try {
547
- lease = await acquireLocalMemoryLease(this.environment, context.store.projectDirectory, this.#options.leaseOptions);
695
+ lease = await acquireLocalMemoryLease(environment, store.projectDirectory, this.#options.leaseOptions);
548
696
  }
549
697
  catch (error) {
550
- context.writerBlocked = true;
698
+ if (cancelled()) {
699
+ context.state = 'idle';
700
+ context.message = driver.closing || driver.detached || this.#disposed
701
+ ? 'Memory processing was cancelled during shutdown' : 'Memory processing was cancelled';
702
+ return;
703
+ }
704
+ this.#scheduleWriterRetry(context);
551
705
  context.state = 'error';
706
+ context.failedDriver = driver;
552
707
  context.message = safeProcessingMessage(error);
553
708
  return;
554
709
  }
555
710
  if (!lease) {
556
- context.writerBlocked = true;
711
+ this.#scheduleWriterRetry(context);
557
712
  context.state = 'blocked';
713
+ context.failedDriver = driver;
558
714
  context.message = 'Another Felan Code process owns the memory writer';
559
715
  return;
560
716
  }
561
717
  context.writerBlocked = false;
562
- const abort = new AbortController();
563
- context.abort = abort;
718
+ if (context.failedDriver === driver)
719
+ context.failedDriver = undefined;
720
+ if (context.writerTimer)
721
+ clearTimeout(context.writerTimer);
722
+ context.writerTimer = undefined;
564
723
  const abortOnLeaseCompromise = () => {
565
724
  abort.abort(new Error('Memory writer lease was lost'));
566
725
  };
@@ -570,7 +729,9 @@ export class LocalMemoryCoordinator {
570
729
  let published = false;
571
730
  let attempt;
572
731
  try {
573
- const control = await this.#reconcileRunDiagnostics(context, lease);
732
+ if (abort.signal.aborted || !this.#enabled || this.#disposed)
733
+ return;
734
+ const control = await this.#reconcileRunDiagnostics(context, environment, lease);
574
735
  if (control.autoDisabled) {
575
736
  context.state = 'disabled';
576
737
  return;
@@ -579,24 +740,25 @@ export class LocalMemoryCoordinator {
579
740
  context.state = 'scheduled';
580
741
  return;
581
742
  }
582
- const snapshot = await context.store.processingSnapshot(this.#batchSize, (checkpoint) => !isBlockedCheckpoint(context, checkpoint));
743
+ // Every pending checkpoint is a candidate so unreadable ones cannot occupy the batch.
744
+ const snapshot = await store.processingSnapshot(Number.POSITIVE_INFINITY, (checkpoint) => !isBlockedCheckpoint(context, checkpoint));
583
745
  if (snapshot.checkpoints.length === 0) {
584
746
  context.state = 'idle';
585
747
  return;
586
748
  }
587
- retainedRun = await LocalMemoryRun.create(this.environment, {
588
- projectDirectory: context.store.projectDirectory,
749
+ retainedRun = await LocalMemoryRun.create(environment, {
750
+ projectDirectory: store.projectDirectory,
589
751
  sessionDirectory: join(this.#options.sessionDir ?? join(this.#options.agentDir, 'sessions'), 'memory'),
590
752
  projectKey: context.project.key,
591
753
  projectRoot: context.project.canonicalRoot,
592
- checkpoints: snapshot.checkpoints.map(({ checkpoint }) => checkpoint),
754
+ checkpoints: snapshot.checkpoints.slice(0, this.#batchSize).map(({ checkpoint }) => checkpoint),
593
755
  baseFingerprint: snapshot.fingerprint,
594
756
  ...(control.lastDisabledRunId ? { protectedRunIds: [control.lastDisabledRunId] } : {}),
595
757
  });
596
758
  staging = retainedRun.workspace;
597
759
  const memoryDirectory = join(staging, '.memory');
598
- await hydrateMemoryDirectory(this.environment, snapshot.artifact, memoryDirectory, { memoryPath: '.memory', mode: 'read' });
599
- const manifest = await materializeMemoryInput(this.environment, {
760
+ await hydrateMemoryDirectory(environment, snapshot.artifact, memoryDirectory, { memoryPath: '.memory', mode: 'read' });
761
+ const manifest = await materializeMemoryInput(environment, {
600
762
  stagingDirectory: staging,
601
763
  checkpoints: snapshot.checkpoints.map(({ checkpoint }) => checkpoint),
602
764
  previousCheckpoints: Object.fromEntries(Object.entries(snapshot.state.processed).map(([sessionId, entry]) => [sessionId, entry.checkpoint])),
@@ -605,26 +767,29 @@ export class LocalMemoryCoordinator {
605
767
  ...(this.#options.maxTranscriptBytes === undefined
606
768
  ? {}
607
769
  : { maxTranscriptBytes: this.#options.maxTranscriptBytes }),
770
+ maxSessions: this.#batchSize,
608
771
  signal: abort.signal,
609
772
  });
610
- for (const failure of manifest.failures) {
611
- context.blocked.set(failure.checkpoint.sessionId, {
612
- leafId: failure.checkpoint.leafId,
613
- transcriptDigest: failure.checkpoint.transcriptDigest,
614
- });
615
- }
773
+ // A cancelled read reports as a failure; it must not drop evidence.
774
+ if (abort.signal.aborted || !this.#enabled || this.#disposed)
775
+ throw new Error('Memory processing was cancelled');
776
+ // A reachable Sessions service shows the failures belong to the checkpoints, not to the service.
777
+ if (manifest.failures.length > 0)
778
+ await environment.sessions.list({ scope: 'project' }, { limit: 1, signal: abort.signal });
779
+ const dropped = manifest.failures.length === 0 ? 0 : await store.dropCheckpoints(lease, manifest.failures.map(({ checkpoint, code }) => ({ checkpoint, reason: code })), this.#now());
780
+ const droppedMessage = dropped === 0 ? undefined : droppedCheckpointsMessage(dropped);
616
781
  const processedCheckpoints = snapshot.checkpoints.filter(({ checkpoint }) => manifest.sessions.some((session) => session.checkpoint.sessionId === checkpoint.sessionId
617
782
  && sameCheckpointCursor(session.checkpoint, checkpoint)));
618
783
  if (processedCheckpoints.length === 0) {
619
- context.state = 'blocked';
620
- context.message = MEMORY_INPUT_BLOCKED_MESSAGE;
621
- await retainedRun.finish('blocked', MEMORY_INPUT_BLOCKED_MESSAGE);
784
+ context.state = 'idle';
785
+ context.message = droppedMessage;
786
+ await retainedRun.finish('blocked', droppedMessage ?? 'Memory checkpoints changed before they could be read');
622
787
  return;
623
788
  }
624
789
  await retainedRun.record({ phase: 'model', checkpoints: processedCheckpoints.map(({ checkpoint }) => checkpoint) });
625
790
  if (abort.signal.aborted || !this.#enabled || this.#disposed)
626
791
  throw new Error('Memory processing was cancelled');
627
- attempt = await context.store.beginAttempt(lease, {
792
+ attempt = await store.beginAttempt(lease, {
628
793
  runId: retainedRun.metadata.sessionId,
629
794
  baseFingerprint: snapshot.fingerprint,
630
795
  checkpoints: processedCheckpoints,
@@ -636,16 +801,16 @@ export class LocalMemoryCoordinator {
636
801
  }
637
802
  context.attempt = attempt;
638
803
  this.#emitStatusChange();
639
- const result = await this.#dreamRunner({
804
+ const result = await (this.#options.dreamRunner ?? createDefaultLocalMemoryDreamRunner(environment))({
640
805
  stagingDirectory: staging,
641
806
  memoryDirectory,
642
807
  inputDirectory: join(staging, '.dreaming', 'input'),
643
808
  baseSnapshot: createMemorySnapshot(snapshot.artifact, '.memory', { mode: 'read' }),
644
809
  manifest,
645
- models: this.#options.models,
646
- ...(this.#selectedModel === undefined ? {} : { selectedModel: this.#selectedModel }),
647
- ...(this.#scopedModels === undefined ? {} : { scopedModels: this.#scopedModels }),
648
- ...(this.#classifier === undefined ? {} : { classifier: this.#classifier }),
810
+ models: driver.models,
811
+ ...(driver.selectedModel === undefined ? {} : { selectedModel: driver.selectedModel }),
812
+ ...(driver.scopedModels === undefined ? {} : { scopedModels: driver.scopedModels }),
813
+ ...(driver.classifier === undefined ? {} : { classifier: driver.classifier }),
649
814
  sessionDirectory: join(this.#options.sessionDir ?? join(this.#options.agentDir, 'sessions'), 'memory'),
650
815
  run: retainedRun,
651
816
  signal: abort.signal,
@@ -653,9 +818,9 @@ export class LocalMemoryCoordinator {
653
818
  if (abort.signal.aborted || !this.#enabled || this.#disposed)
654
819
  throw new Error('Memory processing was cancelled');
655
820
  if (result)
656
- await hydrateMemoryDirectory(this.environment, result, memoryDirectory, { replace: true, memoryPath: '.memory' });
821
+ await hydrateMemoryDirectory(environment, result, memoryDirectory, { replace: true, memoryPath: '.memory' });
657
822
  await retainedRun.record({ phase: 'validate' });
658
- const artifact = await readMemoryDirectory(this.environment, memoryDirectory, {
823
+ const artifact = await readMemoryDirectory(environment, memoryDirectory, {
659
824
  memoryPath: '.memory',
660
825
  sourceSessionIds: allowedSourceIds(snapshot, manifest),
661
826
  requireSources: true,
@@ -664,42 +829,45 @@ export class LocalMemoryCoordinator {
664
829
  phase: 'publish',
665
830
  outputFingerprint: createMemorySnapshot(artifact, '.memory').fingerprint,
666
831
  });
667
- await context.store.commit(lease, snapshot.fingerprint, artifact, processedCheckpoints, attempt);
832
+ await store.commit(lease, snapshot.fingerprint, artifact, processedCheckpoints, attempt);
668
833
  published = true;
669
- await context.store.finishAttempt(lease, attempt, { status: 'success' }, this.#now());
834
+ await store.finishAttempt(lease, attempt, { status: 'success' }, this.#now());
670
835
  await retainedRun.finish('completed');
671
- if (manifest.failures.length > 0) {
672
- context.state = 'blocked';
673
- context.message = MEMORY_INPUT_BLOCKED_MESSAGE;
674
- }
675
- else {
676
- context.state = 'idle';
677
- }
678
- this.#emitStatusChange();
836
+ context.state = 'idle';
837
+ context.failedDriver = undefined;
838
+ context.message = droppedMessage;
679
839
  }
680
840
  catch (error) {
681
- const leaseValid = await lease.verify();
682
- const explicitlyCancelled = !this.#enabled || this.#disposed
683
- || (abort.signal.aborted && !lease.compromised.aborted);
841
+ const leaseValid = await lease.verify().catch(() => false);
842
+ const explicitlyCancelled = cancelled();
684
843
  const unavailableModel = error instanceof MemoryModelUnavailableError;
685
844
  const blockedBeforeInference = retainedRun !== undefined && attempt === undefined && !explicitlyCancelled;
686
845
  if (attempt && (leaseValid || explicitlyCancelled)) {
687
846
  try {
688
- const control = await context.store.finishAttempt(explicitlyCancelled || unavailableModel ? undefined : lease, attempt, explicitlyCancelled || unavailableModel ? { status: 'cancelled' }
847
+ const control = await store.finishAttempt(explicitlyCancelled || unavailableModel ? undefined : lease, attempt, explicitlyCancelled || unavailableModel ? { status: 'cancelled' }
689
848
  : { status: 'failure', reason: sanitizeMemoryDiagnostic(error) }, this.#now());
690
849
  published ||= control.lastOutcome?.runId === attempt.runId && control.lastOutcome.status === 'success';
691
850
  }
692
851
  catch {
693
- context.storageBlocked = true;
852
+ if (!explicitlyCancelled)
853
+ context.storageBlocked = true;
694
854
  }
695
855
  }
856
+ if (explicitlyCancelled) {
857
+ context.state = 'idle';
858
+ context.message = driver.closing || driver.detached || this.#disposed
859
+ ? 'Memory processing was cancelled during shutdown' : 'Memory processing was cancelled';
860
+ await retainedRun?.finish('cancelled', error).catch(() => { });
861
+ return;
862
+ }
696
863
  if (published) {
697
864
  context.state = 'idle';
865
+ context.failedDriver = context.storageBlocked ? driver : undefined;
698
866
  context.message = 'Memory was published, but its diagnostics could not be fully saved';
699
- this.#emitStatusChange();
700
867
  return;
701
868
  }
702
869
  context.state = 'error';
870
+ context.failedDriver = driver;
703
871
  context.message = !leaseValid
704
872
  ? safeProcessingMessage(new Error('Memory writer lease was lost'))
705
873
  : safeProcessingMessage(error);
@@ -713,62 +881,68 @@ export class LocalMemoryCoordinator {
713
881
  context.storageBlocked = true;
714
882
  context.message = `Memory processing is paused: ${sanitizeMemoryDiagnostic(error)}`;
715
883
  }
716
- await retainedRun?.finish(explicitlyCancelled ? 'cancelled'
717
- : unavailableModel || blockedBeforeInference ? 'blocked'
718
- : !leaseValid ? 'interrupted' : 'failed', error).catch(() => {
884
+ await retainedRun?.finish(unavailableModel || blockedBeforeInference ? 'blocked'
885
+ : !leaseValid ? 'interrupted' : 'failed', error).catch(() => {
719
886
  context.storageBlocked = true;
720
887
  context.message = 'Memory diagnostics could not be saved; evidence remains pending';
721
888
  });
722
- this.#emitStatusChange();
723
889
  }
724
890
  finally {
725
891
  lease.compromised.removeEventListener('abort', abortOnLeaseCompromise);
726
- context.abort = undefined;
727
892
  context.attempt = undefined;
728
- await lease.release();
893
+ await lease.release().catch(error => {
894
+ if (!cancelled())
895
+ throw error;
896
+ });
729
897
  }
730
898
  }
731
- async #reconcileRunDiagnostics(context, lease) {
732
- let control = await context.store.readControl();
733
- control = await context.store.reconcileAbandonedAttempt(lease, this.#now());
899
+ async #reconcileRunDiagnostics(context, environment, lease) {
900
+ const store = this.#store(context, environment);
901
+ const previous = await store.readControl();
902
+ const control = await store.reconcileAbandonedAttempt(lease, this.#now());
903
+ if (previous.updatedAt !== control.updatedAt)
904
+ this.#emitStatusChange();
734
905
  const options = {
735
- projectDirectory: context.store.projectDirectory,
906
+ projectDirectory: store.projectDirectory,
736
907
  projectKey: context.project.key,
737
908
  sessionDirectory: join(this.#options.sessionDir ?? join(this.#options.agentDir, 'sessions'), 'memory'),
738
909
  ...(control.lastDisabledRunId ? { protectedRunIds: [control.lastDisabledRunId] } : {}),
739
910
  };
740
911
  if (control.lastOutcome)
741
- await LocalMemoryRun.reconcile(this.environment, options, control.lastOutcome);
742
- await LocalMemoryRun.reconcileAbandoned(this.environment, options, [
912
+ await LocalMemoryRun.reconcile(environment, options, control.lastOutcome);
913
+ await LocalMemoryRun.reconcileAbandoned(environment, options, [
743
914
  ...(control.activeAttempt ? [control.activeAttempt.runId] : []),
744
915
  ...(control.lastOutcome ? [control.lastOutcome.runId] : []),
745
916
  ], this.#now());
746
917
  return control;
747
918
  }
748
- async #recover(context, signal) {
919
+ async #recover(context, driver, signal) {
749
920
  let cursor;
750
921
  let recovered = false;
751
922
  const stableAfter = this.#options.recoveryStableMs ?? 30_000;
923
+ const store = this.#store(context, driver.environment);
752
924
  const stopped = () => this.#disposed || this.#draining || !this.#enabled || signal.aborted;
753
925
  try {
754
926
  do {
755
- const page = await this.environment.sessions.list({ scope: 'project' }, { ...(cursor ? { cursor } : {}), signal });
927
+ const page = await driver.environment.sessions.list({ scope: 'project' }, { ...(cursor ? { cursor } : {}), signal });
756
928
  for (const info of page.items) {
757
929
  if (stopped())
758
930
  return false;
759
931
  if (info.parent || this.#now() - info.updatedAt < stableAfter)
760
932
  continue;
933
+ let checkpoint;
761
934
  try {
762
- const checkpoint = await readMemorySessionCheckpoint(this.environment.sessions, info.id, signal);
763
- if (stopped())
764
- return false;
765
- recovered = (await context.store.recordCheckpoint(checkpoint)) || recovered;
935
+ checkpoint = await readMemorySessionCheckpoint(driver.environment.sessions, info.id, signal);
766
936
  }
767
937
  catch {
768
938
  if (stopped())
769
939
  return false;
770
940
  // A session that cannot be read now is skipped; its live checkpoint or a later run records it.
941
+ continue;
771
942
  }
943
+ if (stopped())
944
+ return false;
945
+ recovered = (await store.recordCheckpoint(checkpoint)) || recovered;
772
946
  }
773
947
  cursor = page.nextCursor;
774
948
  } while (cursor);
@@ -790,13 +964,23 @@ export class LocalMemoryCoordinator {
790
964
  return true;
791
965
  }
792
966
  #emitStatusChange() {
793
- for (const listener of [...this.#statusListeners]) {
967
+ for (const listener of [...this.#statusListeners, ...[...this.#attachments].flatMap(attachment => [...attachment.listeners])]) {
794
968
  try {
795
969
  listener();
796
970
  }
797
971
  catch { }
798
972
  }
799
973
  }
974
+ #cancelTimers(context) {
975
+ this.#cancelScheduledRun(context);
976
+ if (context.recoveryTimer)
977
+ clearTimeout(context.recoveryTimer);
978
+ if (context.writerTimer)
979
+ clearTimeout(context.writerTimer);
980
+ context.recoveryTimer = undefined;
981
+ context.writerTimer = undefined;
982
+ context.writerBlocked = false;
983
+ }
800
984
  #cancelScheduledRun(context) {
801
985
  if (context.timer)
802
986
  clearTimeout(context.timer);
@@ -844,6 +1028,9 @@ function allowedSourceIds(snapshot, manifest) {
844
1028
  ...manifest.sessions.map(({ checkpoint }) => checkpoint.sessionId),
845
1029
  ])];
846
1030
  }
1031
+ function droppedCheckpointsMessage(count) {
1032
+ return `Dropped ${count} memory checkpoint${count === 1 ? '' : 's'} that could not be read`;
1033
+ }
847
1034
  function safeProcessingMessage(error) {
848
1035
  if (error instanceof MemoryModelUnavailableError) {
849
1036
  return 'Low-tier memory model is unavailable; evidence remains pending';