@felan-ai/ext-memory 0.10.1 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -11,48 +11,73 @@ import {} from '@felan-ai/agent-core';
11
11
  import { MemoryModelUnavailableError } from './dreamer.js';
12
12
  import { LocalMemoryControlError } from './store.js';
13
13
  import { sanitizeMemoryDiagnostic } from '../diagnostics.js';
14
- const MEMORY_INPUT_BLOCKED_MESSAGE = 'Some memory checkpoints could not be materialized; evidence remains pending';
14
+ const MEMORY_INPUT_BLOCKED_MESSAGE = 'Memory checkpoints are paused after a failed run; evidence remains pending';
15
15
  const MEMORY_AUTOMATIC_UPDATE_THRESHOLD = 5;
16
16
  const MEMORY_AUTOMATIC_INTERVAL_MS = 24 * 60 * 60 * 1_000;
17
17
  const MEMORY_DREAM_INPUT_MAX_BYTES = 10 * 1024 * 1024;
18
18
  const MEMORY_MAX_BATCH_SESSIONS = 8;
19
19
  export class LocalMemoryCoordinator {
20
- environment;
21
20
  #options;
22
21
  #contexts = new Map();
23
- #projections = new Map();
22
+ #attachments = new Set();
24
23
  #statusListeners = new Set();
25
- #dreamRunner;
26
24
  #batchSize;
27
25
  #enabled;
28
- #selectedModel;
29
- #scopedModels;
30
- #classifier;
31
26
  #disposed = false;
27
+ #disposal;
32
28
  #draining = false;
33
- #monitor;
34
- #monitoring;
35
29
  #enableVersion = 0;
36
- constructor(environment, options) {
37
- this.environment = environment;
30
+ constructor(options) {
38
31
  this.#options = options;
39
32
  this.#batchSize = boundedMemoryBatchSize(options.batchSize);
40
33
  this.#enabled = options.enabled !== false;
41
- this.#selectedModel = options.selectedModel;
42
- this.#scopedModels = options.scopedModels === undefined ? undefined : [...options.scopedModels];
43
- this.#classifier = options.classifier;
44
- this.#dreamRunner = options.dreamRunner ?? createDefaultLocalMemoryDreamRunner(this.environment);
45
- if (this.#enabled)
46
- this.#startMonitor();
47
34
  }
48
- createSessionHost(options) {
49
- return new LocalMemorySessionHost(this, options);
35
+ attach(options) {
36
+ if (this.#disposed)
37
+ throw new Error('Local memory coordinator is closed');
38
+ const state = {
39
+ ...options,
40
+ selectedModel: options.selectedModel,
41
+ scopedModels: options.scopedModels === undefined ? undefined : [...options.scopedModels],
42
+ classifier: options.classifier,
43
+ projections: new Map(),
44
+ listeners: new Set(),
45
+ draining: false,
46
+ closing: false,
47
+ detached: false,
48
+ detaching: undefined,
49
+ };
50
+ this.#attachments.add(state);
51
+ const attachment = {
52
+ createSessionHost: options => new LocalMemorySessionHost(attachment, options),
53
+ readCurrent: (cwd, root) => this.#readCurrent(state, cwd, root),
54
+ recordCheckpoint: (cwd, checkpoint) => this.#recordCheckpoint(state, cwd, checkpoint),
55
+ status: cwd => this.#status(state, cwd),
56
+ runNow: cwd => this.#runNow(state, cwd),
57
+ retryProject: cwd => this.#retryProject(state, cwd),
58
+ canonicalDirectory: async (cwd) => this.#store(await this.#context(state, cwd), state.environment).currentDirectory,
59
+ setSelectedModel: model => this.#setModelSelection(state, model, state.scopedModels, state.classifier),
60
+ setModelSelection: (model, scopedModels, classifier) => this.#setModelSelection(state, model, scopedModels, classifier),
61
+ subscribeStatusChanges: listener => {
62
+ if (state.detached || this.#disposed)
63
+ return () => { };
64
+ state.listeners.add(listener);
65
+ return () => { state.listeners.delete(listener); };
66
+ },
67
+ busy: () => [...this.#contexts.values()].some(context => context.driver === state || context.recoveryDriver === state),
68
+ settle: mode => this.#settleAttachment(state, mode),
69
+ detach: () => {
70
+ state.detaching ??= this.#detach(state);
71
+ return state.detaching;
72
+ },
73
+ };
74
+ return attachment;
50
75
  }
51
76
  isEnabled() {
52
77
  return this.#enabled;
53
78
  }
54
79
  hasActiveWork() {
55
- return !!this.#monitoring || [...this.#contexts.values()].some(context => !!context.run || !!context.ready || !!context.recovery || context.followUps.size > 0);
80
+ return [...this.#contexts.values()].some(context => !!context.run || !!context.ready || !!context.recovery || context.followUps.size > 0);
56
81
  }
57
82
  subscribeStatusChanges(listener) {
58
83
  if (this.#disposed)
@@ -60,34 +85,28 @@ export class LocalMemoryCoordinator {
60
85
  this.#statusListeners.add(listener);
61
86
  return () => this.#statusListeners.delete(listener);
62
87
  }
63
- setModelSelection(model, scopedModels, classifier) {
64
- this.#scopedModels = scopedModels === undefined ? undefined : [...scopedModels];
65
- this.#classifier = classifier;
66
- this.setSelectedModel(model);
67
- }
68
- setSelectedModel(model) {
69
- this.#selectedModel = model;
88
+ #setModelSelection(attachment, model, scopedModels, classifier) {
89
+ if (attachment.detached || this.#disposed)
90
+ return;
91
+ attachment.selectedModel = model;
92
+ attachment.scopedModels = scopedModels === undefined ? undefined : [...scopedModels];
93
+ attachment.classifier = classifier;
70
94
  for (const context of this.#contexts.values()) {
95
+ if (!context.attachments.has(attachment))
96
+ continue;
71
97
  context.blocked.clear();
72
98
  if (context.initialized)
73
- void this.#trackScheduleIfPending(context);
99
+ void this.#trackScheduleIfPending(context, attachment.environment);
74
100
  }
75
101
  }
76
102
  setEnabled(enabled) {
77
103
  this.#enableVersion += 1;
78
- enabled = enabled && !this.#options.unavailableReason;
79
104
  this.#enabled = enabled;
80
- if (enabled)
81
- this.#startMonitor();
82
- else if (this.#monitor) {
83
- clearInterval(this.#monitor);
84
- this.#monitor = undefined;
85
- }
86
105
  for (const context of this.#contexts.values()) {
87
106
  if (!enabled)
88
107
  context.recoveryAbort?.abort();
89
108
  if (!enabled)
90
- this.#cancelScheduledRun(context);
109
+ this.#cancelTimers(context);
91
110
  if (!enabled)
92
111
  this.#cancelContext(context, 'Memory processing was disabled', true);
93
112
  if (enabled) {
@@ -107,26 +126,27 @@ export class LocalMemoryCoordinator {
107
126
  }
108
127
  this.#emitStatusChange();
109
128
  }
110
- async status(cwd) {
129
+ async #status(attachment, cwd) {
111
130
  try {
112
- const context = await this.#context(cwd);
113
- const state = await context.store.status();
114
- const control = await context.store.readControl();
131
+ const context = await this.#context(attachment, cwd);
132
+ const store = this.#store(context, attachment.environment);
133
+ const state = await store.status();
134
+ const control = await store.readControl();
115
135
  const processed = Object.values(state.processed);
116
136
  const lastProcessedAt = processed
117
137
  .map((entry) => entry.processedAt)
118
138
  .sort()
119
139
  .at(-1);
120
140
  return {
121
- enabled: this.#enabled && !control.autoDisabled,
122
- state: !this.#enabled || control.autoDisabled ? 'disabled'
141
+ enabled: this.#enabled && attachment.scheduling && !control.autoDisabled,
142
+ state: !this.#enabled || !attachment.scheduling || control.autoDisabled ? 'disabled'
123
143
  : control.nextRetryAt && Date.parse(control.nextRetryAt) > this.#now() ? 'scheduled'
124
144
  : context.state === 'error' ? 'error'
125
145
  : control.activeAttempt ? 'processing' : context.state,
126
146
  pendingCheckpoints: Object.keys(state.pending).length,
127
147
  memoryFingerprint: state.memoryFingerprint,
128
148
  ...(lastProcessedAt === undefined ? {} : { lastProcessedAt }),
129
- ...(context.message || this.#options.unavailableReason ? { message: context.message ?? this.#options.unavailableReason } : {}),
149
+ ...(context.message || attachment.unavailableReason ? { message: context.message ?? attachment.unavailableReason } : {}),
130
150
  consecutiveFailures: control.consecutiveFailures,
131
151
  ...(control.nextRetryAt ? { nextRetryAt: control.nextRetryAt } : {}),
132
152
  ...(control.autoDisabled ? { autoDisabled: control.autoDisabled } : {}),
@@ -137,37 +157,39 @@ export class LocalMemoryCoordinator {
137
157
  }
138
158
  catch (error) {
139
159
  return {
140
- enabled: this.#enabled,
141
- state: this.#enabled ? 'error' : 'disabled',
160
+ enabled: this.#enabled && attachment.scheduling,
161
+ state: this.#enabled && attachment.scheduling ? 'error' : 'disabled',
142
162
  pendingCheckpoints: 0,
143
163
  message: error instanceof LocalMemoryControlError ? error.message : 'Local memory storage is unavailable',
144
164
  };
145
165
  }
146
166
  }
147
- async readCurrent(cwd, sessionStorageRoot) {
148
- const context = await this.#context(cwd);
149
- const snapshot = await context.store.readCurrent();
167
+ async #readCurrent(attachment, cwd, sessionStorageRoot) {
168
+ const context = await this.#context(attachment, cwd);
169
+ const store = this.#store(context, attachment.environment);
170
+ const snapshot = await store.readCurrent();
150
171
  const projectionKey = `${context.project.key}:${sessionStorageRoot}`;
151
- const memoryPath = join(sessionStorageRoot, context.store.projectionName);
152
- if (this.#projections.get(projectionKey) !== snapshot.fingerprint) {
153
- const projection = await context.store.projectTo(sessionStorageRoot, snapshot);
154
- this.#projections.set(projectionKey, snapshot.fingerprint);
172
+ const memoryPath = join(sessionStorageRoot, store.projectionName);
173
+ if (attachment.projections.get(projectionKey) !== snapshot.fingerprint) {
174
+ const projection = await store.projectTo(sessionStorageRoot, snapshot);
175
+ if (!attachment.detached)
176
+ attachment.projections.set(projectionKey, snapshot.fingerprint);
155
177
  return projection;
156
178
  }
157
179
  return createMemoryProjectionSnapshot(snapshot, memoryPath);
158
180
  }
159
- async recordCheckpoint(cwd, checkpoint) {
160
- const context = await this.#context(cwd);
161
- if (await context.store.recordCheckpoint(checkpoint)) {
181
+ async #recordCheckpoint(attachment, cwd, checkpoint) {
182
+ const context = await this.#context(attachment, cwd);
183
+ if (await this.#store(context, attachment.environment).recordCheckpoint(checkpoint)) {
162
184
  context.blocked.delete(checkpoint.sessionId);
163
- await this.#trackScheduleIfPending(context);
185
+ await this.#trackScheduleIfPending(context, attachment.environment);
164
186
  this.#emitStatusChange();
165
187
  }
166
188
  }
167
- async runNow(cwd) {
168
- if (!this.#enabled || this.#draining)
169
- return this.status(cwd);
170
- const context = await this.#context(cwd);
189
+ async #runNow(attachment, cwd) {
190
+ if (!this.#eligible(attachment))
191
+ return this.#status(attachment, cwd);
192
+ const context = await this.#context(attachment, cwd);
171
193
  this.#cancelScheduledRun(context);
172
194
  while (context.run || context.followUps.size > 0) {
173
195
  await Promise.allSettled([
@@ -177,61 +199,114 @@ export class LocalMemoryCoordinator {
177
199
  this.#cancelScheduledRun(context);
178
200
  }
179
201
  context.blocked.clear();
180
- const run = this.#run(context, true);
181
- context.run = run;
182
- try {
183
- await run;
184
- }
185
- finally {
186
- if (context.run === run)
187
- context.run = undefined;
188
- await this.#trackScheduleIfPending(context);
189
- }
190
- return this.status(cwd);
202
+ context.storageBlocked = false;
203
+ context.message = undefined;
204
+ await this.#launchRun(context, true, attachment);
205
+ return this.#status(attachment, cwd);
191
206
  }
192
- async retryProject(cwd) {
193
- if (!this.#enabled || this.#draining)
194
- return this.status(cwd);
195
- const context = await this.#context(cwd);
196
- if (!this.#enabled || this.#disposed || this.#draining)
197
- return this.status(cwd);
207
+ async #retryProject(attachment, cwd) {
208
+ if (!this.#eligible(attachment))
209
+ return this.#status(attachment, cwd);
210
+ const context = await this.#context(attachment, cwd);
211
+ if (!this.#eligible(attachment))
212
+ return this.#status(attachment, cwd);
198
213
  const enableVersion = this.#enableVersion;
199
214
  this.#cancelScheduledRun(context);
200
- try {
201
- await context.store.resetProjectControl(this.#now(), () => this.#enabled && !this.#disposed && !this.#draining && this.#enableVersion === enableVersion);
202
- }
203
- catch (error) {
204
- throw error;
205
- }
215
+ await this.#store(context, attachment.environment).resetProjectControl(this.#now(), () => this.#eligible(attachment) && this.#enableVersion === enableVersion);
216
+ this.#emitStatusChange();
206
217
  context.abort?.abort(new Error('Memory processing was explicitly retried'));
207
218
  if (context.run)
208
219
  await context.run;
209
220
  context.blocked.clear();
210
221
  context.storageBlocked = false;
211
222
  context.message = undefined;
212
- await this.runNow(cwd);
213
- return this.status(cwd);
223
+ await this.#runNow(attachment, cwd);
224
+ return this.#status(attachment, cwd);
225
+ }
226
+ #eligible(attachment) {
227
+ return this.#enabled && !this.#disposed && !this.#draining
228
+ && attachment.scheduling && !attachment.draining && !attachment.closing && !attachment.detached;
229
+ }
230
+ #driver(context, preferred) {
231
+ if (preferred && context.attachments.has(preferred) && this.#eligible(preferred))
232
+ return preferred;
233
+ return [...this.#attachments].reverse().find(attachment => context.attachments.has(attachment) && this.#eligible(attachment));
234
+ }
235
+ #liveMember(context) {
236
+ return [...this.#attachments].reverse().find(attachment => context.attachments.has(attachment)
237
+ && attachment.scheduling && !attachment.draining && !attachment.closing && !attachment.detached);
238
+ }
239
+ #runEnvironment(context, driver) {
240
+ // The daemon seals closing runtimes outside their own cleanup async context.
241
+ const current = () => (driver.closing || driver.detached ? this.#liveMember(context) ?? driver : driver).environment;
242
+ const environment = {
243
+ get runtime() { return current().runtime; },
244
+ get sessions() { return current().sessions; },
245
+ };
246
+ Object.defineProperty(environment, 'openWorker', { get: () => current().openWorker });
247
+ return environment;
248
+ }
249
+ async #settleAttachment(attachment, mode) {
250
+ attachment.draining = true;
251
+ if (mode === 'force')
252
+ attachment.closing = true;
253
+ for (const context of this.#contexts.values()) {
254
+ if (context.recoveryDriver === attachment)
255
+ context.recoveryAbort?.abort();
256
+ if (mode === 'force' && context.driver === attachment) {
257
+ this.#cancelContext(context, 'Memory processing was cancelled during shutdown', false);
258
+ }
259
+ if (!this.#driver(context))
260
+ this.#cancelTimers(context);
261
+ }
262
+ while (true) {
263
+ const work = [...this.#contexts.values()].flatMap(context => [
264
+ ...(context.driver === attachment ? [...(context.run ? [context.run] : []), ...context.followUps] : []),
265
+ ...(context.recoveryDriver === attachment && context.recovery ? [context.recovery] : []),
266
+ ]);
267
+ if (!work.length)
268
+ break;
269
+ await Promise.allSettled(work);
270
+ }
271
+ for (const context of this.#contexts.values()) {
272
+ if (mode === 'force' && context.failedDriver === attachment) {
273
+ context.storageBlocked = false;
274
+ context.blocked.clear();
275
+ if (context.state === 'error' || context.state === 'blocked')
276
+ context.state = 'idle';
277
+ context.message = undefined;
278
+ context.failedDriver = undefined;
279
+ }
280
+ if (!context.attachments.has(attachment) || !context.initialized)
281
+ continue;
282
+ this.#startRecovery(context);
283
+ void this.#trackScheduleIfPending(context);
284
+ }
214
285
  }
215
- async canonicalDirectory(cwd) {
216
- const context = await this.#context(cwd);
217
- return context.store.currentDirectory;
286
+ async #detach(attachment) {
287
+ attachment.closing = true;
288
+ await this.#settleAttachment(attachment, 'force');
289
+ attachment.detached = true;
290
+ this.#attachments.delete(attachment);
291
+ attachment.listeners.clear();
292
+ attachment.projections.clear();
293
+ for (const context of this.#contexts.values()) {
294
+ context.attachments.delete(attachment);
295
+ if (!this.#driver(context))
296
+ this.#cancelTimers(context);
297
+ }
218
298
  }
219
299
  async drain() {
220
300
  this.#draining = true;
221
- if (this.#monitor)
222
- clearInterval(this.#monitor);
223
- this.#monitor = undefined;
224
301
  for (const context of this.#contexts.values()) {
225
302
  context.recoveryAbort?.abort();
226
- this.#cancelScheduledRun(context);
303
+ this.#cancelTimers(context);
227
304
  }
228
305
  while (true) {
229
306
  const work = [...this.#contexts.values()].flatMap(context => [
230
307
  ...(context.run ? [context.run] : []), ...(context.ready ? [context.ready] : []),
231
308
  ...(context.recovery ? [context.recovery] : []), ...context.followUps,
232
309
  ]);
233
- if (this.#monitoring)
234
- work.push(this.#monitoring);
235
310
  if (!work.length)
236
311
  return;
237
312
  const results = await Promise.allSettled(work);
@@ -240,103 +315,121 @@ export class LocalMemoryCoordinator {
240
315
  throw new AggregateError(failures, 'Memory drain could not settle all work.');
241
316
  }
242
317
  }
243
- async dispose() {
244
- if (this.#disposed)
245
- return;
318
+ dispose() {
319
+ this.#disposal ??= this.#dispose();
320
+ return this.#disposal;
321
+ }
322
+ async #dispose() {
246
323
  this.#disposed = true;
247
- if (this.#monitor)
248
- clearInterval(this.#monitor);
249
- this.#monitor = undefined;
250
324
  for (const context of this.#contexts.values()) {
251
325
  context.recoveryAbort?.abort();
252
- this.#cancelScheduledRun(context);
326
+ this.#cancelTimers(context);
253
327
  this.#cancelContext(context, 'Memory processing was cancelled during shutdown', false);
254
328
  }
255
329
  while (true) {
256
- const work = [...this.#contexts.values()].flatMap((context) => [
330
+ const work = [...this.#contexts.values()].flatMap(context => [
257
331
  ...(context.run ? [context.run] : []),
258
332
  ...(context.ready ? [context.ready] : []),
259
333
  ...(context.recovery ? [context.recovery] : []),
260
334
  ...context.followUps,
261
335
  ]);
262
- if (this.#monitoring)
263
- work.push(this.#monitoring);
264
- if (work.length === 0)
336
+ if (!work.length)
265
337
  break;
266
338
  await Promise.allSettled(work);
267
339
  }
340
+ for (const attachment of this.#attachments) {
341
+ attachment.detached = true;
342
+ attachment.listeners.clear();
343
+ attachment.projections.clear();
344
+ }
345
+ this.#attachments.clear();
268
346
  this.#statusListeners.clear();
269
347
  }
270
- async #context(cwd) {
271
- if (this.#disposed)
348
+ async #context(attachment, cwd) {
349
+ if (this.#disposed || attachment.detached)
272
350
  throw new Error('Local memory coordinator is closed');
273
- const project = await resolveLocalMemoryProject(this.environment, cwd);
274
- if (this.#disposed)
351
+ const project = await resolveLocalMemoryProject(attachment.environment, cwd);
352
+ if (this.#disposed || attachment.detached)
275
353
  throw new Error('Local memory coordinator is closed');
276
354
  const existing = this.#contexts.get(project.key);
277
355
  if (existing) {
278
- await this.#initialize(existing);
356
+ existing.attachments.add(attachment);
357
+ await this.#initialize(existing, attachment);
279
358
  return existing;
280
359
  }
281
- const store = new LocalMemoryStore(this.environment, this.#options.agentDir, project, {
282
- ...(this.#options.retryDelaysMs ? { retryDelaysMs: this.#options.retryDelaysMs } : {}),
283
- });
284
360
  const context = {
285
361
  project,
286
- cwd,
287
- ...(this.#options.sessionDir === undefined ? {} : { sessionDir: this.#options.sessionDir }),
288
- store,
362
+ stores: new WeakMap(),
363
+ attachments: new Set([attachment]),
364
+ driver: undefined,
365
+ failedDriver: undefined,
366
+ recoveryDriver: undefined,
289
367
  ready: undefined,
290
368
  initialized: false,
291
369
  recovery: undefined,
292
370
  recoveryAbort: undefined,
293
371
  recoveryRetryAt: undefined,
372
+ recoveryTimer: undefined,
294
373
  recoveryCompleted: false,
295
374
  diagnosticsReconciled: false,
296
375
  timer: undefined,
297
376
  timerDueAt: undefined,
298
377
  run: undefined,
378
+ runEnvironment: undefined,
299
379
  followUps: new Set(),
300
380
  abort: undefined,
301
381
  attempt: undefined,
302
382
  storageBlocked: false,
303
383
  writerBlocked: false,
304
- statusSignature: undefined,
384
+ writerTimer: undefined,
305
385
  blocked: new Map(),
306
386
  state: this.#enabled ? 'idle' : 'disabled',
307
387
  message: undefined,
308
388
  };
309
389
  this.#contexts.set(project.key, context);
310
- await this.#initialize(context);
390
+ await this.#initialize(context, attachment);
311
391
  return context;
312
392
  }
313
- async #initialize(context) {
393
+ #store(context, environment) {
394
+ let store = context.stores.get(environment);
395
+ if (!store) {
396
+ store = new LocalMemoryStore(environment, this.#options.agentDir, context.project, {
397
+ ...(this.#options.retryDelaysMs ? { retryDelaysMs: this.#options.retryDelaysMs } : {}),
398
+ });
399
+ context.stores.set(environment, store);
400
+ }
401
+ return store;
402
+ }
403
+ async #initialize(context, attachment) {
314
404
  if (context.ready) {
315
405
  await context.ready;
316
406
  return;
317
407
  }
318
408
  if (context.initialized && context.diagnosticsReconciled) {
319
409
  this.#startRecovery(context);
410
+ void this.#trackScheduleIfPending(context, attachment.environment);
320
411
  return;
321
412
  }
322
- const ready = this.#initializeContext(context).finally(() => {
413
+ const ready = this.#initializeContext(context, attachment).finally(() => {
323
414
  if (context.ready === ready)
324
415
  context.ready = undefined;
325
416
  });
326
417
  context.ready = ready;
327
418
  await ready;
328
419
  }
329
- async #initializeContext(context) {
420
+ async #initializeContext(context, attachment) {
421
+ const environment = attachment.environment;
422
+ const store = this.#store(context, environment);
330
423
  if (!context.initialized) {
331
- await context.store.initialize(this.#now()).then(() => {
424
+ await store.initialize(this.#now()).then(() => {
332
425
  context.initialized = true;
333
426
  context.state = this.#enabled ? 'idle' : 'disabled';
334
427
  context.message = undefined;
335
428
  }).catch(async (error) => {
336
429
  if (!(error instanceof LocalMemoryControlError))
337
430
  throw error;
338
- const current = await context.store.readCurrent();
339
- if (current.fingerprint !== (await context.store.status()).memoryFingerprint)
431
+ const current = await store.readCurrent();
432
+ if (current.fingerprint !== (await store.status()).memoryFingerprint)
340
433
  throw error;
341
434
  context.message = error.message;
342
435
  });
@@ -345,11 +438,16 @@ export class LocalMemoryCoordinator {
345
438
  return;
346
439
  if (!context.diagnosticsReconciled) {
347
440
  try {
348
- const lease = await acquireLocalMemoryLease(this.environment, context.store.projectDirectory, this.#options.leaseOptions);
441
+ const lease = await acquireLocalMemoryLease(environment, store.projectDirectory, this.#options.leaseOptions);
349
442
  if (lease) {
350
443
  try {
351
- await this.#reconcileRunDiagnostics(context, lease);
444
+ await this.#reconcileRunDiagnostics(context, environment, lease);
352
445
  context.diagnosticsReconciled = true;
446
+ if (context.storageBlocked) {
447
+ context.storageBlocked = false;
448
+ context.failedDriver = undefined;
449
+ context.message = undefined;
450
+ }
353
451
  }
354
452
  finally {
355
453
  await lease.release();
@@ -357,24 +455,34 @@ export class LocalMemoryCoordinator {
357
455
  }
358
456
  }
359
457
  catch (error) {
360
- context.diagnosticsReconciled = true;
361
458
  context.storageBlocked = true;
459
+ context.failedDriver = attachment;
362
460
  context.message = `Memory diagnostics recovery is blocked: ${sanitizeMemoryDiagnostic(error)}`;
363
461
  }
364
462
  }
365
463
  if (this.#disposed)
366
464
  return;
367
465
  if (this.#enabled)
368
- void this.#trackScheduleIfPending(context);
466
+ void this.#trackScheduleIfPending(context, environment);
369
467
  this.#startRecovery(context);
370
468
  }
371
469
  #startRecovery(context) {
372
- if (this.#disposed || this.#draining || !this.#enabled || this.#options.recover === false
373
- || context.recoveryCompleted || context.recovery
374
- || context.recoveryRetryAt !== undefined && this.#now() < context.recoveryRetryAt)
470
+ if (this.#options.recover === false || context.recoveryCompleted || context.recovery)
471
+ return;
472
+ const driver = this.#driver(context);
473
+ if (!driver)
474
+ return;
475
+ if (context.recoveryRetryAt !== undefined && this.#now() < context.recoveryRetryAt) {
476
+ this.#scheduleRecoveryRetry(context);
375
477
  return;
478
+ }
479
+ if (context.recoveryTimer)
480
+ clearTimeout(context.recoveryTimer);
481
+ context.recoveryTimer = undefined;
482
+ context.recoveryDriver = driver;
376
483
  const abort = new AbortController();
377
- const recovery = this.#recover(context, abort.signal).then((completed) => {
484
+ context.recoveryAbort = abort;
485
+ const recovery = this.#recover(context, driver, abort.signal).then(completed => {
378
486
  if (completed)
379
487
  context.recoveryCompleted = true;
380
488
  }).catch(() => {
@@ -383,71 +491,70 @@ export class LocalMemoryCoordinator {
383
491
  if (context.recovery === recovery) {
384
492
  context.recovery = undefined;
385
493
  context.recoveryAbort = undefined;
494
+ context.recoveryDriver = undefined;
386
495
  }
496
+ if (abort.signal.aborted)
497
+ this.#startRecovery(context);
498
+ else
499
+ this.#scheduleRecoveryRetry(context);
500
+ this.#emitStatusChange();
387
501
  });
388
502
  context.recovery = recovery;
389
- context.recoveryAbort = abort;
390
503
  }
391
- #now() { return this.#options.now?.() ?? Date.now(); }
392
- #startMonitor() {
393
- if (this.#disposed || this.#draining || this.#monitor)
504
+ #scheduleRecoveryRetry(context) {
505
+ if (context.recoveryTimer || context.recoveryCompleted || context.recoveryRetryAt === undefined || !this.#driver(context))
394
506
  return;
395
- const interval = this.#options.monitorIntervalMs ?? 2_000;
396
- if (!Number.isFinite(interval) || interval < 1)
397
- throw new Error('Memory monitor interval must be positive');
398
- this.#monitor = setInterval(() => {
399
- if (this.#monitoring)
507
+ const timer = setTimeout(() => {
508
+ if (context.recoveryTimer !== timer)
400
509
  return;
401
- this.#monitoring = this.#refresh().catch(() => { }).finally(() => { this.#monitoring = undefined; });
402
- }, interval);
403
- this.#monitor.unref?.();
510
+ context.recoveryTimer = undefined;
511
+ this.#startRecovery(context);
512
+ }, Math.max(0, context.recoveryRetryAt - this.#now()));
513
+ context.recoveryTimer = timer;
514
+ timer.unref?.();
404
515
  }
405
- async #refresh() {
406
- if (this.#disposed || this.#draining)
516
+ #scheduleWriterRetry(context) {
517
+ if (context.writerTimer || !this.#driver(context))
407
518
  return;
408
- for (const context of this.#contexts.values()) {
409
- try {
410
- await this.#initialize(context);
411
- const observedAttempt = context.attempt;
412
- const control = await context.store.readControl();
413
- if (observedAttempt && context.attempt === observedAttempt && (!this.#enabled || control.autoDisabled
414
- || control.activeAttempt?.runId !== observedAttempt.runId
415
- || control.activeAttempt?.generation !== observedAttempt.generation)) {
416
- this.#cancelContext(context, 'Memory processing was cancelled by another session', true);
417
- }
418
- context.writerBlocked = false;
419
- if (context.initialized)
420
- await this.#trackScheduleIfPending(context);
421
- const signature = JSON.stringify([this.#enabled, context.state, context.message, control.updatedAt]);
422
- if (signature !== context.statusSignature) {
423
- context.statusSignature = signature;
424
- this.#emitStatusChange();
425
- }
426
- }
427
- catch (error) {
428
- context.state = 'error';
429
- context.message = sanitizeMemoryDiagnostic(error);
430
- context.abort?.abort(error);
431
- this.#emitStatusChange();
519
+ context.writerBlocked = true;
520
+ const timer = setTimeout(() => {
521
+ if (context.writerTimer !== timer)
522
+ return;
523
+ context.writerTimer = undefined;
524
+ context.writerBlocked = false;
525
+ if (context.run) {
526
+ void this.#trackScheduleIfPending(context);
527
+ return;
432
528
  }
433
- }
529
+ context.diagnosticsReconciled = false;
530
+ const driver = this.#driver(context);
531
+ if (driver)
532
+ void this.#initialize(context, driver).catch(() => { });
533
+ }, this.#options.writerRetryMs ?? 30_000);
534
+ context.writerTimer = timer;
535
+ timer.unref?.();
434
536
  }
537
+ #now() { return this.#options.now?.() ?? Date.now(); }
435
538
  #cancelContext(context, reason, cancelPersisted) {
436
539
  const alreadyAborted = context.abort?.signal.aborted ?? false;
437
540
  context.abort?.abort(new Error(reason));
438
541
  if (alreadyAborted || !context.initialized || (!cancelPersisted && !context.attempt))
439
542
  return;
543
+ const environment = context.runEnvironment ?? this.#liveMember(context)?.environment;
544
+ if (!environment)
545
+ return;
546
+ const store = this.#store(context, environment);
440
547
  const cancellation = cancelPersisted
441
- ? context.store.cancelActiveAttempt(this.#now())
442
- : context.store.finishAttempt(undefined, context.attempt, { status: 'cancelled', reason }, this.#now());
548
+ ? store.cancelActiveAttempt(this.#now())
549
+ : store.finishAttempt(undefined, context.attempt, { status: 'cancelled', reason }, this.#now());
443
550
  const followUp = cancellation
444
- .then(() => { })
551
+ .then(() => { this.#emitStatusChange(); })
445
552
  .catch(() => { })
446
553
  .finally(() => { context.followUps.delete(followUp); });
447
554
  context.followUps.add(followUp);
448
555
  }
449
556
  async #schedule(context, dueAt) {
450
- if (this.#disposed || this.#draining || !this.#enabled || context.run || context.storageBlocked)
557
+ if (!this.#driver(context) || context.run || context.storageBlocked)
451
558
  return;
452
559
  if (context.timer && context.timerDueAt === dueAt)
453
560
  return;
@@ -459,28 +566,26 @@ export class LocalMemoryCoordinator {
459
566
  return;
460
567
  context.timer = undefined;
461
568
  context.timerDueAt = undefined;
462
- const run = this.#run(context).catch((error) => {
463
- context.state = 'error';
464
- context.storageBlocked = true;
465
- context.message = sanitizeMemoryDiagnostic(error);
466
- this.#emitStatusChange();
467
- }).finally(async () => {
468
- context.run = undefined;
469
- await this.#trackScheduleIfPending(context);
470
- });
471
- context.run = run;
569
+ void this.#launchRun(context)?.catch(() => { });
472
570
  }, delay);
473
571
  context.timer = timer;
474
572
  context.timerDueAt = dueAt;
475
573
  timer.unref?.();
476
574
  }
477
- async #scheduleIfPending(context) {
478
- if (!this.#enabled || this.#disposed || this.#draining)
575
+ async #scheduleIfPending(context, environment) {
576
+ const driver = this.#driver(context);
577
+ if (!driver) {
578
+ this.#cancelTimers(context);
479
579
  return;
580
+ }
581
+ // Callers capture their environment before awaiting; a member sealed since then cannot read the store.
582
+ const owner = environment && [...context.attachments].find(attachment => attachment.environment === environment);
583
+ const source = owner && (owner.closing || owner.detached) ? (this.#liveMember(context) ?? owner).environment : environment;
480
584
  try {
481
- const control = await context.store.readControl();
585
+ const store = this.#store(context, source ?? driver.environment);
586
+ const control = await store.readControl();
482
587
  if (control.autoDisabled) {
483
- this.#cancelScheduledRun(context);
588
+ this.#cancelTimers(context);
484
589
  context.state = 'disabled';
485
590
  return;
486
591
  }
@@ -488,13 +593,21 @@ export class LocalMemoryCoordinator {
488
593
  return;
489
594
  if (context.writerBlocked)
490
595
  return;
491
- const state = await context.store.status();
596
+ const state = await store.status();
597
+ if (context.state === 'error' && context.message === 'Local memory storage is unavailable') {
598
+ context.state = 'idle';
599
+ context.message = undefined;
600
+ }
492
601
  const pending = Object.values(state.pending);
602
+ if (!this.#driver(context))
603
+ return;
493
604
  if (control.activeAttempt) {
494
605
  this.#cancelScheduledRun(context);
606
+ if (!context.run)
607
+ this.#scheduleWriterRetry(context);
495
608
  }
496
609
  else if (pending.some(({ checkpoint }) => !isBlockedCheckpoint(context, checkpoint))) {
497
- const acceptedUpdates = await context.store.pendingAcceptedUpdates((checkpoint) => !isBlockedCheckpoint(context, checkpoint));
610
+ const acceptedUpdates = await store.pendingAcceptedUpdates((checkpoint) => !isBlockedCheckpoint(context, checkpoint));
498
611
  if (acceptedUpdates < MEMORY_AUTOMATIC_UPDATE_THRESHOLD) {
499
612
  this.#cancelScheduledRun(context);
500
613
  if (context.state === 'scheduled')
@@ -529,37 +642,84 @@ export class LocalMemoryCoordinator {
529
642
  context.message = 'Local memory storage is unavailable';
530
643
  }
531
644
  }
532
- #trackScheduleIfPending(context) {
533
- const followUp = this.#scheduleIfPending(context).finally(() => {
645
+ #trackScheduleIfPending(context, environment) {
646
+ const followUp = this.#scheduleIfPending(context, environment).finally(() => {
534
647
  context.followUps.delete(followUp);
535
648
  });
536
649
  context.followUps.add(followUp);
537
650
  return followUp;
538
651
  }
539
- async #run(context, bypassRetryAt = false) {
652
+ #launchRun(context, bypassRetryAt = false, preferred) {
653
+ if (context.run || context.storageBlocked || preferred && !this.#eligible(preferred))
654
+ return;
655
+ const driver = this.#driver(context, preferred);
656
+ if (!driver)
657
+ return;
658
+ context.driver = driver;
659
+ context.failedDriver = undefined;
660
+ const environment = this.#runEnvironment(context, driver);
661
+ context.runEnvironment = environment;
662
+ const run = this.#run(context, driver, environment, bypassRetryAt).catch((error) => {
663
+ context.state = 'error';
664
+ context.failedDriver = driver;
665
+ if (!bypassRetryAt && !driver.closing && !driver.detached)
666
+ context.storageBlocked = true;
667
+ context.message = sanitizeMemoryDiagnostic(error);
668
+ throw error;
669
+ }).finally(async () => {
670
+ if (context.run === settled) {
671
+ context.run = undefined;
672
+ context.driver = undefined;
673
+ context.runEnvironment = undefined;
674
+ context.abort = undefined;
675
+ }
676
+ await this.#trackScheduleIfPending(context);
677
+ this.#emitStatusChange();
678
+ });
679
+ const settled = run.catch(() => { });
680
+ context.run = settled;
681
+ return run;
682
+ }
683
+ async #run(context, driver, environment, bypassRetryAt) {
540
684
  if (this.#disposed || this.#draining || !this.#enabled || context.storageBlocked)
541
685
  return;
542
686
  context.state = 'processing';
543
687
  context.message = undefined;
688
+ const abort = new AbortController();
689
+ context.abort = abort;
690
+ const store = this.#store(context, environment);
544
691
  let lease;
692
+ const cancelled = () => driver.closing || driver.detached || !this.#enabled || this.#disposed
693
+ || abort.signal.aborted && !lease?.compromised.aborted;
545
694
  try {
546
- lease = await acquireLocalMemoryLease(this.environment, context.store.projectDirectory, this.#options.leaseOptions);
695
+ lease = await acquireLocalMemoryLease(environment, store.projectDirectory, this.#options.leaseOptions);
547
696
  }
548
697
  catch (error) {
549
- context.writerBlocked = true;
698
+ if (cancelled()) {
699
+ context.state = 'idle';
700
+ context.message = driver.closing || driver.detached || this.#disposed
701
+ ? 'Memory processing was cancelled during shutdown' : 'Memory processing was cancelled';
702
+ return;
703
+ }
704
+ this.#scheduleWriterRetry(context);
550
705
  context.state = 'error';
706
+ context.failedDriver = driver;
551
707
  context.message = safeProcessingMessage(error);
552
708
  return;
553
709
  }
554
710
  if (!lease) {
555
- context.writerBlocked = true;
711
+ this.#scheduleWriterRetry(context);
556
712
  context.state = 'blocked';
713
+ context.failedDriver = driver;
557
714
  context.message = 'Another Felan Code process owns the memory writer';
558
715
  return;
559
716
  }
560
717
  context.writerBlocked = false;
561
- const abort = new AbortController();
562
- context.abort = abort;
718
+ if (context.failedDriver === driver)
719
+ context.failedDriver = undefined;
720
+ if (context.writerTimer)
721
+ clearTimeout(context.writerTimer);
722
+ context.writerTimer = undefined;
563
723
  const abortOnLeaseCompromise = () => {
564
724
  abort.abort(new Error('Memory writer lease was lost'));
565
725
  };
@@ -569,7 +729,9 @@ export class LocalMemoryCoordinator {
569
729
  let published = false;
570
730
  let attempt;
571
731
  try {
572
- const control = await this.#reconcileRunDiagnostics(context, lease);
732
+ if (abort.signal.aborted || !this.#enabled || this.#disposed)
733
+ return;
734
+ const control = await this.#reconcileRunDiagnostics(context, environment, lease);
573
735
  if (control.autoDisabled) {
574
736
  context.state = 'disabled';
575
737
  return;
@@ -578,24 +740,25 @@ export class LocalMemoryCoordinator {
578
740
  context.state = 'scheduled';
579
741
  return;
580
742
  }
581
- const snapshot = await context.store.processingSnapshot(this.#batchSize, (checkpoint) => !isBlockedCheckpoint(context, checkpoint));
743
+ // Every pending checkpoint is a candidate so unreadable ones cannot occupy the batch.
744
+ const snapshot = await store.processingSnapshot(Number.POSITIVE_INFINITY, (checkpoint) => !isBlockedCheckpoint(context, checkpoint));
582
745
  if (snapshot.checkpoints.length === 0) {
583
746
  context.state = 'idle';
584
747
  return;
585
748
  }
586
- retainedRun = await LocalMemoryRun.create(this.environment, {
587
- projectDirectory: context.store.projectDirectory,
749
+ retainedRun = await LocalMemoryRun.create(environment, {
750
+ projectDirectory: store.projectDirectory,
588
751
  sessionDirectory: join(this.#options.sessionDir ?? join(this.#options.agentDir, 'sessions'), 'memory'),
589
752
  projectKey: context.project.key,
590
753
  projectRoot: context.project.canonicalRoot,
591
- checkpoints: snapshot.checkpoints.map(({ checkpoint }) => checkpoint),
754
+ checkpoints: snapshot.checkpoints.slice(0, this.#batchSize).map(({ checkpoint }) => checkpoint),
592
755
  baseFingerprint: snapshot.fingerprint,
593
756
  ...(control.lastDisabledRunId ? { protectedRunIds: [control.lastDisabledRunId] } : {}),
594
757
  });
595
758
  staging = retainedRun.workspace;
596
759
  const memoryDirectory = join(staging, '.memory');
597
- await hydrateMemoryDirectory(this.environment, snapshot.artifact, memoryDirectory, { memoryPath: '.memory', mode: 'read' });
598
- const manifest = await materializeMemoryInput(this.environment, {
760
+ await hydrateMemoryDirectory(environment, snapshot.artifact, memoryDirectory, { memoryPath: '.memory', mode: 'read' });
761
+ const manifest = await materializeMemoryInput(environment, {
599
762
  stagingDirectory: staging,
600
763
  checkpoints: snapshot.checkpoints.map(({ checkpoint }) => checkpoint),
601
764
  previousCheckpoints: Object.fromEntries(Object.entries(snapshot.state.processed).map(([sessionId, entry]) => [sessionId, entry.checkpoint])),
@@ -604,26 +767,29 @@ export class LocalMemoryCoordinator {
604
767
  ...(this.#options.maxTranscriptBytes === undefined
605
768
  ? {}
606
769
  : { maxTranscriptBytes: this.#options.maxTranscriptBytes }),
770
+ maxSessions: this.#batchSize,
607
771
  signal: abort.signal,
608
772
  });
609
- for (const failure of manifest.failures) {
610
- context.blocked.set(failure.checkpoint.sessionId, {
611
- leafId: failure.checkpoint.leafId,
612
- transcriptDigest: failure.checkpoint.transcriptDigest,
613
- });
614
- }
773
+ // A cancelled read reports as a failure; it must not drop evidence.
774
+ if (abort.signal.aborted || !this.#enabled || this.#disposed)
775
+ throw new Error('Memory processing was cancelled');
776
+ // A reachable Sessions service shows the failures belong to the checkpoints, not to the service.
777
+ if (manifest.failures.length > 0)
778
+ await environment.sessions.list({ scope: 'project' }, { limit: 1, signal: abort.signal });
779
+ const dropped = manifest.failures.length === 0 ? 0 : await store.dropCheckpoints(lease, manifest.failures.map(({ checkpoint, code }) => ({ checkpoint, reason: code })), this.#now());
780
+ const droppedMessage = dropped === 0 ? undefined : droppedCheckpointsMessage(dropped);
615
781
  const processedCheckpoints = snapshot.checkpoints.filter(({ checkpoint }) => manifest.sessions.some((session) => session.checkpoint.sessionId === checkpoint.sessionId
616
782
  && sameCheckpointCursor(session.checkpoint, checkpoint)));
617
783
  if (processedCheckpoints.length === 0) {
618
- context.state = 'blocked';
619
- context.message = MEMORY_INPUT_BLOCKED_MESSAGE;
620
- await retainedRun.finish('blocked', MEMORY_INPUT_BLOCKED_MESSAGE);
784
+ context.state = 'idle';
785
+ context.message = droppedMessage;
786
+ await retainedRun.finish('blocked', droppedMessage ?? 'Memory checkpoints changed before they could be read');
621
787
  return;
622
788
  }
623
789
  await retainedRun.record({ phase: 'model', checkpoints: processedCheckpoints.map(({ checkpoint }) => checkpoint) });
624
790
  if (abort.signal.aborted || !this.#enabled || this.#disposed)
625
791
  throw new Error('Memory processing was cancelled');
626
- attempt = await context.store.beginAttempt(lease, {
792
+ attempt = await store.beginAttempt(lease, {
627
793
  runId: retainedRun.metadata.sessionId,
628
794
  baseFingerprint: snapshot.fingerprint,
629
795
  checkpoints: processedCheckpoints,
@@ -635,16 +801,16 @@ export class LocalMemoryCoordinator {
635
801
  }
636
802
  context.attempt = attempt;
637
803
  this.#emitStatusChange();
638
- const result = await this.#dreamRunner({
804
+ const result = await (this.#options.dreamRunner ?? createDefaultLocalMemoryDreamRunner(environment))({
639
805
  stagingDirectory: staging,
640
806
  memoryDirectory,
641
807
  inputDirectory: join(staging, '.dreaming', 'input'),
642
808
  baseSnapshot: createMemorySnapshot(snapshot.artifact, '.memory', { mode: 'read' }),
643
809
  manifest,
644
- models: this.#options.models,
645
- ...(this.#selectedModel === undefined ? {} : { selectedModel: this.#selectedModel }),
646
- ...(this.#scopedModels === undefined ? {} : { scopedModels: this.#scopedModels }),
647
- ...(this.#classifier === undefined ? {} : { classifier: this.#classifier }),
810
+ models: driver.models,
811
+ ...(driver.selectedModel === undefined ? {} : { selectedModel: driver.selectedModel }),
812
+ ...(driver.scopedModels === undefined ? {} : { scopedModels: driver.scopedModels }),
813
+ ...(driver.classifier === undefined ? {} : { classifier: driver.classifier }),
648
814
  sessionDirectory: join(this.#options.sessionDir ?? join(this.#options.agentDir, 'sessions'), 'memory'),
649
815
  run: retainedRun,
650
816
  signal: abort.signal,
@@ -652,9 +818,9 @@ export class LocalMemoryCoordinator {
652
818
  if (abort.signal.aborted || !this.#enabled || this.#disposed)
653
819
  throw new Error('Memory processing was cancelled');
654
820
  if (result)
655
- await hydrateMemoryDirectory(this.environment, result, memoryDirectory, { replace: true, memoryPath: '.memory' });
821
+ await hydrateMemoryDirectory(environment, result, memoryDirectory, { replace: true, memoryPath: '.memory' });
656
822
  await retainedRun.record({ phase: 'validate' });
657
- const artifact = await readMemoryDirectory(this.environment, memoryDirectory, {
823
+ const artifact = await readMemoryDirectory(environment, memoryDirectory, {
658
824
  memoryPath: '.memory',
659
825
  sourceSessionIds: allowedSourceIds(snapshot, manifest),
660
826
  requireSources: true,
@@ -663,42 +829,45 @@ export class LocalMemoryCoordinator {
663
829
  phase: 'publish',
664
830
  outputFingerprint: createMemorySnapshot(artifact, '.memory').fingerprint,
665
831
  });
666
- await context.store.commit(lease, snapshot.fingerprint, artifact, processedCheckpoints, attempt);
832
+ await store.commit(lease, snapshot.fingerprint, artifact, processedCheckpoints, attempt);
667
833
  published = true;
668
- await context.store.finishAttempt(lease, attempt, { status: 'success' }, this.#now());
834
+ await store.finishAttempt(lease, attempt, { status: 'success' }, this.#now());
669
835
  await retainedRun.finish('completed');
670
- if (manifest.failures.length > 0) {
671
- context.state = 'blocked';
672
- context.message = MEMORY_INPUT_BLOCKED_MESSAGE;
673
- }
674
- else {
675
- context.state = 'idle';
676
- }
677
- this.#emitStatusChange();
836
+ context.state = 'idle';
837
+ context.failedDriver = undefined;
838
+ context.message = droppedMessage;
678
839
  }
679
840
  catch (error) {
680
- const leaseValid = await lease.verify();
681
- const explicitlyCancelled = !this.#enabled || this.#disposed
682
- || (abort.signal.aborted && !lease.compromised.aborted);
841
+ const leaseValid = await lease.verify().catch(() => false);
842
+ const explicitlyCancelled = cancelled();
683
843
  const unavailableModel = error instanceof MemoryModelUnavailableError;
684
844
  const blockedBeforeInference = retainedRun !== undefined && attempt === undefined && !explicitlyCancelled;
685
845
  if (attempt && (leaseValid || explicitlyCancelled)) {
686
846
  try {
687
- const control = await context.store.finishAttempt(explicitlyCancelled || unavailableModel ? undefined : lease, attempt, explicitlyCancelled || unavailableModel ? { status: 'cancelled' }
847
+ const control = await store.finishAttempt(explicitlyCancelled || unavailableModel ? undefined : lease, attempt, explicitlyCancelled || unavailableModel ? { status: 'cancelled' }
688
848
  : { status: 'failure', reason: sanitizeMemoryDiagnostic(error) }, this.#now());
689
849
  published ||= control.lastOutcome?.runId === attempt.runId && control.lastOutcome.status === 'success';
690
850
  }
691
851
  catch {
692
- context.storageBlocked = true;
852
+ if (!explicitlyCancelled)
853
+ context.storageBlocked = true;
693
854
  }
694
855
  }
856
+ if (explicitlyCancelled) {
857
+ context.state = 'idle';
858
+ context.message = driver.closing || driver.detached || this.#disposed
859
+ ? 'Memory processing was cancelled during shutdown' : 'Memory processing was cancelled';
860
+ await retainedRun?.finish('cancelled', error).catch(() => { });
861
+ return;
862
+ }
695
863
  if (published) {
696
864
  context.state = 'idle';
865
+ context.failedDriver = context.storageBlocked ? driver : undefined;
697
866
  context.message = 'Memory was published, but its diagnostics could not be fully saved';
698
- this.#emitStatusChange();
699
867
  return;
700
868
  }
701
869
  context.state = 'error';
870
+ context.failedDriver = driver;
702
871
  context.message = !leaseValid
703
872
  ? safeProcessingMessage(new Error('Memory writer lease was lost'))
704
873
  : safeProcessingMessage(error);
@@ -712,62 +881,68 @@ export class LocalMemoryCoordinator {
712
881
  context.storageBlocked = true;
713
882
  context.message = `Memory processing is paused: ${sanitizeMemoryDiagnostic(error)}`;
714
883
  }
715
- await retainedRun?.finish(explicitlyCancelled ? 'cancelled'
716
- : unavailableModel || blockedBeforeInference ? 'blocked'
717
- : !leaseValid ? 'interrupted' : 'failed', error).catch(() => {
884
+ await retainedRun?.finish(unavailableModel || blockedBeforeInference ? 'blocked'
885
+ : !leaseValid ? 'interrupted' : 'failed', error).catch(() => {
718
886
  context.storageBlocked = true;
719
887
  context.message = 'Memory diagnostics could not be saved; evidence remains pending';
720
888
  });
721
- this.#emitStatusChange();
722
889
  }
723
890
  finally {
724
891
  lease.compromised.removeEventListener('abort', abortOnLeaseCompromise);
725
- context.abort = undefined;
726
892
  context.attempt = undefined;
727
- await lease.release();
893
+ await lease.release().catch(error => {
894
+ if (!cancelled())
895
+ throw error;
896
+ });
728
897
  }
729
898
  }
730
- async #reconcileRunDiagnostics(context, lease) {
731
- let control = await context.store.readControl();
732
- control = await context.store.reconcileAbandonedAttempt(lease, this.#now());
899
+ async #reconcileRunDiagnostics(context, environment, lease) {
900
+ const store = this.#store(context, environment);
901
+ const previous = await store.readControl();
902
+ const control = await store.reconcileAbandonedAttempt(lease, this.#now());
903
+ if (previous.updatedAt !== control.updatedAt)
904
+ this.#emitStatusChange();
733
905
  const options = {
734
- projectDirectory: context.store.projectDirectory,
906
+ projectDirectory: store.projectDirectory,
735
907
  projectKey: context.project.key,
736
908
  sessionDirectory: join(this.#options.sessionDir ?? join(this.#options.agentDir, 'sessions'), 'memory'),
737
909
  ...(control.lastDisabledRunId ? { protectedRunIds: [control.lastDisabledRunId] } : {}),
738
910
  };
739
911
  if (control.lastOutcome)
740
- await LocalMemoryRun.reconcile(this.environment, options, control.lastOutcome);
741
- await LocalMemoryRun.reconcileAbandoned(this.environment, options, [
912
+ await LocalMemoryRun.reconcile(environment, options, control.lastOutcome);
913
+ await LocalMemoryRun.reconcileAbandoned(environment, options, [
742
914
  ...(control.activeAttempt ? [control.activeAttempt.runId] : []),
743
915
  ...(control.lastOutcome ? [control.lastOutcome.runId] : []),
744
916
  ], this.#now());
745
917
  return control;
746
918
  }
747
- async #recover(context, signal) {
919
+ async #recover(context, driver, signal) {
748
920
  let cursor;
749
921
  let recovered = false;
750
922
  const stableAfter = this.#options.recoveryStableMs ?? 30_000;
923
+ const store = this.#store(context, driver.environment);
751
924
  const stopped = () => this.#disposed || this.#draining || !this.#enabled || signal.aborted;
752
925
  try {
753
926
  do {
754
- const page = await this.environment.sessions.list({ scope: 'project' }, { ...(cursor ? { cursor } : {}), signal });
927
+ const page = await driver.environment.sessions.list({ scope: 'project' }, { ...(cursor ? { cursor } : {}), signal });
755
928
  for (const info of page.items) {
756
929
  if (stopped())
757
930
  return false;
758
931
  if (info.parent || this.#now() - info.updatedAt < stableAfter)
759
932
  continue;
933
+ let checkpoint;
760
934
  try {
761
- const checkpoint = await readMemorySessionCheckpoint(this.environment.sessions, info.id, signal);
762
- if (stopped())
763
- return false;
764
- recovered = (await context.store.recordCheckpoint(checkpoint)) || recovered;
935
+ checkpoint = await readMemorySessionCheckpoint(driver.environment.sessions, info.id, signal);
765
936
  }
766
937
  catch {
767
938
  if (stopped())
768
939
  return false;
769
940
  // A session that cannot be read now is skipped; its live checkpoint or a later run records it.
941
+ continue;
770
942
  }
943
+ if (stopped())
944
+ return false;
945
+ recovered = (await store.recordCheckpoint(checkpoint)) || recovered;
771
946
  }
772
947
  cursor = page.nextCursor;
773
948
  } while (cursor);
@@ -789,13 +964,23 @@ export class LocalMemoryCoordinator {
789
964
  return true;
790
965
  }
791
966
  #emitStatusChange() {
792
- for (const listener of [...this.#statusListeners]) {
967
+ for (const listener of [...this.#statusListeners, ...[...this.#attachments].flatMap(attachment => [...attachment.listeners])]) {
793
968
  try {
794
969
  listener();
795
970
  }
796
971
  catch { }
797
972
  }
798
973
  }
974
+ #cancelTimers(context) {
975
+ this.#cancelScheduledRun(context);
976
+ if (context.recoveryTimer)
977
+ clearTimeout(context.recoveryTimer);
978
+ if (context.writerTimer)
979
+ clearTimeout(context.writerTimer);
980
+ context.recoveryTimer = undefined;
981
+ context.writerTimer = undefined;
982
+ context.writerBlocked = false;
983
+ }
799
984
  #cancelScheduledRun(context) {
800
985
  if (context.timer)
801
986
  clearTimeout(context.timer);
@@ -843,6 +1028,9 @@ function allowedSourceIds(snapshot, manifest) {
843
1028
  ...manifest.sessions.map(({ checkpoint }) => checkpoint.sessionId),
844
1029
  ])];
845
1030
  }
1031
+ function droppedCheckpointsMessage(count) {
1032
+ return `Dropped ${count} memory checkpoint${count === 1 ? '' : 's'} that could not be read`;
1033
+ }
846
1034
  function safeProcessingMessage(error) {
847
1035
  if (error instanceof MemoryModelUnavailableError) {
848
1036
  return 'Low-tier memory model is unavailable; evidence remains pending';