@felan-ai/ext-memory 0.10.1 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -7
- package/dist/extension.d.ts.map +1 -1
- package/dist/extension.js +30 -9
- package/dist/extension.js.map +1 -1
- package/dist/local/control.d.ts +3 -2
- package/dist/local/control.d.ts.map +1 -1
- package/dist/local/control.js.map +1 -1
- package/dist/local/coordinator.d.ts +28 -51
- package/dist/local/coordinator.d.ts.map +1 -1
- package/dist/local/coordinator.js +446 -258
- package/dist/local/coordinator.js.map +1 -1
- package/dist/local/dreamer.d.ts +3 -1
- package/dist/local/dreamer.d.ts.map +1 -1
- package/dist/local/dreamer.js +2 -2
- package/dist/local/dreamer.js.map +1 -1
- package/dist/local/lease.d.ts.map +1 -1
- package/dist/local/lease.js +12 -7
- package/dist/local/lease.js.map +1 -1
- package/dist/local/shared.d.ts +7 -0
- package/dist/local/shared.d.ts.map +1 -0
- package/dist/local/shared.js +30 -0
- package/dist/local/shared.js.map +1 -0
- package/dist/local/store.d.ts +12 -0
- package/dist/local/store.d.ts.map +1 -1
- package/dist/local/store.js +34 -3
- package/dist/local/store.js.map +1 -1
- package/package.json +3 -3
|
@@ -11,48 +11,73 @@ import {} from '@felan-ai/agent-core';
|
|
|
11
11
|
import { MemoryModelUnavailableError } from './dreamer.js';
|
|
12
12
|
import { LocalMemoryControlError } from './store.js';
|
|
13
13
|
import { sanitizeMemoryDiagnostic } from '../diagnostics.js';
|
|
14
|
-
const MEMORY_INPUT_BLOCKED_MESSAGE = '
|
|
14
|
+
const MEMORY_INPUT_BLOCKED_MESSAGE = 'Memory checkpoints are paused after a failed run; evidence remains pending';
|
|
15
15
|
const MEMORY_AUTOMATIC_UPDATE_THRESHOLD = 5;
|
|
16
16
|
const MEMORY_AUTOMATIC_INTERVAL_MS = 24 * 60 * 60 * 1_000;
|
|
17
17
|
const MEMORY_DREAM_INPUT_MAX_BYTES = 10 * 1024 * 1024;
|
|
18
18
|
const MEMORY_MAX_BATCH_SESSIONS = 8;
|
|
19
19
|
export class LocalMemoryCoordinator {
|
|
20
|
-
environment;
|
|
21
20
|
#options;
|
|
22
21
|
#contexts = new Map();
|
|
23
|
-
#
|
|
22
|
+
#attachments = new Set();
|
|
24
23
|
#statusListeners = new Set();
|
|
25
|
-
#dreamRunner;
|
|
26
24
|
#batchSize;
|
|
27
25
|
#enabled;
|
|
28
|
-
#selectedModel;
|
|
29
|
-
#scopedModels;
|
|
30
|
-
#classifier;
|
|
31
26
|
#disposed = false;
|
|
27
|
+
#disposal;
|
|
32
28
|
#draining = false;
|
|
33
|
-
#monitor;
|
|
34
|
-
#monitoring;
|
|
35
29
|
#enableVersion = 0;
|
|
36
|
-
constructor(
|
|
37
|
-
this.environment = environment;
|
|
30
|
+
constructor(options) {
|
|
38
31
|
this.#options = options;
|
|
39
32
|
this.#batchSize = boundedMemoryBatchSize(options.batchSize);
|
|
40
33
|
this.#enabled = options.enabled !== false;
|
|
41
|
-
this.#selectedModel = options.selectedModel;
|
|
42
|
-
this.#scopedModels = options.scopedModels === undefined ? undefined : [...options.scopedModels];
|
|
43
|
-
this.#classifier = options.classifier;
|
|
44
|
-
this.#dreamRunner = options.dreamRunner ?? createDefaultLocalMemoryDreamRunner(this.environment);
|
|
45
|
-
if (this.#enabled)
|
|
46
|
-
this.#startMonitor();
|
|
47
34
|
}
|
|
48
|
-
|
|
49
|
-
|
|
35
|
+
attach(options) {
|
|
36
|
+
if (this.#disposed)
|
|
37
|
+
throw new Error('Local memory coordinator is closed');
|
|
38
|
+
const state = {
|
|
39
|
+
...options,
|
|
40
|
+
selectedModel: options.selectedModel,
|
|
41
|
+
scopedModels: options.scopedModels === undefined ? undefined : [...options.scopedModels],
|
|
42
|
+
classifier: options.classifier,
|
|
43
|
+
projections: new Map(),
|
|
44
|
+
listeners: new Set(),
|
|
45
|
+
draining: false,
|
|
46
|
+
closing: false,
|
|
47
|
+
detached: false,
|
|
48
|
+
detaching: undefined,
|
|
49
|
+
};
|
|
50
|
+
this.#attachments.add(state);
|
|
51
|
+
const attachment = {
|
|
52
|
+
createSessionHost: options => new LocalMemorySessionHost(attachment, options),
|
|
53
|
+
readCurrent: (cwd, root) => this.#readCurrent(state, cwd, root),
|
|
54
|
+
recordCheckpoint: (cwd, checkpoint) => this.#recordCheckpoint(state, cwd, checkpoint),
|
|
55
|
+
status: cwd => this.#status(state, cwd),
|
|
56
|
+
runNow: cwd => this.#runNow(state, cwd),
|
|
57
|
+
retryProject: cwd => this.#retryProject(state, cwd),
|
|
58
|
+
canonicalDirectory: async (cwd) => this.#store(await this.#context(state, cwd), state.environment).currentDirectory,
|
|
59
|
+
setSelectedModel: model => this.#setModelSelection(state, model, state.scopedModels, state.classifier),
|
|
60
|
+
setModelSelection: (model, scopedModels, classifier) => this.#setModelSelection(state, model, scopedModels, classifier),
|
|
61
|
+
subscribeStatusChanges: listener => {
|
|
62
|
+
if (state.detached || this.#disposed)
|
|
63
|
+
return () => { };
|
|
64
|
+
state.listeners.add(listener);
|
|
65
|
+
return () => { state.listeners.delete(listener); };
|
|
66
|
+
},
|
|
67
|
+
busy: () => [...this.#contexts.values()].some(context => context.driver === state || context.recoveryDriver === state),
|
|
68
|
+
settle: mode => this.#settleAttachment(state, mode),
|
|
69
|
+
detach: () => {
|
|
70
|
+
state.detaching ??= this.#detach(state);
|
|
71
|
+
return state.detaching;
|
|
72
|
+
},
|
|
73
|
+
};
|
|
74
|
+
return attachment;
|
|
50
75
|
}
|
|
51
76
|
isEnabled() {
|
|
52
77
|
return this.#enabled;
|
|
53
78
|
}
|
|
54
79
|
hasActiveWork() {
|
|
55
|
-
return
|
|
80
|
+
return [...this.#contexts.values()].some(context => !!context.run || !!context.ready || !!context.recovery || context.followUps.size > 0);
|
|
56
81
|
}
|
|
57
82
|
subscribeStatusChanges(listener) {
|
|
58
83
|
if (this.#disposed)
|
|
@@ -60,34 +85,28 @@ export class LocalMemoryCoordinator {
|
|
|
60
85
|
this.#statusListeners.add(listener);
|
|
61
86
|
return () => this.#statusListeners.delete(listener);
|
|
62
87
|
}
|
|
63
|
-
setModelSelection(model, scopedModels, classifier) {
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
this.#selectedModel = model;
|
|
88
|
+
#setModelSelection(attachment, model, scopedModels, classifier) {
|
|
89
|
+
if (attachment.detached || this.#disposed)
|
|
90
|
+
return;
|
|
91
|
+
attachment.selectedModel = model;
|
|
92
|
+
attachment.scopedModels = scopedModels === undefined ? undefined : [...scopedModels];
|
|
93
|
+
attachment.classifier = classifier;
|
|
70
94
|
for (const context of this.#contexts.values()) {
|
|
95
|
+
if (!context.attachments.has(attachment))
|
|
96
|
+
continue;
|
|
71
97
|
context.blocked.clear();
|
|
72
98
|
if (context.initialized)
|
|
73
|
-
void this.#trackScheduleIfPending(context);
|
|
99
|
+
void this.#trackScheduleIfPending(context, attachment.environment);
|
|
74
100
|
}
|
|
75
101
|
}
|
|
76
102
|
setEnabled(enabled) {
|
|
77
103
|
this.#enableVersion += 1;
|
|
78
|
-
enabled = enabled && !this.#options.unavailableReason;
|
|
79
104
|
this.#enabled = enabled;
|
|
80
|
-
if (enabled)
|
|
81
|
-
this.#startMonitor();
|
|
82
|
-
else if (this.#monitor) {
|
|
83
|
-
clearInterval(this.#monitor);
|
|
84
|
-
this.#monitor = undefined;
|
|
85
|
-
}
|
|
86
105
|
for (const context of this.#contexts.values()) {
|
|
87
106
|
if (!enabled)
|
|
88
107
|
context.recoveryAbort?.abort();
|
|
89
108
|
if (!enabled)
|
|
90
|
-
this.#
|
|
109
|
+
this.#cancelTimers(context);
|
|
91
110
|
if (!enabled)
|
|
92
111
|
this.#cancelContext(context, 'Memory processing was disabled', true);
|
|
93
112
|
if (enabled) {
|
|
@@ -107,26 +126,27 @@ export class LocalMemoryCoordinator {
|
|
|
107
126
|
}
|
|
108
127
|
this.#emitStatusChange();
|
|
109
128
|
}
|
|
110
|
-
async status(cwd) {
|
|
129
|
+
async #status(attachment, cwd) {
|
|
111
130
|
try {
|
|
112
|
-
const context = await this.#context(cwd);
|
|
113
|
-
const
|
|
114
|
-
const
|
|
131
|
+
const context = await this.#context(attachment, cwd);
|
|
132
|
+
const store = this.#store(context, attachment.environment);
|
|
133
|
+
const state = await store.status();
|
|
134
|
+
const control = await store.readControl();
|
|
115
135
|
const processed = Object.values(state.processed);
|
|
116
136
|
const lastProcessedAt = processed
|
|
117
137
|
.map((entry) => entry.processedAt)
|
|
118
138
|
.sort()
|
|
119
139
|
.at(-1);
|
|
120
140
|
return {
|
|
121
|
-
enabled: this.#enabled && !control.autoDisabled,
|
|
122
|
-
state: !this.#enabled || control.autoDisabled ? 'disabled'
|
|
141
|
+
enabled: this.#enabled && attachment.scheduling && !control.autoDisabled,
|
|
142
|
+
state: !this.#enabled || !attachment.scheduling || control.autoDisabled ? 'disabled'
|
|
123
143
|
: control.nextRetryAt && Date.parse(control.nextRetryAt) > this.#now() ? 'scheduled'
|
|
124
144
|
: context.state === 'error' ? 'error'
|
|
125
145
|
: control.activeAttempt ? 'processing' : context.state,
|
|
126
146
|
pendingCheckpoints: Object.keys(state.pending).length,
|
|
127
147
|
memoryFingerprint: state.memoryFingerprint,
|
|
128
148
|
...(lastProcessedAt === undefined ? {} : { lastProcessedAt }),
|
|
129
|
-
...(context.message ||
|
|
149
|
+
...(context.message || attachment.unavailableReason ? { message: context.message ?? attachment.unavailableReason } : {}),
|
|
130
150
|
consecutiveFailures: control.consecutiveFailures,
|
|
131
151
|
...(control.nextRetryAt ? { nextRetryAt: control.nextRetryAt } : {}),
|
|
132
152
|
...(control.autoDisabled ? { autoDisabled: control.autoDisabled } : {}),
|
|
@@ -137,37 +157,39 @@ export class LocalMemoryCoordinator {
|
|
|
137
157
|
}
|
|
138
158
|
catch (error) {
|
|
139
159
|
return {
|
|
140
|
-
enabled: this.#enabled,
|
|
141
|
-
state: this.#enabled ? 'error' : 'disabled',
|
|
160
|
+
enabled: this.#enabled && attachment.scheduling,
|
|
161
|
+
state: this.#enabled && attachment.scheduling ? 'error' : 'disabled',
|
|
142
162
|
pendingCheckpoints: 0,
|
|
143
163
|
message: error instanceof LocalMemoryControlError ? error.message : 'Local memory storage is unavailable',
|
|
144
164
|
};
|
|
145
165
|
}
|
|
146
166
|
}
|
|
147
|
-
async readCurrent(cwd, sessionStorageRoot) {
|
|
148
|
-
const context = await this.#context(cwd);
|
|
149
|
-
const
|
|
167
|
+
async #readCurrent(attachment, cwd, sessionStorageRoot) {
|
|
168
|
+
const context = await this.#context(attachment, cwd);
|
|
169
|
+
const store = this.#store(context, attachment.environment);
|
|
170
|
+
const snapshot = await store.readCurrent();
|
|
150
171
|
const projectionKey = `${context.project.key}:${sessionStorageRoot}`;
|
|
151
|
-
const memoryPath = join(sessionStorageRoot,
|
|
152
|
-
if (
|
|
153
|
-
const projection = await
|
|
154
|
-
|
|
172
|
+
const memoryPath = join(sessionStorageRoot, store.projectionName);
|
|
173
|
+
if (attachment.projections.get(projectionKey) !== snapshot.fingerprint) {
|
|
174
|
+
const projection = await store.projectTo(sessionStorageRoot, snapshot);
|
|
175
|
+
if (!attachment.detached)
|
|
176
|
+
attachment.projections.set(projectionKey, snapshot.fingerprint);
|
|
155
177
|
return projection;
|
|
156
178
|
}
|
|
157
179
|
return createMemoryProjectionSnapshot(snapshot, memoryPath);
|
|
158
180
|
}
|
|
159
|
-
async recordCheckpoint(cwd, checkpoint) {
|
|
160
|
-
const context = await this.#context(cwd);
|
|
161
|
-
if (await context.
|
|
181
|
+
async #recordCheckpoint(attachment, cwd, checkpoint) {
|
|
182
|
+
const context = await this.#context(attachment, cwd);
|
|
183
|
+
if (await this.#store(context, attachment.environment).recordCheckpoint(checkpoint)) {
|
|
162
184
|
context.blocked.delete(checkpoint.sessionId);
|
|
163
|
-
await this.#trackScheduleIfPending(context);
|
|
185
|
+
await this.#trackScheduleIfPending(context, attachment.environment);
|
|
164
186
|
this.#emitStatusChange();
|
|
165
187
|
}
|
|
166
188
|
}
|
|
167
|
-
async runNow(cwd) {
|
|
168
|
-
if (!this.#
|
|
169
|
-
return this
|
|
170
|
-
const context = await this.#context(cwd);
|
|
189
|
+
async #runNow(attachment, cwd) {
|
|
190
|
+
if (!this.#eligible(attachment))
|
|
191
|
+
return this.#status(attachment, cwd);
|
|
192
|
+
const context = await this.#context(attachment, cwd);
|
|
171
193
|
this.#cancelScheduledRun(context);
|
|
172
194
|
while (context.run || context.followUps.size > 0) {
|
|
173
195
|
await Promise.allSettled([
|
|
@@ -177,61 +199,114 @@ export class LocalMemoryCoordinator {
|
|
|
177
199
|
this.#cancelScheduledRun(context);
|
|
178
200
|
}
|
|
179
201
|
context.blocked.clear();
|
|
180
|
-
|
|
181
|
-
context.
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
}
|
|
185
|
-
finally {
|
|
186
|
-
if (context.run === run)
|
|
187
|
-
context.run = undefined;
|
|
188
|
-
await this.#trackScheduleIfPending(context);
|
|
189
|
-
}
|
|
190
|
-
return this.status(cwd);
|
|
202
|
+
context.storageBlocked = false;
|
|
203
|
+
context.message = undefined;
|
|
204
|
+
await this.#launchRun(context, true, attachment);
|
|
205
|
+
return this.#status(attachment, cwd);
|
|
191
206
|
}
|
|
192
|
-
async retryProject(cwd) {
|
|
193
|
-
if (!this.#
|
|
194
|
-
return this
|
|
195
|
-
const context = await this.#context(cwd);
|
|
196
|
-
if (!this.#
|
|
197
|
-
return this
|
|
207
|
+
async #retryProject(attachment, cwd) {
|
|
208
|
+
if (!this.#eligible(attachment))
|
|
209
|
+
return this.#status(attachment, cwd);
|
|
210
|
+
const context = await this.#context(attachment, cwd);
|
|
211
|
+
if (!this.#eligible(attachment))
|
|
212
|
+
return this.#status(attachment, cwd);
|
|
198
213
|
const enableVersion = this.#enableVersion;
|
|
199
214
|
this.#cancelScheduledRun(context);
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
}
|
|
203
|
-
catch (error) {
|
|
204
|
-
throw error;
|
|
205
|
-
}
|
|
215
|
+
await this.#store(context, attachment.environment).resetProjectControl(this.#now(), () => this.#eligible(attachment) && this.#enableVersion === enableVersion);
|
|
216
|
+
this.#emitStatusChange();
|
|
206
217
|
context.abort?.abort(new Error('Memory processing was explicitly retried'));
|
|
207
218
|
if (context.run)
|
|
208
219
|
await context.run;
|
|
209
220
|
context.blocked.clear();
|
|
210
221
|
context.storageBlocked = false;
|
|
211
222
|
context.message = undefined;
|
|
212
|
-
await this
|
|
213
|
-
return this
|
|
223
|
+
await this.#runNow(attachment, cwd);
|
|
224
|
+
return this.#status(attachment, cwd);
|
|
225
|
+
}
|
|
226
|
+
#eligible(attachment) {
|
|
227
|
+
return this.#enabled && !this.#disposed && !this.#draining
|
|
228
|
+
&& attachment.scheduling && !attachment.draining && !attachment.closing && !attachment.detached;
|
|
229
|
+
}
|
|
230
|
+
#driver(context, preferred) {
|
|
231
|
+
if (preferred && context.attachments.has(preferred) && this.#eligible(preferred))
|
|
232
|
+
return preferred;
|
|
233
|
+
return [...this.#attachments].reverse().find(attachment => context.attachments.has(attachment) && this.#eligible(attachment));
|
|
234
|
+
}
|
|
235
|
+
#liveMember(context) {
|
|
236
|
+
return [...this.#attachments].reverse().find(attachment => context.attachments.has(attachment)
|
|
237
|
+
&& attachment.scheduling && !attachment.draining && !attachment.closing && !attachment.detached);
|
|
238
|
+
}
|
|
239
|
+
#runEnvironment(context, driver) {
|
|
240
|
+
// The daemon seals closing runtimes outside their own cleanup async context.
|
|
241
|
+
const current = () => (driver.closing || driver.detached ? this.#liveMember(context) ?? driver : driver).environment;
|
|
242
|
+
const environment = {
|
|
243
|
+
get runtime() { return current().runtime; },
|
|
244
|
+
get sessions() { return current().sessions; },
|
|
245
|
+
};
|
|
246
|
+
Object.defineProperty(environment, 'openWorker', { get: () => current().openWorker });
|
|
247
|
+
return environment;
|
|
248
|
+
}
|
|
249
|
+
async #settleAttachment(attachment, mode) {
|
|
250
|
+
attachment.draining = true;
|
|
251
|
+
if (mode === 'force')
|
|
252
|
+
attachment.closing = true;
|
|
253
|
+
for (const context of this.#contexts.values()) {
|
|
254
|
+
if (context.recoveryDriver === attachment)
|
|
255
|
+
context.recoveryAbort?.abort();
|
|
256
|
+
if (mode === 'force' && context.driver === attachment) {
|
|
257
|
+
this.#cancelContext(context, 'Memory processing was cancelled during shutdown', false);
|
|
258
|
+
}
|
|
259
|
+
if (!this.#driver(context))
|
|
260
|
+
this.#cancelTimers(context);
|
|
261
|
+
}
|
|
262
|
+
while (true) {
|
|
263
|
+
const work = [...this.#contexts.values()].flatMap(context => [
|
|
264
|
+
...(context.driver === attachment ? [...(context.run ? [context.run] : []), ...context.followUps] : []),
|
|
265
|
+
...(context.recoveryDriver === attachment && context.recovery ? [context.recovery] : []),
|
|
266
|
+
]);
|
|
267
|
+
if (!work.length)
|
|
268
|
+
break;
|
|
269
|
+
await Promise.allSettled(work);
|
|
270
|
+
}
|
|
271
|
+
for (const context of this.#contexts.values()) {
|
|
272
|
+
if (mode === 'force' && context.failedDriver === attachment) {
|
|
273
|
+
context.storageBlocked = false;
|
|
274
|
+
context.blocked.clear();
|
|
275
|
+
if (context.state === 'error' || context.state === 'blocked')
|
|
276
|
+
context.state = 'idle';
|
|
277
|
+
context.message = undefined;
|
|
278
|
+
context.failedDriver = undefined;
|
|
279
|
+
}
|
|
280
|
+
if (!context.attachments.has(attachment) || !context.initialized)
|
|
281
|
+
continue;
|
|
282
|
+
this.#startRecovery(context);
|
|
283
|
+
void this.#trackScheduleIfPending(context);
|
|
284
|
+
}
|
|
214
285
|
}
|
|
215
|
-
async
|
|
216
|
-
|
|
217
|
-
|
|
286
|
+
async #detach(attachment) {
|
|
287
|
+
attachment.closing = true;
|
|
288
|
+
await this.#settleAttachment(attachment, 'force');
|
|
289
|
+
attachment.detached = true;
|
|
290
|
+
this.#attachments.delete(attachment);
|
|
291
|
+
attachment.listeners.clear();
|
|
292
|
+
attachment.projections.clear();
|
|
293
|
+
for (const context of this.#contexts.values()) {
|
|
294
|
+
context.attachments.delete(attachment);
|
|
295
|
+
if (!this.#driver(context))
|
|
296
|
+
this.#cancelTimers(context);
|
|
297
|
+
}
|
|
218
298
|
}
|
|
219
299
|
async drain() {
|
|
220
300
|
this.#draining = true;
|
|
221
|
-
if (this.#monitor)
|
|
222
|
-
clearInterval(this.#monitor);
|
|
223
|
-
this.#monitor = undefined;
|
|
224
301
|
for (const context of this.#contexts.values()) {
|
|
225
302
|
context.recoveryAbort?.abort();
|
|
226
|
-
this.#
|
|
303
|
+
this.#cancelTimers(context);
|
|
227
304
|
}
|
|
228
305
|
while (true) {
|
|
229
306
|
const work = [...this.#contexts.values()].flatMap(context => [
|
|
230
307
|
...(context.run ? [context.run] : []), ...(context.ready ? [context.ready] : []),
|
|
231
308
|
...(context.recovery ? [context.recovery] : []), ...context.followUps,
|
|
232
309
|
]);
|
|
233
|
-
if (this.#monitoring)
|
|
234
|
-
work.push(this.#monitoring);
|
|
235
310
|
if (!work.length)
|
|
236
311
|
return;
|
|
237
312
|
const results = await Promise.allSettled(work);
|
|
@@ -240,103 +315,121 @@ export class LocalMemoryCoordinator {
|
|
|
240
315
|
throw new AggregateError(failures, 'Memory drain could not settle all work.');
|
|
241
316
|
}
|
|
242
317
|
}
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
318
|
+
dispose() {
|
|
319
|
+
this.#disposal ??= this.#dispose();
|
|
320
|
+
return this.#disposal;
|
|
321
|
+
}
|
|
322
|
+
async #dispose() {
|
|
246
323
|
this.#disposed = true;
|
|
247
|
-
if (this.#monitor)
|
|
248
|
-
clearInterval(this.#monitor);
|
|
249
|
-
this.#monitor = undefined;
|
|
250
324
|
for (const context of this.#contexts.values()) {
|
|
251
325
|
context.recoveryAbort?.abort();
|
|
252
|
-
this.#
|
|
326
|
+
this.#cancelTimers(context);
|
|
253
327
|
this.#cancelContext(context, 'Memory processing was cancelled during shutdown', false);
|
|
254
328
|
}
|
|
255
329
|
while (true) {
|
|
256
|
-
const work = [...this.#contexts.values()].flatMap(
|
|
330
|
+
const work = [...this.#contexts.values()].flatMap(context => [
|
|
257
331
|
...(context.run ? [context.run] : []),
|
|
258
332
|
...(context.ready ? [context.ready] : []),
|
|
259
333
|
...(context.recovery ? [context.recovery] : []),
|
|
260
334
|
...context.followUps,
|
|
261
335
|
]);
|
|
262
|
-
if (
|
|
263
|
-
work.push(this.#monitoring);
|
|
264
|
-
if (work.length === 0)
|
|
336
|
+
if (!work.length)
|
|
265
337
|
break;
|
|
266
338
|
await Promise.allSettled(work);
|
|
267
339
|
}
|
|
340
|
+
for (const attachment of this.#attachments) {
|
|
341
|
+
attachment.detached = true;
|
|
342
|
+
attachment.listeners.clear();
|
|
343
|
+
attachment.projections.clear();
|
|
344
|
+
}
|
|
345
|
+
this.#attachments.clear();
|
|
268
346
|
this.#statusListeners.clear();
|
|
269
347
|
}
|
|
270
|
-
async #context(cwd) {
|
|
271
|
-
if (this.#disposed)
|
|
348
|
+
async #context(attachment, cwd) {
|
|
349
|
+
if (this.#disposed || attachment.detached)
|
|
272
350
|
throw new Error('Local memory coordinator is closed');
|
|
273
|
-
const project = await resolveLocalMemoryProject(
|
|
274
|
-
if (this.#disposed)
|
|
351
|
+
const project = await resolveLocalMemoryProject(attachment.environment, cwd);
|
|
352
|
+
if (this.#disposed || attachment.detached)
|
|
275
353
|
throw new Error('Local memory coordinator is closed');
|
|
276
354
|
const existing = this.#contexts.get(project.key);
|
|
277
355
|
if (existing) {
|
|
278
|
-
|
|
356
|
+
existing.attachments.add(attachment);
|
|
357
|
+
await this.#initialize(existing, attachment);
|
|
279
358
|
return existing;
|
|
280
359
|
}
|
|
281
|
-
const store = new LocalMemoryStore(this.environment, this.#options.agentDir, project, {
|
|
282
|
-
...(this.#options.retryDelaysMs ? { retryDelaysMs: this.#options.retryDelaysMs } : {}),
|
|
283
|
-
});
|
|
284
360
|
const context = {
|
|
285
361
|
project,
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
362
|
+
stores: new WeakMap(),
|
|
363
|
+
attachments: new Set([attachment]),
|
|
364
|
+
driver: undefined,
|
|
365
|
+
failedDriver: undefined,
|
|
366
|
+
recoveryDriver: undefined,
|
|
289
367
|
ready: undefined,
|
|
290
368
|
initialized: false,
|
|
291
369
|
recovery: undefined,
|
|
292
370
|
recoveryAbort: undefined,
|
|
293
371
|
recoveryRetryAt: undefined,
|
|
372
|
+
recoveryTimer: undefined,
|
|
294
373
|
recoveryCompleted: false,
|
|
295
374
|
diagnosticsReconciled: false,
|
|
296
375
|
timer: undefined,
|
|
297
376
|
timerDueAt: undefined,
|
|
298
377
|
run: undefined,
|
|
378
|
+
runEnvironment: undefined,
|
|
299
379
|
followUps: new Set(),
|
|
300
380
|
abort: undefined,
|
|
301
381
|
attempt: undefined,
|
|
302
382
|
storageBlocked: false,
|
|
303
383
|
writerBlocked: false,
|
|
304
|
-
|
|
384
|
+
writerTimer: undefined,
|
|
305
385
|
blocked: new Map(),
|
|
306
386
|
state: this.#enabled ? 'idle' : 'disabled',
|
|
307
387
|
message: undefined,
|
|
308
388
|
};
|
|
309
389
|
this.#contexts.set(project.key, context);
|
|
310
|
-
await this.#initialize(context);
|
|
390
|
+
await this.#initialize(context, attachment);
|
|
311
391
|
return context;
|
|
312
392
|
}
|
|
313
|
-
|
|
393
|
+
#store(context, environment) {
|
|
394
|
+
let store = context.stores.get(environment);
|
|
395
|
+
if (!store) {
|
|
396
|
+
store = new LocalMemoryStore(environment, this.#options.agentDir, context.project, {
|
|
397
|
+
...(this.#options.retryDelaysMs ? { retryDelaysMs: this.#options.retryDelaysMs } : {}),
|
|
398
|
+
});
|
|
399
|
+
context.stores.set(environment, store);
|
|
400
|
+
}
|
|
401
|
+
return store;
|
|
402
|
+
}
|
|
403
|
+
async #initialize(context, attachment) {
|
|
314
404
|
if (context.ready) {
|
|
315
405
|
await context.ready;
|
|
316
406
|
return;
|
|
317
407
|
}
|
|
318
408
|
if (context.initialized && context.diagnosticsReconciled) {
|
|
319
409
|
this.#startRecovery(context);
|
|
410
|
+
void this.#trackScheduleIfPending(context, attachment.environment);
|
|
320
411
|
return;
|
|
321
412
|
}
|
|
322
|
-
const ready = this.#initializeContext(context).finally(() => {
|
|
413
|
+
const ready = this.#initializeContext(context, attachment).finally(() => {
|
|
323
414
|
if (context.ready === ready)
|
|
324
415
|
context.ready = undefined;
|
|
325
416
|
});
|
|
326
417
|
context.ready = ready;
|
|
327
418
|
await ready;
|
|
328
419
|
}
|
|
329
|
-
async #initializeContext(context) {
|
|
420
|
+
async #initializeContext(context, attachment) {
|
|
421
|
+
const environment = attachment.environment;
|
|
422
|
+
const store = this.#store(context, environment);
|
|
330
423
|
if (!context.initialized) {
|
|
331
|
-
await
|
|
424
|
+
await store.initialize(this.#now()).then(() => {
|
|
332
425
|
context.initialized = true;
|
|
333
426
|
context.state = this.#enabled ? 'idle' : 'disabled';
|
|
334
427
|
context.message = undefined;
|
|
335
428
|
}).catch(async (error) => {
|
|
336
429
|
if (!(error instanceof LocalMemoryControlError))
|
|
337
430
|
throw error;
|
|
338
|
-
const current = await
|
|
339
|
-
if (current.fingerprint !== (await
|
|
431
|
+
const current = await store.readCurrent();
|
|
432
|
+
if (current.fingerprint !== (await store.status()).memoryFingerprint)
|
|
340
433
|
throw error;
|
|
341
434
|
context.message = error.message;
|
|
342
435
|
});
|
|
@@ -345,11 +438,16 @@ export class LocalMemoryCoordinator {
|
|
|
345
438
|
return;
|
|
346
439
|
if (!context.diagnosticsReconciled) {
|
|
347
440
|
try {
|
|
348
|
-
const lease = await acquireLocalMemoryLease(
|
|
441
|
+
const lease = await acquireLocalMemoryLease(environment, store.projectDirectory, this.#options.leaseOptions);
|
|
349
442
|
if (lease) {
|
|
350
443
|
try {
|
|
351
|
-
await this.#reconcileRunDiagnostics(context, lease);
|
|
444
|
+
await this.#reconcileRunDiagnostics(context, environment, lease);
|
|
352
445
|
context.diagnosticsReconciled = true;
|
|
446
|
+
if (context.storageBlocked) {
|
|
447
|
+
context.storageBlocked = false;
|
|
448
|
+
context.failedDriver = undefined;
|
|
449
|
+
context.message = undefined;
|
|
450
|
+
}
|
|
353
451
|
}
|
|
354
452
|
finally {
|
|
355
453
|
await lease.release();
|
|
@@ -357,24 +455,34 @@ export class LocalMemoryCoordinator {
|
|
|
357
455
|
}
|
|
358
456
|
}
|
|
359
457
|
catch (error) {
|
|
360
|
-
context.diagnosticsReconciled = true;
|
|
361
458
|
context.storageBlocked = true;
|
|
459
|
+
context.failedDriver = attachment;
|
|
362
460
|
context.message = `Memory diagnostics recovery is blocked: ${sanitizeMemoryDiagnostic(error)}`;
|
|
363
461
|
}
|
|
364
462
|
}
|
|
365
463
|
if (this.#disposed)
|
|
366
464
|
return;
|
|
367
465
|
if (this.#enabled)
|
|
368
|
-
void this.#trackScheduleIfPending(context);
|
|
466
|
+
void this.#trackScheduleIfPending(context, environment);
|
|
369
467
|
this.#startRecovery(context);
|
|
370
468
|
}
|
|
371
469
|
#startRecovery(context) {
|
|
372
|
-
if (this.#
|
|
373
|
-
|
|
374
|
-
|
|
470
|
+
if (this.#options.recover === false || context.recoveryCompleted || context.recovery)
|
|
471
|
+
return;
|
|
472
|
+
const driver = this.#driver(context);
|
|
473
|
+
if (!driver)
|
|
474
|
+
return;
|
|
475
|
+
if (context.recoveryRetryAt !== undefined && this.#now() < context.recoveryRetryAt) {
|
|
476
|
+
this.#scheduleRecoveryRetry(context);
|
|
375
477
|
return;
|
|
478
|
+
}
|
|
479
|
+
if (context.recoveryTimer)
|
|
480
|
+
clearTimeout(context.recoveryTimer);
|
|
481
|
+
context.recoveryTimer = undefined;
|
|
482
|
+
context.recoveryDriver = driver;
|
|
376
483
|
const abort = new AbortController();
|
|
377
|
-
|
|
484
|
+
context.recoveryAbort = abort;
|
|
485
|
+
const recovery = this.#recover(context, driver, abort.signal).then(completed => {
|
|
378
486
|
if (completed)
|
|
379
487
|
context.recoveryCompleted = true;
|
|
380
488
|
}).catch(() => {
|
|
@@ -383,71 +491,70 @@ export class LocalMemoryCoordinator {
|
|
|
383
491
|
if (context.recovery === recovery) {
|
|
384
492
|
context.recovery = undefined;
|
|
385
493
|
context.recoveryAbort = undefined;
|
|
494
|
+
context.recoveryDriver = undefined;
|
|
386
495
|
}
|
|
496
|
+
if (abort.signal.aborted)
|
|
497
|
+
this.#startRecovery(context);
|
|
498
|
+
else
|
|
499
|
+
this.#scheduleRecoveryRetry(context);
|
|
500
|
+
this.#emitStatusChange();
|
|
387
501
|
});
|
|
388
502
|
context.recovery = recovery;
|
|
389
|
-
context.recoveryAbort = abort;
|
|
390
503
|
}
|
|
391
|
-
#
|
|
392
|
-
|
|
393
|
-
if (this.#disposed || this.#draining || this.#monitor)
|
|
504
|
+
#scheduleRecoveryRetry(context) {
|
|
505
|
+
if (context.recoveryTimer || context.recoveryCompleted || context.recoveryRetryAt === undefined || !this.#driver(context))
|
|
394
506
|
return;
|
|
395
|
-
const
|
|
396
|
-
|
|
397
|
-
throw new Error('Memory monitor interval must be positive');
|
|
398
|
-
this.#monitor = setInterval(() => {
|
|
399
|
-
if (this.#monitoring)
|
|
507
|
+
const timer = setTimeout(() => {
|
|
508
|
+
if (context.recoveryTimer !== timer)
|
|
400
509
|
return;
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
this.#
|
|
510
|
+
context.recoveryTimer = undefined;
|
|
511
|
+
this.#startRecovery(context);
|
|
512
|
+
}, Math.max(0, context.recoveryRetryAt - this.#now()));
|
|
513
|
+
context.recoveryTimer = timer;
|
|
514
|
+
timer.unref?.();
|
|
404
515
|
}
|
|
405
|
-
|
|
406
|
-
if (
|
|
516
|
+
#scheduleWriterRetry(context) {
|
|
517
|
+
if (context.writerTimer || !this.#driver(context))
|
|
407
518
|
return;
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
}
|
|
418
|
-
context.writerBlocked = false;
|
|
419
|
-
if (context.initialized)
|
|
420
|
-
await this.#trackScheduleIfPending(context);
|
|
421
|
-
const signature = JSON.stringify([this.#enabled, context.state, context.message, control.updatedAt]);
|
|
422
|
-
if (signature !== context.statusSignature) {
|
|
423
|
-
context.statusSignature = signature;
|
|
424
|
-
this.#emitStatusChange();
|
|
425
|
-
}
|
|
426
|
-
}
|
|
427
|
-
catch (error) {
|
|
428
|
-
context.state = 'error';
|
|
429
|
-
context.message = sanitizeMemoryDiagnostic(error);
|
|
430
|
-
context.abort?.abort(error);
|
|
431
|
-
this.#emitStatusChange();
|
|
519
|
+
context.writerBlocked = true;
|
|
520
|
+
const timer = setTimeout(() => {
|
|
521
|
+
if (context.writerTimer !== timer)
|
|
522
|
+
return;
|
|
523
|
+
context.writerTimer = undefined;
|
|
524
|
+
context.writerBlocked = false;
|
|
525
|
+
if (context.run) {
|
|
526
|
+
void this.#trackScheduleIfPending(context);
|
|
527
|
+
return;
|
|
432
528
|
}
|
|
433
|
-
|
|
529
|
+
context.diagnosticsReconciled = false;
|
|
530
|
+
const driver = this.#driver(context);
|
|
531
|
+
if (driver)
|
|
532
|
+
void this.#initialize(context, driver).catch(() => { });
|
|
533
|
+
}, this.#options.writerRetryMs ?? 30_000);
|
|
534
|
+
context.writerTimer = timer;
|
|
535
|
+
timer.unref?.();
|
|
434
536
|
}
|
|
537
|
+
#now() { return this.#options.now?.() ?? Date.now(); }
|
|
435
538
|
#cancelContext(context, reason, cancelPersisted) {
|
|
436
539
|
const alreadyAborted = context.abort?.signal.aborted ?? false;
|
|
437
540
|
context.abort?.abort(new Error(reason));
|
|
438
541
|
if (alreadyAborted || !context.initialized || (!cancelPersisted && !context.attempt))
|
|
439
542
|
return;
|
|
543
|
+
const environment = context.runEnvironment ?? this.#liveMember(context)?.environment;
|
|
544
|
+
if (!environment)
|
|
545
|
+
return;
|
|
546
|
+
const store = this.#store(context, environment);
|
|
440
547
|
const cancellation = cancelPersisted
|
|
441
|
-
?
|
|
442
|
-
:
|
|
548
|
+
? store.cancelActiveAttempt(this.#now())
|
|
549
|
+
: store.finishAttempt(undefined, context.attempt, { status: 'cancelled', reason }, this.#now());
|
|
443
550
|
const followUp = cancellation
|
|
444
|
-
.then(() => { })
|
|
551
|
+
.then(() => { this.#emitStatusChange(); })
|
|
445
552
|
.catch(() => { })
|
|
446
553
|
.finally(() => { context.followUps.delete(followUp); });
|
|
447
554
|
context.followUps.add(followUp);
|
|
448
555
|
}
|
|
449
556
|
async #schedule(context, dueAt) {
|
|
450
|
-
if (
|
|
557
|
+
if (!this.#driver(context) || context.run || context.storageBlocked)
|
|
451
558
|
return;
|
|
452
559
|
if (context.timer && context.timerDueAt === dueAt)
|
|
453
560
|
return;
|
|
@@ -459,28 +566,26 @@ export class LocalMemoryCoordinator {
|
|
|
459
566
|
return;
|
|
460
567
|
context.timer = undefined;
|
|
461
568
|
context.timerDueAt = undefined;
|
|
462
|
-
|
|
463
|
-
context.state = 'error';
|
|
464
|
-
context.storageBlocked = true;
|
|
465
|
-
context.message = sanitizeMemoryDiagnostic(error);
|
|
466
|
-
this.#emitStatusChange();
|
|
467
|
-
}).finally(async () => {
|
|
468
|
-
context.run = undefined;
|
|
469
|
-
await this.#trackScheduleIfPending(context);
|
|
470
|
-
});
|
|
471
|
-
context.run = run;
|
|
569
|
+
void this.#launchRun(context)?.catch(() => { });
|
|
472
570
|
}, delay);
|
|
473
571
|
context.timer = timer;
|
|
474
572
|
context.timerDueAt = dueAt;
|
|
475
573
|
timer.unref?.();
|
|
476
574
|
}
|
|
477
|
-
async #scheduleIfPending(context) {
|
|
478
|
-
|
|
575
|
+
async #scheduleIfPending(context, environment) {
|
|
576
|
+
const driver = this.#driver(context);
|
|
577
|
+
if (!driver) {
|
|
578
|
+
this.#cancelTimers(context);
|
|
479
579
|
return;
|
|
580
|
+
}
|
|
581
|
+
// Callers capture their environment before awaiting; a member sealed since then cannot read the store.
|
|
582
|
+
const owner = environment && [...context.attachments].find(attachment => attachment.environment === environment);
|
|
583
|
+
const source = owner && (owner.closing || owner.detached) ? (this.#liveMember(context) ?? owner).environment : environment;
|
|
480
584
|
try {
|
|
481
|
-
const
|
|
585
|
+
const store = this.#store(context, source ?? driver.environment);
|
|
586
|
+
const control = await store.readControl();
|
|
482
587
|
if (control.autoDisabled) {
|
|
483
|
-
this.#
|
|
588
|
+
this.#cancelTimers(context);
|
|
484
589
|
context.state = 'disabled';
|
|
485
590
|
return;
|
|
486
591
|
}
|
|
@@ -488,13 +593,21 @@ export class LocalMemoryCoordinator {
|
|
|
488
593
|
return;
|
|
489
594
|
if (context.writerBlocked)
|
|
490
595
|
return;
|
|
491
|
-
const state = await
|
|
596
|
+
const state = await store.status();
|
|
597
|
+
if (context.state === 'error' && context.message === 'Local memory storage is unavailable') {
|
|
598
|
+
context.state = 'idle';
|
|
599
|
+
context.message = undefined;
|
|
600
|
+
}
|
|
492
601
|
const pending = Object.values(state.pending);
|
|
602
|
+
if (!this.#driver(context))
|
|
603
|
+
return;
|
|
493
604
|
if (control.activeAttempt) {
|
|
494
605
|
this.#cancelScheduledRun(context);
|
|
606
|
+
if (!context.run)
|
|
607
|
+
this.#scheduleWriterRetry(context);
|
|
495
608
|
}
|
|
496
609
|
else if (pending.some(({ checkpoint }) => !isBlockedCheckpoint(context, checkpoint))) {
|
|
497
|
-
const acceptedUpdates = await
|
|
610
|
+
const acceptedUpdates = await store.pendingAcceptedUpdates((checkpoint) => !isBlockedCheckpoint(context, checkpoint));
|
|
498
611
|
if (acceptedUpdates < MEMORY_AUTOMATIC_UPDATE_THRESHOLD) {
|
|
499
612
|
this.#cancelScheduledRun(context);
|
|
500
613
|
if (context.state === 'scheduled')
|
|
@@ -529,37 +642,84 @@ export class LocalMemoryCoordinator {
|
|
|
529
642
|
context.message = 'Local memory storage is unavailable';
|
|
530
643
|
}
|
|
531
644
|
}
|
|
532
|
-
#trackScheduleIfPending(context) {
|
|
533
|
-
const followUp = this.#scheduleIfPending(context).finally(() => {
|
|
645
|
+
#trackScheduleIfPending(context, environment) {
|
|
646
|
+
const followUp = this.#scheduleIfPending(context, environment).finally(() => {
|
|
534
647
|
context.followUps.delete(followUp);
|
|
535
648
|
});
|
|
536
649
|
context.followUps.add(followUp);
|
|
537
650
|
return followUp;
|
|
538
651
|
}
|
|
539
|
-
|
|
652
|
+
#launchRun(context, bypassRetryAt = false, preferred) {
|
|
653
|
+
if (context.run || context.storageBlocked || preferred && !this.#eligible(preferred))
|
|
654
|
+
return;
|
|
655
|
+
const driver = this.#driver(context, preferred);
|
|
656
|
+
if (!driver)
|
|
657
|
+
return;
|
|
658
|
+
context.driver = driver;
|
|
659
|
+
context.failedDriver = undefined;
|
|
660
|
+
const environment = this.#runEnvironment(context, driver);
|
|
661
|
+
context.runEnvironment = environment;
|
|
662
|
+
const run = this.#run(context, driver, environment, bypassRetryAt).catch((error) => {
|
|
663
|
+
context.state = 'error';
|
|
664
|
+
context.failedDriver = driver;
|
|
665
|
+
if (!bypassRetryAt && !driver.closing && !driver.detached)
|
|
666
|
+
context.storageBlocked = true;
|
|
667
|
+
context.message = sanitizeMemoryDiagnostic(error);
|
|
668
|
+
throw error;
|
|
669
|
+
}).finally(async () => {
|
|
670
|
+
if (context.run === settled) {
|
|
671
|
+
context.run = undefined;
|
|
672
|
+
context.driver = undefined;
|
|
673
|
+
context.runEnvironment = undefined;
|
|
674
|
+
context.abort = undefined;
|
|
675
|
+
}
|
|
676
|
+
await this.#trackScheduleIfPending(context);
|
|
677
|
+
this.#emitStatusChange();
|
|
678
|
+
});
|
|
679
|
+
const settled = run.catch(() => { });
|
|
680
|
+
context.run = settled;
|
|
681
|
+
return run;
|
|
682
|
+
}
|
|
683
|
+
async #run(context, driver, environment, bypassRetryAt) {
|
|
540
684
|
if (this.#disposed || this.#draining || !this.#enabled || context.storageBlocked)
|
|
541
685
|
return;
|
|
542
686
|
context.state = 'processing';
|
|
543
687
|
context.message = undefined;
|
|
688
|
+
const abort = new AbortController();
|
|
689
|
+
context.abort = abort;
|
|
690
|
+
const store = this.#store(context, environment);
|
|
544
691
|
let lease;
|
|
692
|
+
const cancelled = () => driver.closing || driver.detached || !this.#enabled || this.#disposed
|
|
693
|
+
|| abort.signal.aborted && !lease?.compromised.aborted;
|
|
545
694
|
try {
|
|
546
|
-
lease = await acquireLocalMemoryLease(
|
|
695
|
+
lease = await acquireLocalMemoryLease(environment, store.projectDirectory, this.#options.leaseOptions);
|
|
547
696
|
}
|
|
548
697
|
catch (error) {
|
|
549
|
-
|
|
698
|
+
if (cancelled()) {
|
|
699
|
+
context.state = 'idle';
|
|
700
|
+
context.message = driver.closing || driver.detached || this.#disposed
|
|
701
|
+
? 'Memory processing was cancelled during shutdown' : 'Memory processing was cancelled';
|
|
702
|
+
return;
|
|
703
|
+
}
|
|
704
|
+
this.#scheduleWriterRetry(context);
|
|
550
705
|
context.state = 'error';
|
|
706
|
+
context.failedDriver = driver;
|
|
551
707
|
context.message = safeProcessingMessage(error);
|
|
552
708
|
return;
|
|
553
709
|
}
|
|
554
710
|
if (!lease) {
|
|
555
|
-
context
|
|
711
|
+
this.#scheduleWriterRetry(context);
|
|
556
712
|
context.state = 'blocked';
|
|
713
|
+
context.failedDriver = driver;
|
|
557
714
|
context.message = 'Another Felan Code process owns the memory writer';
|
|
558
715
|
return;
|
|
559
716
|
}
|
|
560
717
|
context.writerBlocked = false;
|
|
561
|
-
|
|
562
|
-
|
|
718
|
+
if (context.failedDriver === driver)
|
|
719
|
+
context.failedDriver = undefined;
|
|
720
|
+
if (context.writerTimer)
|
|
721
|
+
clearTimeout(context.writerTimer);
|
|
722
|
+
context.writerTimer = undefined;
|
|
563
723
|
const abortOnLeaseCompromise = () => {
|
|
564
724
|
abort.abort(new Error('Memory writer lease was lost'));
|
|
565
725
|
};
|
|
@@ -569,7 +729,9 @@ export class LocalMemoryCoordinator {
|
|
|
569
729
|
let published = false;
|
|
570
730
|
let attempt;
|
|
571
731
|
try {
|
|
572
|
-
|
|
732
|
+
if (abort.signal.aborted || !this.#enabled || this.#disposed)
|
|
733
|
+
return;
|
|
734
|
+
const control = await this.#reconcileRunDiagnostics(context, environment, lease);
|
|
573
735
|
if (control.autoDisabled) {
|
|
574
736
|
context.state = 'disabled';
|
|
575
737
|
return;
|
|
@@ -578,24 +740,25 @@ export class LocalMemoryCoordinator {
|
|
|
578
740
|
context.state = 'scheduled';
|
|
579
741
|
return;
|
|
580
742
|
}
|
|
581
|
-
|
|
743
|
+
// Every pending checkpoint is a candidate so unreadable ones cannot occupy the batch.
|
|
744
|
+
const snapshot = await store.processingSnapshot(Number.POSITIVE_INFINITY, (checkpoint) => !isBlockedCheckpoint(context, checkpoint));
|
|
582
745
|
if (snapshot.checkpoints.length === 0) {
|
|
583
746
|
context.state = 'idle';
|
|
584
747
|
return;
|
|
585
748
|
}
|
|
586
|
-
retainedRun = await LocalMemoryRun.create(
|
|
587
|
-
projectDirectory:
|
|
749
|
+
retainedRun = await LocalMemoryRun.create(environment, {
|
|
750
|
+
projectDirectory: store.projectDirectory,
|
|
588
751
|
sessionDirectory: join(this.#options.sessionDir ?? join(this.#options.agentDir, 'sessions'), 'memory'),
|
|
589
752
|
projectKey: context.project.key,
|
|
590
753
|
projectRoot: context.project.canonicalRoot,
|
|
591
|
-
checkpoints: snapshot.checkpoints.map(({ checkpoint }) => checkpoint),
|
|
754
|
+
checkpoints: snapshot.checkpoints.slice(0, this.#batchSize).map(({ checkpoint }) => checkpoint),
|
|
592
755
|
baseFingerprint: snapshot.fingerprint,
|
|
593
756
|
...(control.lastDisabledRunId ? { protectedRunIds: [control.lastDisabledRunId] } : {}),
|
|
594
757
|
});
|
|
595
758
|
staging = retainedRun.workspace;
|
|
596
759
|
const memoryDirectory = join(staging, '.memory');
|
|
597
|
-
await hydrateMemoryDirectory(
|
|
598
|
-
const manifest = await materializeMemoryInput(
|
|
760
|
+
await hydrateMemoryDirectory(environment, snapshot.artifact, memoryDirectory, { memoryPath: '.memory', mode: 'read' });
|
|
761
|
+
const manifest = await materializeMemoryInput(environment, {
|
|
599
762
|
stagingDirectory: staging,
|
|
600
763
|
checkpoints: snapshot.checkpoints.map(({ checkpoint }) => checkpoint),
|
|
601
764
|
previousCheckpoints: Object.fromEntries(Object.entries(snapshot.state.processed).map(([sessionId, entry]) => [sessionId, entry.checkpoint])),
|
|
@@ -604,26 +767,29 @@ export class LocalMemoryCoordinator {
|
|
|
604
767
|
...(this.#options.maxTranscriptBytes === undefined
|
|
605
768
|
? {}
|
|
606
769
|
: { maxTranscriptBytes: this.#options.maxTranscriptBytes }),
|
|
770
|
+
maxSessions: this.#batchSize,
|
|
607
771
|
signal: abort.signal,
|
|
608
772
|
});
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
773
|
+
// A cancelled read reports as a failure; it must not drop evidence.
|
|
774
|
+
if (abort.signal.aborted || !this.#enabled || this.#disposed)
|
|
775
|
+
throw new Error('Memory processing was cancelled');
|
|
776
|
+
// A reachable Sessions service shows the failures belong to the checkpoints, not to the service.
|
|
777
|
+
if (manifest.failures.length > 0)
|
|
778
|
+
await environment.sessions.list({ scope: 'project' }, { limit: 1, signal: abort.signal });
|
|
779
|
+
const dropped = manifest.failures.length === 0 ? 0 : await store.dropCheckpoints(lease, manifest.failures.map(({ checkpoint, code }) => ({ checkpoint, reason: code })), this.#now());
|
|
780
|
+
const droppedMessage = dropped === 0 ? undefined : droppedCheckpointsMessage(dropped);
|
|
615
781
|
const processedCheckpoints = snapshot.checkpoints.filter(({ checkpoint }) => manifest.sessions.some((session) => session.checkpoint.sessionId === checkpoint.sessionId
|
|
616
782
|
&& sameCheckpointCursor(session.checkpoint, checkpoint)));
|
|
617
783
|
if (processedCheckpoints.length === 0) {
|
|
618
|
-
context.state = '
|
|
619
|
-
context.message =
|
|
620
|
-
await retainedRun.finish('blocked',
|
|
784
|
+
context.state = 'idle';
|
|
785
|
+
context.message = droppedMessage;
|
|
786
|
+
await retainedRun.finish('blocked', droppedMessage ?? 'Memory checkpoints changed before they could be read');
|
|
621
787
|
return;
|
|
622
788
|
}
|
|
623
789
|
await retainedRun.record({ phase: 'model', checkpoints: processedCheckpoints.map(({ checkpoint }) => checkpoint) });
|
|
624
790
|
if (abort.signal.aborted || !this.#enabled || this.#disposed)
|
|
625
791
|
throw new Error('Memory processing was cancelled');
|
|
626
|
-
attempt = await
|
|
792
|
+
attempt = await store.beginAttempt(lease, {
|
|
627
793
|
runId: retainedRun.metadata.sessionId,
|
|
628
794
|
baseFingerprint: snapshot.fingerprint,
|
|
629
795
|
checkpoints: processedCheckpoints,
|
|
@@ -635,16 +801,16 @@ export class LocalMemoryCoordinator {
|
|
|
635
801
|
}
|
|
636
802
|
context.attempt = attempt;
|
|
637
803
|
this.#emitStatusChange();
|
|
638
|
-
const result = await this.#dreamRunner({
|
|
804
|
+
const result = await (this.#options.dreamRunner ?? createDefaultLocalMemoryDreamRunner(environment))({
|
|
639
805
|
stagingDirectory: staging,
|
|
640
806
|
memoryDirectory,
|
|
641
807
|
inputDirectory: join(staging, '.dreaming', 'input'),
|
|
642
808
|
baseSnapshot: createMemorySnapshot(snapshot.artifact, '.memory', { mode: 'read' }),
|
|
643
809
|
manifest,
|
|
644
|
-
models:
|
|
645
|
-
...(
|
|
646
|
-
...(
|
|
647
|
-
...(
|
|
810
|
+
models: driver.models,
|
|
811
|
+
...(driver.selectedModel === undefined ? {} : { selectedModel: driver.selectedModel }),
|
|
812
|
+
...(driver.scopedModels === undefined ? {} : { scopedModels: driver.scopedModels }),
|
|
813
|
+
...(driver.classifier === undefined ? {} : { classifier: driver.classifier }),
|
|
648
814
|
sessionDirectory: join(this.#options.sessionDir ?? join(this.#options.agentDir, 'sessions'), 'memory'),
|
|
649
815
|
run: retainedRun,
|
|
650
816
|
signal: abort.signal,
|
|
@@ -652,9 +818,9 @@ export class LocalMemoryCoordinator {
|
|
|
652
818
|
if (abort.signal.aborted || !this.#enabled || this.#disposed)
|
|
653
819
|
throw new Error('Memory processing was cancelled');
|
|
654
820
|
if (result)
|
|
655
|
-
await hydrateMemoryDirectory(
|
|
821
|
+
await hydrateMemoryDirectory(environment, result, memoryDirectory, { replace: true, memoryPath: '.memory' });
|
|
656
822
|
await retainedRun.record({ phase: 'validate' });
|
|
657
|
-
const artifact = await readMemoryDirectory(
|
|
823
|
+
const artifact = await readMemoryDirectory(environment, memoryDirectory, {
|
|
658
824
|
memoryPath: '.memory',
|
|
659
825
|
sourceSessionIds: allowedSourceIds(snapshot, manifest),
|
|
660
826
|
requireSources: true,
|
|
@@ -663,42 +829,45 @@ export class LocalMemoryCoordinator {
|
|
|
663
829
|
phase: 'publish',
|
|
664
830
|
outputFingerprint: createMemorySnapshot(artifact, '.memory').fingerprint,
|
|
665
831
|
});
|
|
666
|
-
await
|
|
832
|
+
await store.commit(lease, snapshot.fingerprint, artifact, processedCheckpoints, attempt);
|
|
667
833
|
published = true;
|
|
668
|
-
await
|
|
834
|
+
await store.finishAttempt(lease, attempt, { status: 'success' }, this.#now());
|
|
669
835
|
await retainedRun.finish('completed');
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
}
|
|
674
|
-
else {
|
|
675
|
-
context.state = 'idle';
|
|
676
|
-
}
|
|
677
|
-
this.#emitStatusChange();
|
|
836
|
+
context.state = 'idle';
|
|
837
|
+
context.failedDriver = undefined;
|
|
838
|
+
context.message = droppedMessage;
|
|
678
839
|
}
|
|
679
840
|
catch (error) {
|
|
680
|
-
const leaseValid = await lease.verify();
|
|
681
|
-
const explicitlyCancelled =
|
|
682
|
-
|| (abort.signal.aborted && !lease.compromised.aborted);
|
|
841
|
+
const leaseValid = await lease.verify().catch(() => false);
|
|
842
|
+
const explicitlyCancelled = cancelled();
|
|
683
843
|
const unavailableModel = error instanceof MemoryModelUnavailableError;
|
|
684
844
|
const blockedBeforeInference = retainedRun !== undefined && attempt === undefined && !explicitlyCancelled;
|
|
685
845
|
if (attempt && (leaseValid || explicitlyCancelled)) {
|
|
686
846
|
try {
|
|
687
|
-
const control = await
|
|
847
|
+
const control = await store.finishAttempt(explicitlyCancelled || unavailableModel ? undefined : lease, attempt, explicitlyCancelled || unavailableModel ? { status: 'cancelled' }
|
|
688
848
|
: { status: 'failure', reason: sanitizeMemoryDiagnostic(error) }, this.#now());
|
|
689
849
|
published ||= control.lastOutcome?.runId === attempt.runId && control.lastOutcome.status === 'success';
|
|
690
850
|
}
|
|
691
851
|
catch {
|
|
692
|
-
|
|
852
|
+
if (!explicitlyCancelled)
|
|
853
|
+
context.storageBlocked = true;
|
|
693
854
|
}
|
|
694
855
|
}
|
|
856
|
+
if (explicitlyCancelled) {
|
|
857
|
+
context.state = 'idle';
|
|
858
|
+
context.message = driver.closing || driver.detached || this.#disposed
|
|
859
|
+
? 'Memory processing was cancelled during shutdown' : 'Memory processing was cancelled';
|
|
860
|
+
await retainedRun?.finish('cancelled', error).catch(() => { });
|
|
861
|
+
return;
|
|
862
|
+
}
|
|
695
863
|
if (published) {
|
|
696
864
|
context.state = 'idle';
|
|
865
|
+
context.failedDriver = context.storageBlocked ? driver : undefined;
|
|
697
866
|
context.message = 'Memory was published, but its diagnostics could not be fully saved';
|
|
698
|
-
this.#emitStatusChange();
|
|
699
867
|
return;
|
|
700
868
|
}
|
|
701
869
|
context.state = 'error';
|
|
870
|
+
context.failedDriver = driver;
|
|
702
871
|
context.message = !leaseValid
|
|
703
872
|
? safeProcessingMessage(new Error('Memory writer lease was lost'))
|
|
704
873
|
: safeProcessingMessage(error);
|
|
@@ -712,62 +881,68 @@ export class LocalMemoryCoordinator {
|
|
|
712
881
|
context.storageBlocked = true;
|
|
713
882
|
context.message = `Memory processing is paused: ${sanitizeMemoryDiagnostic(error)}`;
|
|
714
883
|
}
|
|
715
|
-
await retainedRun?.finish(
|
|
716
|
-
:
|
|
717
|
-
: !leaseValid ? 'interrupted' : 'failed', error).catch(() => {
|
|
884
|
+
await retainedRun?.finish(unavailableModel || blockedBeforeInference ? 'blocked'
|
|
885
|
+
: !leaseValid ? 'interrupted' : 'failed', error).catch(() => {
|
|
718
886
|
context.storageBlocked = true;
|
|
719
887
|
context.message = 'Memory diagnostics could not be saved; evidence remains pending';
|
|
720
888
|
});
|
|
721
|
-
this.#emitStatusChange();
|
|
722
889
|
}
|
|
723
890
|
finally {
|
|
724
891
|
lease.compromised.removeEventListener('abort', abortOnLeaseCompromise);
|
|
725
|
-
context.abort = undefined;
|
|
726
892
|
context.attempt = undefined;
|
|
727
|
-
await lease.release()
|
|
893
|
+
await lease.release().catch(error => {
|
|
894
|
+
if (!cancelled())
|
|
895
|
+
throw error;
|
|
896
|
+
});
|
|
728
897
|
}
|
|
729
898
|
}
|
|
730
|
-
async #reconcileRunDiagnostics(context, lease) {
|
|
731
|
-
|
|
732
|
-
|
|
899
|
+
async #reconcileRunDiagnostics(context, environment, lease) {
|
|
900
|
+
const store = this.#store(context, environment);
|
|
901
|
+
const previous = await store.readControl();
|
|
902
|
+
const control = await store.reconcileAbandonedAttempt(lease, this.#now());
|
|
903
|
+
if (previous.updatedAt !== control.updatedAt)
|
|
904
|
+
this.#emitStatusChange();
|
|
733
905
|
const options = {
|
|
734
|
-
projectDirectory:
|
|
906
|
+
projectDirectory: store.projectDirectory,
|
|
735
907
|
projectKey: context.project.key,
|
|
736
908
|
sessionDirectory: join(this.#options.sessionDir ?? join(this.#options.agentDir, 'sessions'), 'memory'),
|
|
737
909
|
...(control.lastDisabledRunId ? { protectedRunIds: [control.lastDisabledRunId] } : {}),
|
|
738
910
|
};
|
|
739
911
|
if (control.lastOutcome)
|
|
740
|
-
await LocalMemoryRun.reconcile(
|
|
741
|
-
await LocalMemoryRun.reconcileAbandoned(
|
|
912
|
+
await LocalMemoryRun.reconcile(environment, options, control.lastOutcome);
|
|
913
|
+
await LocalMemoryRun.reconcileAbandoned(environment, options, [
|
|
742
914
|
...(control.activeAttempt ? [control.activeAttempt.runId] : []),
|
|
743
915
|
...(control.lastOutcome ? [control.lastOutcome.runId] : []),
|
|
744
916
|
], this.#now());
|
|
745
917
|
return control;
|
|
746
918
|
}
|
|
747
|
-
async #recover(context, signal) {
|
|
919
|
+
async #recover(context, driver, signal) {
|
|
748
920
|
let cursor;
|
|
749
921
|
let recovered = false;
|
|
750
922
|
const stableAfter = this.#options.recoveryStableMs ?? 30_000;
|
|
923
|
+
const store = this.#store(context, driver.environment);
|
|
751
924
|
const stopped = () => this.#disposed || this.#draining || !this.#enabled || signal.aborted;
|
|
752
925
|
try {
|
|
753
926
|
do {
|
|
754
|
-
const page = await
|
|
927
|
+
const page = await driver.environment.sessions.list({ scope: 'project' }, { ...(cursor ? { cursor } : {}), signal });
|
|
755
928
|
for (const info of page.items) {
|
|
756
929
|
if (stopped())
|
|
757
930
|
return false;
|
|
758
931
|
if (info.parent || this.#now() - info.updatedAt < stableAfter)
|
|
759
932
|
continue;
|
|
933
|
+
let checkpoint;
|
|
760
934
|
try {
|
|
761
|
-
|
|
762
|
-
if (stopped())
|
|
763
|
-
return false;
|
|
764
|
-
recovered = (await context.store.recordCheckpoint(checkpoint)) || recovered;
|
|
935
|
+
checkpoint = await readMemorySessionCheckpoint(driver.environment.sessions, info.id, signal);
|
|
765
936
|
}
|
|
766
937
|
catch {
|
|
767
938
|
if (stopped())
|
|
768
939
|
return false;
|
|
769
940
|
// A session that cannot be read now is skipped; its live checkpoint or a later run records it.
|
|
941
|
+
continue;
|
|
770
942
|
}
|
|
943
|
+
if (stopped())
|
|
944
|
+
return false;
|
|
945
|
+
recovered = (await store.recordCheckpoint(checkpoint)) || recovered;
|
|
771
946
|
}
|
|
772
947
|
cursor = page.nextCursor;
|
|
773
948
|
} while (cursor);
|
|
@@ -789,13 +964,23 @@ export class LocalMemoryCoordinator {
|
|
|
789
964
|
return true;
|
|
790
965
|
}
|
|
791
966
|
#emitStatusChange() {
|
|
792
|
-
for (const listener of [...this.#statusListeners]) {
|
|
967
|
+
for (const listener of [...this.#statusListeners, ...[...this.#attachments].flatMap(attachment => [...attachment.listeners])]) {
|
|
793
968
|
try {
|
|
794
969
|
listener();
|
|
795
970
|
}
|
|
796
971
|
catch { }
|
|
797
972
|
}
|
|
798
973
|
}
|
|
974
|
+
#cancelTimers(context) {
|
|
975
|
+
this.#cancelScheduledRun(context);
|
|
976
|
+
if (context.recoveryTimer)
|
|
977
|
+
clearTimeout(context.recoveryTimer);
|
|
978
|
+
if (context.writerTimer)
|
|
979
|
+
clearTimeout(context.writerTimer);
|
|
980
|
+
context.recoveryTimer = undefined;
|
|
981
|
+
context.writerTimer = undefined;
|
|
982
|
+
context.writerBlocked = false;
|
|
983
|
+
}
|
|
799
984
|
#cancelScheduledRun(context) {
|
|
800
985
|
if (context.timer)
|
|
801
986
|
clearTimeout(context.timer);
|
|
@@ -843,6 +1028,9 @@ function allowedSourceIds(snapshot, manifest) {
|
|
|
843
1028
|
...manifest.sessions.map(({ checkpoint }) => checkpoint.sessionId),
|
|
844
1029
|
])];
|
|
845
1030
|
}
|
|
1031
|
+
function droppedCheckpointsMessage(count) {
|
|
1032
|
+
return `Dropped ${count} memory checkpoint${count === 1 ? '' : 's'} that could not be read`;
|
|
1033
|
+
}
|
|
846
1034
|
function safeProcessingMessage(error) {
|
|
847
1035
|
if (error instanceof MemoryModelUnavailableError) {
|
|
848
1036
|
return 'Low-tier memory model is unavailable; evidence remains pending';
|