@felan-ai/ext-memory 0.10.1 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -10
- package/dist/contracts.d.ts +8 -0
- package/dist/contracts.d.ts.map +1 -1
- package/dist/contracts.js +2 -0
- package/dist/contracts.js.map +1 -1
- package/dist/daemon-provider.d.ts +49 -0
- package/dist/daemon-provider.d.ts.map +1 -0
- package/dist/daemon-provider.js +231 -0
- package/dist/daemon-provider.js.map +1 -0
- package/dist/extension.d.ts.map +1 -1
- package/dist/extension.js +30 -9
- package/dist/extension.js.map +1 -1
- package/dist/local/context.d.ts +2 -2
- package/dist/local/control.d.ts +3 -2
- package/dist/local/control.d.ts.map +1 -1
- package/dist/local/control.js.map +1 -1
- package/dist/local/coordinator.d.ts +30 -51
- package/dist/local/coordinator.d.ts.map +1 -1
- package/dist/local/coordinator.js +495 -258
- package/dist/local/coordinator.js.map +1 -1
- package/dist/local/dreamer.d.ts +3 -1
- package/dist/local/dreamer.d.ts.map +1 -1
- package/dist/local/dreamer.js +3 -3
- package/dist/local/dreamer.js.map +1 -1
- package/dist/local/lease.d.ts.map +1 -1
- package/dist/local/lease.js +12 -7
- package/dist/local/lease.js.map +1 -1
- package/dist/local/project.d.ts +1 -0
- package/dist/local/project.d.ts.map +1 -1
- package/dist/local/project.js +124 -2
- package/dist/local/project.js.map +1 -1
- package/dist/local/shared.d.ts +7 -0
- package/dist/local/shared.d.ts.map +1 -0
- package/dist/local/shared.js +30 -0
- package/dist/local/shared.js.map +1 -0
- package/dist/local/store.d.ts +12 -0
- package/dist/local/store.d.ts.map +1 -1
- package/dist/local/store.js +49 -3
- package/dist/local/store.js.map +1 -1
- package/package.json +7 -3
|
@@ -11,48 +11,91 @@ import {} from '@felan-ai/agent-core';
|
|
|
11
11
|
import { MemoryModelUnavailableError } from './dreamer.js';
|
|
12
12
|
import { LocalMemoryControlError } from './store.js';
|
|
13
13
|
import { sanitizeMemoryDiagnostic } from '../diagnostics.js';
|
|
14
|
-
const MEMORY_INPUT_BLOCKED_MESSAGE = '
|
|
14
|
+
const MEMORY_INPUT_BLOCKED_MESSAGE = 'Memory checkpoints are paused after a failed run; evidence remains pending';
|
|
15
|
+
const MEMORY_PROJECT_UNAVAILABLE_MESSAGE = 'Memory project root is unavailable; evidence remains pending';
|
|
15
16
|
const MEMORY_AUTOMATIC_UPDATE_THRESHOLD = 5;
|
|
16
17
|
const MEMORY_AUTOMATIC_INTERVAL_MS = 24 * 60 * 60 * 1_000;
|
|
17
18
|
const MEMORY_DREAM_INPUT_MAX_BYTES = 10 * 1024 * 1024;
|
|
18
19
|
const MEMORY_MAX_BATCH_SESSIONS = 8;
|
|
19
20
|
export class LocalMemoryCoordinator {
|
|
20
|
-
environment;
|
|
21
21
|
#options;
|
|
22
22
|
#contexts = new Map();
|
|
23
|
-
#
|
|
23
|
+
#attachments = new Set();
|
|
24
24
|
#statusListeners = new Set();
|
|
25
|
-
#dreamRunner;
|
|
26
25
|
#batchSize;
|
|
27
26
|
#enabled;
|
|
28
|
-
#selectedModel;
|
|
29
|
-
#scopedModels;
|
|
30
|
-
#classifier;
|
|
31
27
|
#disposed = false;
|
|
28
|
+
#disposal;
|
|
32
29
|
#draining = false;
|
|
33
|
-
#monitor;
|
|
34
|
-
#monitoring;
|
|
35
30
|
#enableVersion = 0;
|
|
36
|
-
constructor(
|
|
37
|
-
this.environment = environment;
|
|
31
|
+
constructor(options) {
|
|
38
32
|
this.#options = options;
|
|
39
33
|
this.#batchSize = boundedMemoryBatchSize(options.batchSize);
|
|
40
34
|
this.#enabled = options.enabled !== false;
|
|
41
|
-
this.#selectedModel = options.selectedModel;
|
|
42
|
-
this.#scopedModels = options.scopedModels === undefined ? undefined : [...options.scopedModels];
|
|
43
|
-
this.#classifier = options.classifier;
|
|
44
|
-
this.#dreamRunner = options.dreamRunner ?? createDefaultLocalMemoryDreamRunner(this.environment);
|
|
45
|
-
if (this.#enabled)
|
|
46
|
-
this.#startMonitor();
|
|
47
35
|
}
|
|
48
|
-
|
|
49
|
-
|
|
36
|
+
attach(options) {
|
|
37
|
+
if (this.#disposed)
|
|
38
|
+
throw new Error('Local memory coordinator is closed');
|
|
39
|
+
const state = {
|
|
40
|
+
...options,
|
|
41
|
+
selectedModel: options.selectedModel,
|
|
42
|
+
scopedModels: options.scopedModels === undefined ? undefined : [...options.scopedModels],
|
|
43
|
+
classifier: options.classifier,
|
|
44
|
+
unavailableReason: options.unavailableReason,
|
|
45
|
+
projections: new Map(),
|
|
46
|
+
listeners: new Set(),
|
|
47
|
+
draining: false,
|
|
48
|
+
closing: false,
|
|
49
|
+
detached: false,
|
|
50
|
+
detaching: undefined,
|
|
51
|
+
};
|
|
52
|
+
this.#attachments.add(state);
|
|
53
|
+
const attachment = {
|
|
54
|
+
createSessionHost: options => new LocalMemorySessionHost(attachment, options),
|
|
55
|
+
readCurrent: (cwd, root) => this.#readCurrent(state, cwd, root),
|
|
56
|
+
recordCheckpoint: (cwd, checkpoint) => this.#recordCheckpoint(state, cwd, checkpoint),
|
|
57
|
+
status: cwd => this.#status(state, cwd),
|
|
58
|
+
runNow: cwd => this.#runNow(state, cwd),
|
|
59
|
+
retryProject: cwd => this.#retryProject(state, cwd),
|
|
60
|
+
canonicalDirectory: async (cwd) => this.#store(await this.#context(state, cwd), state.environment).currentDirectory,
|
|
61
|
+
setSelectedModel: model => this.#setModelSelection(state, model, state.scopedModels, state.classifier),
|
|
62
|
+
setModelSelection: (model, scopedModels, classifier) => this.#setModelSelection(state, model, scopedModels, classifier),
|
|
63
|
+
setScheduling: (scheduling, unavailableReason) => {
|
|
64
|
+
if (state.detached || this.#disposed)
|
|
65
|
+
return;
|
|
66
|
+
state.scheduling = scheduling;
|
|
67
|
+
state.unavailableReason = unavailableReason;
|
|
68
|
+
for (const context of this.#contexts.values()) {
|
|
69
|
+
if (!context.attachments.has(state))
|
|
70
|
+
continue;
|
|
71
|
+
if (!this.#driver(context))
|
|
72
|
+
this.#cancelTimers(context);
|
|
73
|
+
if (scheduling && context.initialized) {
|
|
74
|
+
this.#startRecovery(context);
|
|
75
|
+
void this.#trackScheduleIfPending(context, state.environment);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
},
|
|
79
|
+
subscribeStatusChanges: listener => {
|
|
80
|
+
if (state.detached || this.#disposed)
|
|
81
|
+
return () => { };
|
|
82
|
+
state.listeners.add(listener);
|
|
83
|
+
return () => { state.listeners.delete(listener); };
|
|
84
|
+
},
|
|
85
|
+
busy: () => [...this.#contexts.values()].some(context => context.driver === state || context.recoveryDriver === state),
|
|
86
|
+
settle: mode => this.#settleAttachment(state, mode),
|
|
87
|
+
detach: () => {
|
|
88
|
+
state.detaching ??= this.#detach(state);
|
|
89
|
+
return state.detaching;
|
|
90
|
+
},
|
|
91
|
+
};
|
|
92
|
+
return attachment;
|
|
50
93
|
}
|
|
51
94
|
isEnabled() {
|
|
52
95
|
return this.#enabled;
|
|
53
96
|
}
|
|
54
97
|
hasActiveWork() {
|
|
55
|
-
return
|
|
98
|
+
return [...this.#contexts.values()].some(context => !!context.run || !!context.ready || !!context.recovery || context.followUps.size > 0);
|
|
56
99
|
}
|
|
57
100
|
subscribeStatusChanges(listener) {
|
|
58
101
|
if (this.#disposed)
|
|
@@ -60,34 +103,28 @@ export class LocalMemoryCoordinator {
|
|
|
60
103
|
this.#statusListeners.add(listener);
|
|
61
104
|
return () => this.#statusListeners.delete(listener);
|
|
62
105
|
}
|
|
63
|
-
setModelSelection(model, scopedModels, classifier) {
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
this.#selectedModel = model;
|
|
106
|
+
#setModelSelection(attachment, model, scopedModels, classifier) {
|
|
107
|
+
if (attachment.detached || this.#disposed)
|
|
108
|
+
return;
|
|
109
|
+
attachment.selectedModel = model;
|
|
110
|
+
attachment.scopedModels = scopedModels === undefined ? undefined : [...scopedModels];
|
|
111
|
+
attachment.classifier = classifier;
|
|
70
112
|
for (const context of this.#contexts.values()) {
|
|
113
|
+
if (!context.attachments.has(attachment))
|
|
114
|
+
continue;
|
|
71
115
|
context.blocked.clear();
|
|
72
116
|
if (context.initialized)
|
|
73
|
-
void this.#trackScheduleIfPending(context);
|
|
117
|
+
void this.#trackScheduleIfPending(context, attachment.environment);
|
|
74
118
|
}
|
|
75
119
|
}
|
|
76
120
|
setEnabled(enabled) {
|
|
77
121
|
this.#enableVersion += 1;
|
|
78
|
-
enabled = enabled && !this.#options.unavailableReason;
|
|
79
122
|
this.#enabled = enabled;
|
|
80
|
-
if (enabled)
|
|
81
|
-
this.#startMonitor();
|
|
82
|
-
else if (this.#monitor) {
|
|
83
|
-
clearInterval(this.#monitor);
|
|
84
|
-
this.#monitor = undefined;
|
|
85
|
-
}
|
|
86
123
|
for (const context of this.#contexts.values()) {
|
|
87
124
|
if (!enabled)
|
|
88
125
|
context.recoveryAbort?.abort();
|
|
89
126
|
if (!enabled)
|
|
90
|
-
this.#
|
|
127
|
+
this.#cancelTimers(context);
|
|
91
128
|
if (!enabled)
|
|
92
129
|
this.#cancelContext(context, 'Memory processing was disabled', true);
|
|
93
130
|
if (enabled) {
|
|
@@ -107,26 +144,27 @@ export class LocalMemoryCoordinator {
|
|
|
107
144
|
}
|
|
108
145
|
this.#emitStatusChange();
|
|
109
146
|
}
|
|
110
|
-
async status(cwd) {
|
|
147
|
+
async #status(attachment, cwd) {
|
|
111
148
|
try {
|
|
112
|
-
const context = await this.#context(cwd);
|
|
113
|
-
const
|
|
114
|
-
const
|
|
149
|
+
const context = await this.#context(attachment, cwd);
|
|
150
|
+
const store = this.#store(context, attachment.environment);
|
|
151
|
+
const state = await store.status();
|
|
152
|
+
const control = await store.readControl();
|
|
115
153
|
const processed = Object.values(state.processed);
|
|
116
154
|
const lastProcessedAt = processed
|
|
117
155
|
.map((entry) => entry.processedAt)
|
|
118
156
|
.sort()
|
|
119
157
|
.at(-1);
|
|
120
158
|
return {
|
|
121
|
-
enabled: this.#enabled && !control.autoDisabled,
|
|
122
|
-
state: !this.#enabled || control.autoDisabled ? 'disabled'
|
|
159
|
+
enabled: this.#enabled && attachment.scheduling && !control.autoDisabled,
|
|
160
|
+
state: !this.#enabled || !attachment.scheduling || control.autoDisabled ? 'disabled'
|
|
123
161
|
: control.nextRetryAt && Date.parse(control.nextRetryAt) > this.#now() ? 'scheduled'
|
|
124
162
|
: context.state === 'error' ? 'error'
|
|
125
163
|
: control.activeAttempt ? 'processing' : context.state,
|
|
126
164
|
pendingCheckpoints: Object.keys(state.pending).length,
|
|
127
165
|
memoryFingerprint: state.memoryFingerprint,
|
|
128
166
|
...(lastProcessedAt === undefined ? {} : { lastProcessedAt }),
|
|
129
|
-
...(context.message ||
|
|
167
|
+
...(context.message || attachment.unavailableReason ? { message: context.message ?? attachment.unavailableReason } : {}),
|
|
130
168
|
consecutiveFailures: control.consecutiveFailures,
|
|
131
169
|
...(control.nextRetryAt ? { nextRetryAt: control.nextRetryAt } : {}),
|
|
132
170
|
...(control.autoDisabled ? { autoDisabled: control.autoDisabled } : {}),
|
|
@@ -137,37 +175,39 @@ export class LocalMemoryCoordinator {
|
|
|
137
175
|
}
|
|
138
176
|
catch (error) {
|
|
139
177
|
return {
|
|
140
|
-
enabled: this.#enabled,
|
|
141
|
-
state: this.#enabled ? 'error' : 'disabled',
|
|
178
|
+
enabled: this.#enabled && attachment.scheduling,
|
|
179
|
+
state: this.#enabled && attachment.scheduling ? 'error' : 'disabled',
|
|
142
180
|
pendingCheckpoints: 0,
|
|
143
181
|
message: error instanceof LocalMemoryControlError ? error.message : 'Local memory storage is unavailable',
|
|
144
182
|
};
|
|
145
183
|
}
|
|
146
184
|
}
|
|
147
|
-
async readCurrent(cwd, sessionStorageRoot) {
|
|
148
|
-
const context = await this.#context(cwd);
|
|
149
|
-
const
|
|
185
|
+
async #readCurrent(attachment, cwd, sessionStorageRoot) {
|
|
186
|
+
const context = await this.#context(attachment, cwd);
|
|
187
|
+
const store = this.#store(context, attachment.environment);
|
|
188
|
+
const snapshot = await store.readCurrent();
|
|
150
189
|
const projectionKey = `${context.project.key}:${sessionStorageRoot}`;
|
|
151
|
-
const memoryPath = join(sessionStorageRoot,
|
|
152
|
-
if (
|
|
153
|
-
const projection = await
|
|
154
|
-
|
|
190
|
+
const memoryPath = join(sessionStorageRoot, store.projectionName);
|
|
191
|
+
if (attachment.projections.get(projectionKey) !== snapshot.fingerprint) {
|
|
192
|
+
const projection = await store.projectTo(sessionStorageRoot, snapshot);
|
|
193
|
+
if (!attachment.detached)
|
|
194
|
+
attachment.projections.set(projectionKey, snapshot.fingerprint);
|
|
155
195
|
return projection;
|
|
156
196
|
}
|
|
157
197
|
return createMemoryProjectionSnapshot(snapshot, memoryPath);
|
|
158
198
|
}
|
|
159
|
-
async recordCheckpoint(cwd, checkpoint) {
|
|
160
|
-
const context = await this.#context(cwd);
|
|
161
|
-
if (await context.
|
|
199
|
+
async #recordCheckpoint(attachment, cwd, checkpoint) {
|
|
200
|
+
const context = await this.#context(attachment, cwd);
|
|
201
|
+
if (await this.#store(context, attachment.environment).recordCheckpoint(checkpoint)) {
|
|
162
202
|
context.blocked.delete(checkpoint.sessionId);
|
|
163
|
-
await this.#trackScheduleIfPending(context);
|
|
203
|
+
await this.#trackScheduleIfPending(context, attachment.environment);
|
|
164
204
|
this.#emitStatusChange();
|
|
165
205
|
}
|
|
166
206
|
}
|
|
167
|
-
async runNow(cwd) {
|
|
168
|
-
if (!this.#
|
|
169
|
-
return this
|
|
170
|
-
const context = await this.#context(cwd);
|
|
207
|
+
async #runNow(attachment, cwd) {
|
|
208
|
+
if (!this.#eligible(attachment))
|
|
209
|
+
return this.#status(attachment, cwd);
|
|
210
|
+
const context = await this.#context(attachment, cwd);
|
|
171
211
|
this.#cancelScheduledRun(context);
|
|
172
212
|
while (context.run || context.followUps.size > 0) {
|
|
173
213
|
await Promise.allSettled([
|
|
@@ -177,61 +217,117 @@ export class LocalMemoryCoordinator {
|
|
|
177
217
|
this.#cancelScheduledRun(context);
|
|
178
218
|
}
|
|
179
219
|
context.blocked.clear();
|
|
180
|
-
|
|
181
|
-
context.
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
}
|
|
185
|
-
finally {
|
|
186
|
-
if (context.run === run)
|
|
187
|
-
context.run = undefined;
|
|
188
|
-
await this.#trackScheduleIfPending(context);
|
|
189
|
-
}
|
|
190
|
-
return this.status(cwd);
|
|
220
|
+
context.storageBlocked = false;
|
|
221
|
+
context.message = undefined;
|
|
222
|
+
await this.#launchRun(context, true, attachment);
|
|
223
|
+
return this.#status(attachment, cwd);
|
|
191
224
|
}
|
|
192
|
-
async retryProject(cwd) {
|
|
193
|
-
if (!this.#
|
|
194
|
-
return this
|
|
195
|
-
const context = await this.#context(cwd);
|
|
196
|
-
if (!this.#
|
|
197
|
-
return this
|
|
225
|
+
async #retryProject(attachment, cwd) {
|
|
226
|
+
if (!this.#eligible(attachment))
|
|
227
|
+
return this.#status(attachment, cwd);
|
|
228
|
+
const context = await this.#context(attachment, cwd);
|
|
229
|
+
if (!this.#eligible(attachment))
|
|
230
|
+
return this.#status(attachment, cwd);
|
|
198
231
|
const enableVersion = this.#enableVersion;
|
|
199
232
|
this.#cancelScheduledRun(context);
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
}
|
|
203
|
-
catch (error) {
|
|
204
|
-
throw error;
|
|
205
|
-
}
|
|
233
|
+
await this.#store(context, attachment.environment).resetProjectControl(this.#now(), () => this.#eligible(attachment) && this.#enableVersion === enableVersion);
|
|
234
|
+
this.#emitStatusChange();
|
|
206
235
|
context.abort?.abort(new Error('Memory processing was explicitly retried'));
|
|
207
236
|
if (context.run)
|
|
208
237
|
await context.run;
|
|
209
238
|
context.blocked.clear();
|
|
210
239
|
context.storageBlocked = false;
|
|
211
240
|
context.message = undefined;
|
|
212
|
-
await this
|
|
213
|
-
return this
|
|
241
|
+
await this.#runNow(attachment, cwd);
|
|
242
|
+
return this.#status(attachment, cwd);
|
|
243
|
+
}
|
|
244
|
+
#eligible(attachment) {
|
|
245
|
+
return this.#enabled && !this.#disposed && !this.#draining
|
|
246
|
+
&& attachment.scheduling && !attachment.draining && !attachment.closing && !attachment.detached;
|
|
247
|
+
}
|
|
248
|
+
#driver(context, preferred) {
|
|
249
|
+
const members = [...this.#attachments].reverse().filter(attachment => context.attachments.has(attachment) && this.#eligible(attachment));
|
|
250
|
+
if (preferred && !preferred.persistent && members.includes(preferred))
|
|
251
|
+
return preferred;
|
|
252
|
+
return members.find(attachment => !attachment.persistent)
|
|
253
|
+
?? (preferred && members.includes(preferred) ? preferred : members[0]);
|
|
254
|
+
}
|
|
255
|
+
#liveMember(context) {
|
|
256
|
+
const members = [...this.#attachments].reverse().filter(attachment => context.attachments.has(attachment)
|
|
257
|
+
&& attachment.scheduling && !attachment.draining && !attachment.closing && !attachment.detached);
|
|
258
|
+
return members.find(attachment => !attachment.persistent) ?? members[0];
|
|
259
|
+
}
|
|
260
|
+
#runEnvironment(context, driver) {
|
|
261
|
+
// The daemon seals closing runtimes outside their own cleanup async context.
|
|
262
|
+
const current = () => (driver.closing || driver.detached ? this.#liveMember(context) ?? driver : driver).environment;
|
|
263
|
+
const environment = {
|
|
264
|
+
get runtime() { return current().runtime; },
|
|
265
|
+
get sessions() { return current().sessions; },
|
|
266
|
+
};
|
|
267
|
+
Object.defineProperty(environment, 'openWorker', { get: () => current().openWorker });
|
|
268
|
+
return environment;
|
|
269
|
+
}
|
|
270
|
+
async #settleAttachment(attachment, mode) {
|
|
271
|
+
attachment.draining = true;
|
|
272
|
+
if (mode === 'force')
|
|
273
|
+
attachment.closing = true;
|
|
274
|
+
for (const context of this.#contexts.values()) {
|
|
275
|
+
if (context.recoveryDriver === attachment)
|
|
276
|
+
context.recoveryAbort?.abort();
|
|
277
|
+
if (mode === 'force' && context.driver === attachment) {
|
|
278
|
+
this.#cancelContext(context, 'Memory processing was cancelled during shutdown', false);
|
|
279
|
+
}
|
|
280
|
+
if (!this.#driver(context))
|
|
281
|
+
this.#cancelTimers(context);
|
|
282
|
+
}
|
|
283
|
+
while (true) {
|
|
284
|
+
const work = [...this.#contexts.values()].flatMap(context => [
|
|
285
|
+
...(context.driver === attachment ? [...(context.run ? [context.run] : []), ...context.followUps] : []),
|
|
286
|
+
...(context.recoveryDriver === attachment && context.recovery ? [context.recovery] : []),
|
|
287
|
+
]);
|
|
288
|
+
if (!work.length)
|
|
289
|
+
break;
|
|
290
|
+
await Promise.allSettled(work);
|
|
291
|
+
}
|
|
292
|
+
for (const context of this.#contexts.values()) {
|
|
293
|
+
if (mode === 'force' && context.failedDriver === attachment) {
|
|
294
|
+
context.storageBlocked = false;
|
|
295
|
+
context.blocked.clear();
|
|
296
|
+
if (context.state === 'error' || context.state === 'blocked')
|
|
297
|
+
context.state = 'idle';
|
|
298
|
+
context.message = undefined;
|
|
299
|
+
context.failedDriver = undefined;
|
|
300
|
+
}
|
|
301
|
+
if (!context.attachments.has(attachment) || !context.initialized)
|
|
302
|
+
continue;
|
|
303
|
+
this.#startRecovery(context);
|
|
304
|
+
void this.#trackScheduleIfPending(context);
|
|
305
|
+
}
|
|
214
306
|
}
|
|
215
|
-
async
|
|
216
|
-
|
|
217
|
-
|
|
307
|
+
async #detach(attachment) {
|
|
308
|
+
attachment.closing = true;
|
|
309
|
+
await this.#settleAttachment(attachment, 'force');
|
|
310
|
+
attachment.detached = true;
|
|
311
|
+
this.#attachments.delete(attachment);
|
|
312
|
+
attachment.listeners.clear();
|
|
313
|
+
attachment.projections.clear();
|
|
314
|
+
for (const context of this.#contexts.values()) {
|
|
315
|
+
context.attachments.delete(attachment);
|
|
316
|
+
if (!this.#driver(context))
|
|
317
|
+
this.#cancelTimers(context);
|
|
318
|
+
}
|
|
218
319
|
}
|
|
219
320
|
async drain() {
|
|
220
321
|
this.#draining = true;
|
|
221
|
-
if (this.#monitor)
|
|
222
|
-
clearInterval(this.#monitor);
|
|
223
|
-
this.#monitor = undefined;
|
|
224
322
|
for (const context of this.#contexts.values()) {
|
|
225
323
|
context.recoveryAbort?.abort();
|
|
226
|
-
this.#
|
|
324
|
+
this.#cancelTimers(context);
|
|
227
325
|
}
|
|
228
326
|
while (true) {
|
|
229
327
|
const work = [...this.#contexts.values()].flatMap(context => [
|
|
230
328
|
...(context.run ? [context.run] : []), ...(context.ready ? [context.ready] : []),
|
|
231
329
|
...(context.recovery ? [context.recovery] : []), ...context.followUps,
|
|
232
330
|
]);
|
|
233
|
-
if (this.#monitoring)
|
|
234
|
-
work.push(this.#monitoring);
|
|
235
331
|
if (!work.length)
|
|
236
332
|
return;
|
|
237
333
|
const results = await Promise.allSettled(work);
|
|
@@ -240,103 +336,121 @@ export class LocalMemoryCoordinator {
|
|
|
240
336
|
throw new AggregateError(failures, 'Memory drain could not settle all work.');
|
|
241
337
|
}
|
|
242
338
|
}
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
339
|
+
dispose() {
|
|
340
|
+
this.#disposal ??= this.#dispose();
|
|
341
|
+
return this.#disposal;
|
|
342
|
+
}
|
|
343
|
+
async #dispose() {
|
|
246
344
|
this.#disposed = true;
|
|
247
|
-
if (this.#monitor)
|
|
248
|
-
clearInterval(this.#monitor);
|
|
249
|
-
this.#monitor = undefined;
|
|
250
345
|
for (const context of this.#contexts.values()) {
|
|
251
346
|
context.recoveryAbort?.abort();
|
|
252
|
-
this.#
|
|
347
|
+
this.#cancelTimers(context);
|
|
253
348
|
this.#cancelContext(context, 'Memory processing was cancelled during shutdown', false);
|
|
254
349
|
}
|
|
255
350
|
while (true) {
|
|
256
|
-
const work = [...this.#contexts.values()].flatMap(
|
|
351
|
+
const work = [...this.#contexts.values()].flatMap(context => [
|
|
257
352
|
...(context.run ? [context.run] : []),
|
|
258
353
|
...(context.ready ? [context.ready] : []),
|
|
259
354
|
...(context.recovery ? [context.recovery] : []),
|
|
260
355
|
...context.followUps,
|
|
261
356
|
]);
|
|
262
|
-
if (
|
|
263
|
-
work.push(this.#monitoring);
|
|
264
|
-
if (work.length === 0)
|
|
357
|
+
if (!work.length)
|
|
265
358
|
break;
|
|
266
359
|
await Promise.allSettled(work);
|
|
267
360
|
}
|
|
361
|
+
for (const attachment of this.#attachments) {
|
|
362
|
+
attachment.detached = true;
|
|
363
|
+
attachment.listeners.clear();
|
|
364
|
+
attachment.projections.clear();
|
|
365
|
+
}
|
|
366
|
+
this.#attachments.clear();
|
|
268
367
|
this.#statusListeners.clear();
|
|
269
368
|
}
|
|
270
|
-
async #context(cwd) {
|
|
271
|
-
if (this.#disposed)
|
|
369
|
+
async #context(attachment, cwd) {
|
|
370
|
+
if (this.#disposed || attachment.detached)
|
|
272
371
|
throw new Error('Local memory coordinator is closed');
|
|
273
|
-
const project = await resolveLocalMemoryProject(
|
|
274
|
-
if (this.#disposed)
|
|
372
|
+
const project = await resolveLocalMemoryProject(attachment.environment, cwd);
|
|
373
|
+
if (this.#disposed || attachment.detached)
|
|
275
374
|
throw new Error('Local memory coordinator is closed');
|
|
276
375
|
const existing = this.#contexts.get(project.key);
|
|
277
376
|
if (existing) {
|
|
278
|
-
|
|
377
|
+
existing.attachments.add(attachment);
|
|
378
|
+
await this.#initialize(existing, attachment);
|
|
279
379
|
return existing;
|
|
280
380
|
}
|
|
281
|
-
const store = new LocalMemoryStore(this.environment, this.#options.agentDir, project, {
|
|
282
|
-
...(this.#options.retryDelaysMs ? { retryDelaysMs: this.#options.retryDelaysMs } : {}),
|
|
283
|
-
});
|
|
284
381
|
const context = {
|
|
285
382
|
project,
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
383
|
+
stores: new WeakMap(),
|
|
384
|
+
attachments: new Set([attachment]),
|
|
385
|
+
driver: undefined,
|
|
386
|
+
failedDriver: undefined,
|
|
387
|
+
recoveryDriver: undefined,
|
|
289
388
|
ready: undefined,
|
|
290
389
|
initialized: false,
|
|
291
390
|
recovery: undefined,
|
|
292
391
|
recoveryAbort: undefined,
|
|
293
392
|
recoveryRetryAt: undefined,
|
|
393
|
+
recoveryTimer: undefined,
|
|
294
394
|
recoveryCompleted: false,
|
|
295
395
|
diagnosticsReconciled: false,
|
|
296
396
|
timer: undefined,
|
|
297
397
|
timerDueAt: undefined,
|
|
298
398
|
run: undefined,
|
|
399
|
+
runEnvironment: undefined,
|
|
299
400
|
followUps: new Set(),
|
|
300
401
|
abort: undefined,
|
|
301
402
|
attempt: undefined,
|
|
302
403
|
storageBlocked: false,
|
|
303
404
|
writerBlocked: false,
|
|
304
|
-
|
|
405
|
+
writerTimer: undefined,
|
|
305
406
|
blocked: new Map(),
|
|
306
407
|
state: this.#enabled ? 'idle' : 'disabled',
|
|
307
408
|
message: undefined,
|
|
308
409
|
};
|
|
309
410
|
this.#contexts.set(project.key, context);
|
|
310
|
-
await this.#initialize(context);
|
|
411
|
+
await this.#initialize(context, attachment);
|
|
311
412
|
return context;
|
|
312
413
|
}
|
|
313
|
-
|
|
414
|
+
#store(context, environment) {
|
|
415
|
+
let store = context.stores.get(environment);
|
|
416
|
+
if (!store) {
|
|
417
|
+
store = new LocalMemoryStore(environment, this.#options.agentDir, context.project, {
|
|
418
|
+
...(this.#options.retryDelaysMs ? { retryDelaysMs: this.#options.retryDelaysMs } : {}),
|
|
419
|
+
});
|
|
420
|
+
context.stores.set(environment, store);
|
|
421
|
+
}
|
|
422
|
+
return store;
|
|
423
|
+
}
|
|
424
|
+
async #initialize(context, attachment) {
|
|
314
425
|
if (context.ready) {
|
|
315
426
|
await context.ready;
|
|
316
427
|
return;
|
|
317
428
|
}
|
|
318
429
|
if (context.initialized && context.diagnosticsReconciled) {
|
|
319
430
|
this.#startRecovery(context);
|
|
431
|
+
void this.#trackScheduleIfPending(context, attachment.environment);
|
|
320
432
|
return;
|
|
321
433
|
}
|
|
322
|
-
const ready = this.#initializeContext(context).finally(() => {
|
|
434
|
+
const ready = this.#initializeContext(context, attachment).finally(() => {
|
|
323
435
|
if (context.ready === ready)
|
|
324
436
|
context.ready = undefined;
|
|
325
437
|
});
|
|
326
438
|
context.ready = ready;
|
|
327
439
|
await ready;
|
|
328
440
|
}
|
|
329
|
-
async #initializeContext(context) {
|
|
441
|
+
async #initializeContext(context, attachment) {
|
|
442
|
+
const environment = attachment.environment;
|
|
443
|
+
const store = this.#store(context, environment);
|
|
330
444
|
if (!context.initialized) {
|
|
331
|
-
await
|
|
445
|
+
await store.initialize(this.#now()).then(() => {
|
|
332
446
|
context.initialized = true;
|
|
333
447
|
context.state = this.#enabled ? 'idle' : 'disabled';
|
|
334
448
|
context.message = undefined;
|
|
335
449
|
}).catch(async (error) => {
|
|
336
450
|
if (!(error instanceof LocalMemoryControlError))
|
|
337
451
|
throw error;
|
|
338
|
-
const current = await
|
|
339
|
-
if (current.fingerprint !== (await
|
|
452
|
+
const current = await store.readCurrent();
|
|
453
|
+
if (current.fingerprint !== (await store.status()).memoryFingerprint)
|
|
340
454
|
throw error;
|
|
341
455
|
context.message = error.message;
|
|
342
456
|
});
|
|
@@ -345,11 +459,16 @@ export class LocalMemoryCoordinator {
|
|
|
345
459
|
return;
|
|
346
460
|
if (!context.diagnosticsReconciled) {
|
|
347
461
|
try {
|
|
348
|
-
const lease = await acquireLocalMemoryLease(
|
|
462
|
+
const lease = await acquireLocalMemoryLease(environment, store.projectDirectory, this.#options.leaseOptions);
|
|
349
463
|
if (lease) {
|
|
350
464
|
try {
|
|
351
|
-
await this.#reconcileRunDiagnostics(context, lease);
|
|
465
|
+
await this.#reconcileRunDiagnostics(context, environment, lease);
|
|
352
466
|
context.diagnosticsReconciled = true;
|
|
467
|
+
if (context.storageBlocked) {
|
|
468
|
+
context.storageBlocked = false;
|
|
469
|
+
context.failedDriver = undefined;
|
|
470
|
+
context.message = undefined;
|
|
471
|
+
}
|
|
353
472
|
}
|
|
354
473
|
finally {
|
|
355
474
|
await lease.release();
|
|
@@ -357,24 +476,34 @@ export class LocalMemoryCoordinator {
|
|
|
357
476
|
}
|
|
358
477
|
}
|
|
359
478
|
catch (error) {
|
|
360
|
-
context.diagnosticsReconciled = true;
|
|
361
479
|
context.storageBlocked = true;
|
|
480
|
+
context.failedDriver = attachment;
|
|
362
481
|
context.message = `Memory diagnostics recovery is blocked: ${sanitizeMemoryDiagnostic(error)}`;
|
|
363
482
|
}
|
|
364
483
|
}
|
|
365
484
|
if (this.#disposed)
|
|
366
485
|
return;
|
|
367
486
|
if (this.#enabled)
|
|
368
|
-
void this.#trackScheduleIfPending(context);
|
|
487
|
+
void this.#trackScheduleIfPending(context, environment);
|
|
369
488
|
this.#startRecovery(context);
|
|
370
489
|
}
|
|
371
490
|
#startRecovery(context) {
|
|
372
|
-
if (this.#
|
|
373
|
-
|
|
374
|
-
|
|
491
|
+
if (this.#options.recover === false || context.recoveryCompleted || context.recovery)
|
|
492
|
+
return;
|
|
493
|
+
const driver = this.#driver(context);
|
|
494
|
+
if (!driver)
|
|
495
|
+
return;
|
|
496
|
+
if (context.recoveryRetryAt !== undefined && this.#now() < context.recoveryRetryAt) {
|
|
497
|
+
this.#scheduleRecoveryRetry(context);
|
|
375
498
|
return;
|
|
499
|
+
}
|
|
500
|
+
if (context.recoveryTimer)
|
|
501
|
+
clearTimeout(context.recoveryTimer);
|
|
502
|
+
context.recoveryTimer = undefined;
|
|
503
|
+
context.recoveryDriver = driver;
|
|
376
504
|
const abort = new AbortController();
|
|
377
|
-
|
|
505
|
+
context.recoveryAbort = abort;
|
|
506
|
+
const recovery = this.#recover(context, driver, abort.signal).then(completed => {
|
|
378
507
|
if (completed)
|
|
379
508
|
context.recoveryCompleted = true;
|
|
380
509
|
}).catch(() => {
|
|
@@ -383,71 +512,70 @@ export class LocalMemoryCoordinator {
|
|
|
383
512
|
if (context.recovery === recovery) {
|
|
384
513
|
context.recovery = undefined;
|
|
385
514
|
context.recoveryAbort = undefined;
|
|
515
|
+
context.recoveryDriver = undefined;
|
|
386
516
|
}
|
|
517
|
+
if (abort.signal.aborted)
|
|
518
|
+
this.#startRecovery(context);
|
|
519
|
+
else
|
|
520
|
+
this.#scheduleRecoveryRetry(context);
|
|
521
|
+
this.#emitStatusChange();
|
|
387
522
|
});
|
|
388
523
|
context.recovery = recovery;
|
|
389
|
-
context.recoveryAbort = abort;
|
|
390
524
|
}
|
|
391
|
-
#
|
|
392
|
-
|
|
393
|
-
if (this.#disposed || this.#draining || this.#monitor)
|
|
525
|
+
#scheduleRecoveryRetry(context) {
|
|
526
|
+
if (context.recoveryTimer || context.recoveryCompleted || context.recoveryRetryAt === undefined || !this.#driver(context))
|
|
394
527
|
return;
|
|
395
|
-
const
|
|
396
|
-
|
|
397
|
-
throw new Error('Memory monitor interval must be positive');
|
|
398
|
-
this.#monitor = setInterval(() => {
|
|
399
|
-
if (this.#monitoring)
|
|
528
|
+
const timer = setTimeout(() => {
|
|
529
|
+
if (context.recoveryTimer !== timer)
|
|
400
530
|
return;
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
this.#
|
|
531
|
+
context.recoveryTimer = undefined;
|
|
532
|
+
this.#startRecovery(context);
|
|
533
|
+
}, Math.max(0, context.recoveryRetryAt - this.#now()));
|
|
534
|
+
context.recoveryTimer = timer;
|
|
535
|
+
timer.unref?.();
|
|
404
536
|
}
|
|
405
|
-
|
|
406
|
-
if (
|
|
537
|
+
#scheduleWriterRetry(context) {
|
|
538
|
+
if (context.writerTimer || !this.#driver(context))
|
|
407
539
|
return;
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
}
|
|
418
|
-
context.writerBlocked = false;
|
|
419
|
-
if (context.initialized)
|
|
420
|
-
await this.#trackScheduleIfPending(context);
|
|
421
|
-
const signature = JSON.stringify([this.#enabled, context.state, context.message, control.updatedAt]);
|
|
422
|
-
if (signature !== context.statusSignature) {
|
|
423
|
-
context.statusSignature = signature;
|
|
424
|
-
this.#emitStatusChange();
|
|
425
|
-
}
|
|
426
|
-
}
|
|
427
|
-
catch (error) {
|
|
428
|
-
context.state = 'error';
|
|
429
|
-
context.message = sanitizeMemoryDiagnostic(error);
|
|
430
|
-
context.abort?.abort(error);
|
|
431
|
-
this.#emitStatusChange();
|
|
540
|
+
context.writerBlocked = true;
|
|
541
|
+
const timer = setTimeout(() => {
|
|
542
|
+
if (context.writerTimer !== timer)
|
|
543
|
+
return;
|
|
544
|
+
context.writerTimer = undefined;
|
|
545
|
+
context.writerBlocked = false;
|
|
546
|
+
if (context.run) {
|
|
547
|
+
void this.#trackScheduleIfPending(context);
|
|
548
|
+
return;
|
|
432
549
|
}
|
|
433
|
-
|
|
550
|
+
context.diagnosticsReconciled = false;
|
|
551
|
+
const driver = this.#driver(context);
|
|
552
|
+
if (driver)
|
|
553
|
+
void this.#initialize(context, driver).catch(() => { });
|
|
554
|
+
}, this.#options.writerRetryMs ?? 30_000);
|
|
555
|
+
context.writerTimer = timer;
|
|
556
|
+
timer.unref?.();
|
|
434
557
|
}
|
|
558
|
+
#now() { return this.#options.now?.() ?? Date.now(); }
|
|
435
559
|
#cancelContext(context, reason, cancelPersisted) {
|
|
436
560
|
const alreadyAborted = context.abort?.signal.aborted ?? false;
|
|
437
561
|
context.abort?.abort(new Error(reason));
|
|
438
562
|
if (alreadyAborted || !context.initialized || (!cancelPersisted && !context.attempt))
|
|
439
563
|
return;
|
|
564
|
+
const environment = context.runEnvironment ?? this.#liveMember(context)?.environment;
|
|
565
|
+
if (!environment)
|
|
566
|
+
return;
|
|
567
|
+
const store = this.#store(context, environment);
|
|
440
568
|
const cancellation = cancelPersisted
|
|
441
|
-
?
|
|
442
|
-
:
|
|
569
|
+
? store.cancelActiveAttempt(this.#now())
|
|
570
|
+
: store.finishAttempt(undefined, context.attempt, { status: 'cancelled', reason }, this.#now());
|
|
443
571
|
const followUp = cancellation
|
|
444
|
-
.then(() => { })
|
|
572
|
+
.then(() => { this.#emitStatusChange(); })
|
|
445
573
|
.catch(() => { })
|
|
446
574
|
.finally(() => { context.followUps.delete(followUp); });
|
|
447
575
|
context.followUps.add(followUp);
|
|
448
576
|
}
|
|
449
577
|
async #schedule(context, dueAt) {
|
|
450
|
-
if (
|
|
578
|
+
if (!this.#driver(context) || context.run || context.storageBlocked)
|
|
451
579
|
return;
|
|
452
580
|
if (context.timer && context.timerDueAt === dueAt)
|
|
453
581
|
return;
|
|
@@ -459,28 +587,26 @@ export class LocalMemoryCoordinator {
|
|
|
459
587
|
return;
|
|
460
588
|
context.timer = undefined;
|
|
461
589
|
context.timerDueAt = undefined;
|
|
462
|
-
|
|
463
|
-
context.state = 'error';
|
|
464
|
-
context.storageBlocked = true;
|
|
465
|
-
context.message = sanitizeMemoryDiagnostic(error);
|
|
466
|
-
this.#emitStatusChange();
|
|
467
|
-
}).finally(async () => {
|
|
468
|
-
context.run = undefined;
|
|
469
|
-
await this.#trackScheduleIfPending(context);
|
|
470
|
-
});
|
|
471
|
-
context.run = run;
|
|
590
|
+
void this.#launchRun(context)?.catch(() => { });
|
|
472
591
|
}, delay);
|
|
473
592
|
context.timer = timer;
|
|
474
593
|
context.timerDueAt = dueAt;
|
|
475
594
|
timer.unref?.();
|
|
476
595
|
}
|
|
477
|
-
async #scheduleIfPending(context) {
|
|
478
|
-
|
|
596
|
+
async #scheduleIfPending(context, environment) {
|
|
597
|
+
const driver = this.#driver(context);
|
|
598
|
+
if (!driver) {
|
|
599
|
+
this.#cancelTimers(context);
|
|
479
600
|
return;
|
|
601
|
+
}
|
|
602
|
+
// Callers capture their environment before awaiting; a member sealed since then cannot read the store.
|
|
603
|
+
const owner = environment && [...context.attachments].find(attachment => attachment.environment === environment);
|
|
604
|
+
const source = owner && (owner.closing || owner.detached) ? (this.#liveMember(context) ?? owner).environment : environment;
|
|
480
605
|
try {
|
|
481
|
-
const
|
|
606
|
+
const store = this.#store(context, source ?? driver.environment);
|
|
607
|
+
const control = await store.readControl();
|
|
482
608
|
if (control.autoDisabled) {
|
|
483
|
-
this.#
|
|
609
|
+
this.#cancelTimers(context);
|
|
484
610
|
context.state = 'disabled';
|
|
485
611
|
return;
|
|
486
612
|
}
|
|
@@ -488,13 +614,25 @@ export class LocalMemoryCoordinator {
|
|
|
488
614
|
return;
|
|
489
615
|
if (context.writerBlocked)
|
|
490
616
|
return;
|
|
491
|
-
const state = await
|
|
617
|
+
const state = await store.status();
|
|
618
|
+
if (context.state === 'error' && context.message === 'Local memory storage is unavailable') {
|
|
619
|
+
context.state = 'idle';
|
|
620
|
+
context.message = undefined;
|
|
621
|
+
}
|
|
492
622
|
const pending = Object.values(state.pending);
|
|
623
|
+
if (!this.#driver(context))
|
|
624
|
+
return;
|
|
625
|
+
if (pending.length && !await this.#projectAvailable(context, source ?? driver.environment)) {
|
|
626
|
+
this.#cancelScheduledRun(context);
|
|
627
|
+
return;
|
|
628
|
+
}
|
|
493
629
|
if (control.activeAttempt) {
|
|
494
630
|
this.#cancelScheduledRun(context);
|
|
631
|
+
if (!context.run)
|
|
632
|
+
this.#scheduleWriterRetry(context);
|
|
495
633
|
}
|
|
496
634
|
else if (pending.some(({ checkpoint }) => !isBlockedCheckpoint(context, checkpoint))) {
|
|
497
|
-
const acceptedUpdates = await
|
|
635
|
+
const acceptedUpdates = await store.pendingAcceptedUpdates((checkpoint) => !isBlockedCheckpoint(context, checkpoint));
|
|
498
636
|
if (acceptedUpdates < MEMORY_AUTOMATIC_UPDATE_THRESHOLD) {
|
|
499
637
|
this.#cancelScheduledRun(context);
|
|
500
638
|
if (context.state === 'scheduled')
|
|
@@ -529,37 +667,105 @@ export class LocalMemoryCoordinator {
|
|
|
529
667
|
context.message = 'Local memory storage is unavailable';
|
|
530
668
|
}
|
|
531
669
|
}
|
|
532
|
-
#trackScheduleIfPending(context) {
|
|
533
|
-
const followUp = this.#scheduleIfPending(context).finally(() => {
|
|
670
|
+
#trackScheduleIfPending(context, environment) {
|
|
671
|
+
const followUp = this.#scheduleIfPending(context, environment).finally(() => {
|
|
534
672
|
context.followUps.delete(followUp);
|
|
535
673
|
});
|
|
536
674
|
context.followUps.add(followUp);
|
|
537
675
|
return followUp;
|
|
538
676
|
}
|
|
539
|
-
async #
|
|
677
|
+
async #projectAvailable(context, environment) {
|
|
678
|
+
if (!environment.openWorker)
|
|
679
|
+
return true;
|
|
680
|
+
try {
|
|
681
|
+
if ((await environment.runtime.files.inspect(context.project.canonicalRoot)).kind === 'directory') {
|
|
682
|
+
if (context.message === MEMORY_PROJECT_UNAVAILABLE_MESSAGE) {
|
|
683
|
+
context.message = undefined;
|
|
684
|
+
if (context.state === 'blocked')
|
|
685
|
+
context.state = 'idle';
|
|
686
|
+
}
|
|
687
|
+
return true;
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
catch (error) {
|
|
691
|
+
if (!error || typeof error !== 'object' || !('code' in error) || error.code !== 'ENOENT')
|
|
692
|
+
throw error;
|
|
693
|
+
}
|
|
694
|
+
context.state = 'blocked';
|
|
695
|
+
context.message = MEMORY_PROJECT_UNAVAILABLE_MESSAGE;
|
|
696
|
+
return false;
|
|
697
|
+
}
|
|
698
|
+
#launchRun(context, bypassRetryAt = false, preferred) {
|
|
699
|
+
if (context.run || context.storageBlocked || preferred && !this.#eligible(preferred))
|
|
700
|
+
return;
|
|
701
|
+
const driver = this.#driver(context, preferred);
|
|
702
|
+
if (!driver)
|
|
703
|
+
return;
|
|
704
|
+
context.driver = driver;
|
|
705
|
+
context.failedDriver = undefined;
|
|
706
|
+
const environment = this.#runEnvironment(context, driver);
|
|
707
|
+
context.runEnvironment = environment;
|
|
708
|
+
const run = this.#run(context, driver, environment, bypassRetryAt).catch((error) => {
|
|
709
|
+
context.state = 'error';
|
|
710
|
+
context.failedDriver = driver;
|
|
711
|
+
if (!bypassRetryAt && !driver.closing && !driver.detached)
|
|
712
|
+
context.storageBlocked = true;
|
|
713
|
+
context.message = sanitizeMemoryDiagnostic(error);
|
|
714
|
+
throw error;
|
|
715
|
+
}).finally(async () => {
|
|
716
|
+
if (context.run === settled) {
|
|
717
|
+
context.run = undefined;
|
|
718
|
+
context.driver = undefined;
|
|
719
|
+
context.runEnvironment = undefined;
|
|
720
|
+
context.abort = undefined;
|
|
721
|
+
}
|
|
722
|
+
await this.#trackScheduleIfPending(context);
|
|
723
|
+
this.#emitStatusChange();
|
|
724
|
+
});
|
|
725
|
+
const settled = run.catch(() => { });
|
|
726
|
+
context.run = settled;
|
|
727
|
+
return run;
|
|
728
|
+
}
|
|
729
|
+
async #run(context, driver, environment, bypassRetryAt) {
|
|
540
730
|
if (this.#disposed || this.#draining || !this.#enabled || context.storageBlocked)
|
|
541
731
|
return;
|
|
542
732
|
context.state = 'processing';
|
|
543
733
|
context.message = undefined;
|
|
734
|
+
const abort = new AbortController();
|
|
735
|
+
context.abort = abort;
|
|
736
|
+
const store = this.#store(context, environment);
|
|
544
737
|
let lease;
|
|
738
|
+
const cancelled = () => driver.closing || driver.detached || !this.#enabled || this.#disposed
|
|
739
|
+
|| abort.signal.aborted && !lease?.compromised.aborted;
|
|
545
740
|
try {
|
|
546
|
-
lease = await acquireLocalMemoryLease(
|
|
741
|
+
lease = await acquireLocalMemoryLease(environment, store.projectDirectory, this.#options.leaseOptions);
|
|
547
742
|
}
|
|
548
743
|
catch (error) {
|
|
549
|
-
|
|
744
|
+
if (cancelled()) {
|
|
745
|
+
context.state = 'idle';
|
|
746
|
+
context.message = driver.closing || driver.detached || this.#disposed
|
|
747
|
+
? 'Memory processing was cancelled during shutdown' : 'Memory processing was cancelled';
|
|
748
|
+
return;
|
|
749
|
+
}
|
|
750
|
+
this.#scheduleWriterRetry(context);
|
|
550
751
|
context.state = 'error';
|
|
752
|
+
context.failedDriver = driver;
|
|
551
753
|
context.message = safeProcessingMessage(error);
|
|
552
754
|
return;
|
|
553
755
|
}
|
|
554
756
|
if (!lease) {
|
|
555
|
-
context
|
|
757
|
+
this.#scheduleWriterRetry(context);
|
|
556
758
|
context.state = 'blocked';
|
|
759
|
+
context.failedDriver = driver;
|
|
557
760
|
context.message = 'Another Felan Code process owns the memory writer';
|
|
558
761
|
return;
|
|
559
762
|
}
|
|
560
763
|
context.writerBlocked = false;
|
|
561
|
-
|
|
562
|
-
|
|
764
|
+
if (context.failedDriver === driver)
|
|
765
|
+
context.failedDriver = undefined;
|
|
766
|
+
if (context.writerTimer)
|
|
767
|
+
clearTimeout(context.writerTimer);
|
|
768
|
+
context.writerTimer = undefined;
|
|
563
769
|
const abortOnLeaseCompromise = () => {
|
|
564
770
|
abort.abort(new Error('Memory writer lease was lost'));
|
|
565
771
|
};
|
|
@@ -569,7 +775,9 @@ export class LocalMemoryCoordinator {
|
|
|
569
775
|
let published = false;
|
|
570
776
|
let attempt;
|
|
571
777
|
try {
|
|
572
|
-
|
|
778
|
+
if (abort.signal.aborted || !this.#enabled || this.#disposed)
|
|
779
|
+
return;
|
|
780
|
+
const control = await this.#reconcileRunDiagnostics(context, environment, lease);
|
|
573
781
|
if (control.autoDisabled) {
|
|
574
782
|
context.state = 'disabled';
|
|
575
783
|
return;
|
|
@@ -578,24 +786,27 @@ export class LocalMemoryCoordinator {
|
|
|
578
786
|
context.state = 'scheduled';
|
|
579
787
|
return;
|
|
580
788
|
}
|
|
581
|
-
|
|
789
|
+
// Every pending checkpoint is a candidate so unreadable ones cannot occupy the batch.
|
|
790
|
+
const snapshot = await store.processingSnapshot(Number.POSITIVE_INFINITY, (checkpoint) => !isBlockedCheckpoint(context, checkpoint));
|
|
582
791
|
if (snapshot.checkpoints.length === 0) {
|
|
583
792
|
context.state = 'idle';
|
|
584
793
|
return;
|
|
585
794
|
}
|
|
586
|
-
|
|
587
|
-
|
|
795
|
+
if (!await this.#projectAvailable(context, environment))
|
|
796
|
+
return;
|
|
797
|
+
retainedRun = await LocalMemoryRun.create(environment, {
|
|
798
|
+
projectDirectory: store.projectDirectory,
|
|
588
799
|
sessionDirectory: join(this.#options.sessionDir ?? join(this.#options.agentDir, 'sessions'), 'memory'),
|
|
589
800
|
projectKey: context.project.key,
|
|
590
801
|
projectRoot: context.project.canonicalRoot,
|
|
591
|
-
checkpoints: snapshot.checkpoints.map(({ checkpoint }) => checkpoint),
|
|
802
|
+
checkpoints: snapshot.checkpoints.slice(0, this.#batchSize).map(({ checkpoint }) => checkpoint),
|
|
592
803
|
baseFingerprint: snapshot.fingerprint,
|
|
593
804
|
...(control.lastDisabledRunId ? { protectedRunIds: [control.lastDisabledRunId] } : {}),
|
|
594
805
|
});
|
|
595
806
|
staging = retainedRun.workspace;
|
|
596
807
|
const memoryDirectory = join(staging, '.memory');
|
|
597
|
-
await hydrateMemoryDirectory(
|
|
598
|
-
const manifest = await materializeMemoryInput(
|
|
808
|
+
await hydrateMemoryDirectory(environment, snapshot.artifact, memoryDirectory, { memoryPath: '.memory', mode: 'read' });
|
|
809
|
+
const manifest = await materializeMemoryInput(environment, {
|
|
599
810
|
stagingDirectory: staging,
|
|
600
811
|
checkpoints: snapshot.checkpoints.map(({ checkpoint }) => checkpoint),
|
|
601
812
|
previousCheckpoints: Object.fromEntries(Object.entries(snapshot.state.processed).map(([sessionId, entry]) => [sessionId, entry.checkpoint])),
|
|
@@ -604,26 +815,29 @@ export class LocalMemoryCoordinator {
|
|
|
604
815
|
...(this.#options.maxTranscriptBytes === undefined
|
|
605
816
|
? {}
|
|
606
817
|
: { maxTranscriptBytes: this.#options.maxTranscriptBytes }),
|
|
818
|
+
maxSessions: this.#batchSize,
|
|
607
819
|
signal: abort.signal,
|
|
608
820
|
});
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
821
|
+
// A cancelled read reports as a failure; it must not drop evidence.
|
|
822
|
+
if (abort.signal.aborted || !this.#enabled || this.#disposed)
|
|
823
|
+
throw new Error('Memory processing was cancelled');
|
|
824
|
+
// A reachable Sessions service shows the failures belong to the checkpoints, not to the service.
|
|
825
|
+
if (manifest.failures.length > 0)
|
|
826
|
+
await environment.sessions.list({ scope: 'project' }, { limit: 1, signal: abort.signal });
|
|
827
|
+
const dropped = manifest.failures.length === 0 ? 0 : await store.dropCheckpoints(lease, manifest.failures.map(({ checkpoint, code }) => ({ checkpoint, reason: code })), this.#now());
|
|
828
|
+
const droppedMessage = dropped === 0 ? undefined : droppedCheckpointsMessage(dropped);
|
|
615
829
|
const processedCheckpoints = snapshot.checkpoints.filter(({ checkpoint }) => manifest.sessions.some((session) => session.checkpoint.sessionId === checkpoint.sessionId
|
|
616
830
|
&& sameCheckpointCursor(session.checkpoint, checkpoint)));
|
|
617
831
|
if (processedCheckpoints.length === 0) {
|
|
618
|
-
context.state = '
|
|
619
|
-
context.message =
|
|
620
|
-
await retainedRun.finish('blocked',
|
|
832
|
+
context.state = 'idle';
|
|
833
|
+
context.message = droppedMessage;
|
|
834
|
+
await retainedRun.finish('blocked', droppedMessage ?? 'Memory checkpoints changed before they could be read');
|
|
621
835
|
return;
|
|
622
836
|
}
|
|
623
837
|
await retainedRun.record({ phase: 'model', checkpoints: processedCheckpoints.map(({ checkpoint }) => checkpoint) });
|
|
624
838
|
if (abort.signal.aborted || !this.#enabled || this.#disposed)
|
|
625
839
|
throw new Error('Memory processing was cancelled');
|
|
626
|
-
attempt = await
|
|
840
|
+
attempt = await store.beginAttempt(lease, {
|
|
627
841
|
runId: retainedRun.metadata.sessionId,
|
|
628
842
|
baseFingerprint: snapshot.fingerprint,
|
|
629
843
|
checkpoints: processedCheckpoints,
|
|
@@ -635,16 +849,16 @@ export class LocalMemoryCoordinator {
|
|
|
635
849
|
}
|
|
636
850
|
context.attempt = attempt;
|
|
637
851
|
this.#emitStatusChange();
|
|
638
|
-
const result = await this.#dreamRunner({
|
|
852
|
+
const result = await (this.#options.dreamRunner ?? createDefaultLocalMemoryDreamRunner(environment))({
|
|
639
853
|
stagingDirectory: staging,
|
|
640
854
|
memoryDirectory,
|
|
641
855
|
inputDirectory: join(staging, '.dreaming', 'input'),
|
|
642
856
|
baseSnapshot: createMemorySnapshot(snapshot.artifact, '.memory', { mode: 'read' }),
|
|
643
857
|
manifest,
|
|
644
|
-
models:
|
|
645
|
-
...(
|
|
646
|
-
...(
|
|
647
|
-
...(
|
|
858
|
+
models: driver.models,
|
|
859
|
+
...(driver.selectedModel === undefined ? {} : { selectedModel: driver.selectedModel }),
|
|
860
|
+
...(driver.scopedModels === undefined ? {} : { scopedModels: driver.scopedModels }),
|
|
861
|
+
...(driver.classifier === undefined ? {} : { classifier: driver.classifier }),
|
|
648
862
|
sessionDirectory: join(this.#options.sessionDir ?? join(this.#options.agentDir, 'sessions'), 'memory'),
|
|
649
863
|
run: retainedRun,
|
|
650
864
|
signal: abort.signal,
|
|
@@ -652,9 +866,9 @@ export class LocalMemoryCoordinator {
|
|
|
652
866
|
if (abort.signal.aborted || !this.#enabled || this.#disposed)
|
|
653
867
|
throw new Error('Memory processing was cancelled');
|
|
654
868
|
if (result)
|
|
655
|
-
await hydrateMemoryDirectory(
|
|
869
|
+
await hydrateMemoryDirectory(environment, result, memoryDirectory, { replace: true, memoryPath: '.memory' });
|
|
656
870
|
await retainedRun.record({ phase: 'validate' });
|
|
657
|
-
const artifact = await readMemoryDirectory(
|
|
871
|
+
const artifact = await readMemoryDirectory(environment, memoryDirectory, {
|
|
658
872
|
memoryPath: '.memory',
|
|
659
873
|
sourceSessionIds: allowedSourceIds(snapshot, manifest),
|
|
660
874
|
requireSources: true,
|
|
@@ -663,42 +877,46 @@ export class LocalMemoryCoordinator {
|
|
|
663
877
|
phase: 'publish',
|
|
664
878
|
outputFingerprint: createMemorySnapshot(artifact, '.memory').fingerprint,
|
|
665
879
|
});
|
|
666
|
-
await
|
|
880
|
+
await store.commit(lease, snapshot.fingerprint, artifact, processedCheckpoints, attempt);
|
|
667
881
|
published = true;
|
|
668
|
-
await
|
|
882
|
+
await store.finishAttempt(lease, attempt, { status: 'success' }, this.#now());
|
|
669
883
|
await retainedRun.finish('completed');
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
}
|
|
674
|
-
else {
|
|
675
|
-
context.state = 'idle';
|
|
676
|
-
}
|
|
677
|
-
this.#emitStatusChange();
|
|
884
|
+
context.state = 'idle';
|
|
885
|
+
context.failedDriver = undefined;
|
|
886
|
+
context.message = droppedMessage;
|
|
678
887
|
}
|
|
679
888
|
catch (error) {
|
|
680
|
-
const leaseValid = await lease.verify();
|
|
681
|
-
const explicitlyCancelled =
|
|
682
|
-
|
|
889
|
+
const leaseValid = await lease.verify().catch(() => false);
|
|
890
|
+
const explicitlyCancelled = cancelled() || !!error && typeof error === 'object'
|
|
891
|
+
&& 'name' in error && error.name === 'AbortError';
|
|
683
892
|
const unavailableModel = error instanceof MemoryModelUnavailableError;
|
|
684
893
|
const blockedBeforeInference = retainedRun !== undefined && attempt === undefined && !explicitlyCancelled;
|
|
685
894
|
if (attempt && (leaseValid || explicitlyCancelled)) {
|
|
686
895
|
try {
|
|
687
|
-
const control = await
|
|
896
|
+
const control = await store.finishAttempt(explicitlyCancelled || unavailableModel ? undefined : lease, attempt, explicitlyCancelled || unavailableModel ? { status: 'cancelled' }
|
|
688
897
|
: { status: 'failure', reason: sanitizeMemoryDiagnostic(error) }, this.#now());
|
|
689
898
|
published ||= control.lastOutcome?.runId === attempt.runId && control.lastOutcome.status === 'success';
|
|
690
899
|
}
|
|
691
900
|
catch {
|
|
692
|
-
|
|
901
|
+
if (!explicitlyCancelled)
|
|
902
|
+
context.storageBlocked = true;
|
|
693
903
|
}
|
|
694
904
|
}
|
|
905
|
+
if (explicitlyCancelled) {
|
|
906
|
+
context.state = 'idle';
|
|
907
|
+
context.message = driver.closing || driver.detached || this.#disposed
|
|
908
|
+
? 'Memory processing was cancelled during shutdown' : 'Memory processing was cancelled';
|
|
909
|
+
await retainedRun?.finish('cancelled', error).catch(() => { });
|
|
910
|
+
return;
|
|
911
|
+
}
|
|
695
912
|
if (published) {
|
|
696
913
|
context.state = 'idle';
|
|
914
|
+
context.failedDriver = context.storageBlocked ? driver : undefined;
|
|
697
915
|
context.message = 'Memory was published, but its diagnostics could not be fully saved';
|
|
698
|
-
this.#emitStatusChange();
|
|
699
916
|
return;
|
|
700
917
|
}
|
|
701
918
|
context.state = 'error';
|
|
919
|
+
context.failedDriver = driver;
|
|
702
920
|
context.message = !leaseValid
|
|
703
921
|
? safeProcessingMessage(new Error('Memory writer lease was lost'))
|
|
704
922
|
: safeProcessingMessage(error);
|
|
@@ -712,62 +930,68 @@ export class LocalMemoryCoordinator {
|
|
|
712
930
|
context.storageBlocked = true;
|
|
713
931
|
context.message = `Memory processing is paused: ${sanitizeMemoryDiagnostic(error)}`;
|
|
714
932
|
}
|
|
715
|
-
await retainedRun?.finish(
|
|
716
|
-
:
|
|
717
|
-
: !leaseValid ? 'interrupted' : 'failed', error).catch(() => {
|
|
933
|
+
await retainedRun?.finish(unavailableModel || blockedBeforeInference ? 'blocked'
|
|
934
|
+
: !leaseValid ? 'interrupted' : 'failed', error).catch(() => {
|
|
718
935
|
context.storageBlocked = true;
|
|
719
936
|
context.message = 'Memory diagnostics could not be saved; evidence remains pending';
|
|
720
937
|
});
|
|
721
|
-
this.#emitStatusChange();
|
|
722
938
|
}
|
|
723
939
|
finally {
|
|
724
940
|
lease.compromised.removeEventListener('abort', abortOnLeaseCompromise);
|
|
725
|
-
context.abort = undefined;
|
|
726
941
|
context.attempt = undefined;
|
|
727
|
-
await lease.release()
|
|
942
|
+
await lease.release().catch(error => {
|
|
943
|
+
if (!cancelled())
|
|
944
|
+
throw error;
|
|
945
|
+
});
|
|
728
946
|
}
|
|
729
947
|
}
|
|
730
|
-
async #reconcileRunDiagnostics(context, lease) {
|
|
731
|
-
|
|
732
|
-
|
|
948
|
+
async #reconcileRunDiagnostics(context, environment, lease) {
|
|
949
|
+
const store = this.#store(context, environment);
|
|
950
|
+
const previous = await store.readControl();
|
|
951
|
+
const control = await store.reconcileAbandonedAttempt(lease, this.#now());
|
|
952
|
+
if (previous.updatedAt !== control.updatedAt)
|
|
953
|
+
this.#emitStatusChange();
|
|
733
954
|
const options = {
|
|
734
|
-
projectDirectory:
|
|
955
|
+
projectDirectory: store.projectDirectory,
|
|
735
956
|
projectKey: context.project.key,
|
|
736
957
|
sessionDirectory: join(this.#options.sessionDir ?? join(this.#options.agentDir, 'sessions'), 'memory'),
|
|
737
958
|
...(control.lastDisabledRunId ? { protectedRunIds: [control.lastDisabledRunId] } : {}),
|
|
738
959
|
};
|
|
739
960
|
if (control.lastOutcome)
|
|
740
|
-
await LocalMemoryRun.reconcile(
|
|
741
|
-
await LocalMemoryRun.reconcileAbandoned(
|
|
961
|
+
await LocalMemoryRun.reconcile(environment, options, control.lastOutcome);
|
|
962
|
+
await LocalMemoryRun.reconcileAbandoned(environment, options, [
|
|
742
963
|
...(control.activeAttempt ? [control.activeAttempt.runId] : []),
|
|
743
964
|
...(control.lastOutcome ? [control.lastOutcome.runId] : []),
|
|
744
965
|
], this.#now());
|
|
745
966
|
return control;
|
|
746
967
|
}
|
|
747
|
-
async #recover(context, signal) {
|
|
968
|
+
async #recover(context, driver, signal) {
|
|
748
969
|
let cursor;
|
|
749
970
|
let recovered = false;
|
|
750
971
|
const stableAfter = this.#options.recoveryStableMs ?? 30_000;
|
|
972
|
+
const store = this.#store(context, driver.environment);
|
|
751
973
|
const stopped = () => this.#disposed || this.#draining || !this.#enabled || signal.aborted;
|
|
752
974
|
try {
|
|
753
975
|
do {
|
|
754
|
-
const page = await
|
|
976
|
+
const page = await driver.environment.sessions.list({ scope: 'project' }, { ...(cursor ? { cursor } : {}), signal });
|
|
755
977
|
for (const info of page.items) {
|
|
756
978
|
if (stopped())
|
|
757
979
|
return false;
|
|
758
980
|
if (info.parent || this.#now() - info.updatedAt < stableAfter)
|
|
759
981
|
continue;
|
|
982
|
+
let checkpoint;
|
|
760
983
|
try {
|
|
761
|
-
|
|
762
|
-
if (stopped())
|
|
763
|
-
return false;
|
|
764
|
-
recovered = (await context.store.recordCheckpoint(checkpoint)) || recovered;
|
|
984
|
+
checkpoint = await readMemorySessionCheckpoint(driver.environment.sessions, info.id, signal);
|
|
765
985
|
}
|
|
766
986
|
catch {
|
|
767
987
|
if (stopped())
|
|
768
988
|
return false;
|
|
769
989
|
// A session that cannot be read now is skipped; its live checkpoint or a later run records it.
|
|
990
|
+
continue;
|
|
770
991
|
}
|
|
992
|
+
if (stopped())
|
|
993
|
+
return false;
|
|
994
|
+
recovered = (await store.recordCheckpoint(checkpoint)) || recovered;
|
|
771
995
|
}
|
|
772
996
|
cursor = page.nextCursor;
|
|
773
997
|
} while (cursor);
|
|
@@ -789,13 +1013,23 @@ export class LocalMemoryCoordinator {
|
|
|
789
1013
|
return true;
|
|
790
1014
|
}
|
|
791
1015
|
#emitStatusChange() {
|
|
792
|
-
for (const listener of [...this.#statusListeners]) {
|
|
1016
|
+
for (const listener of [...this.#statusListeners, ...[...this.#attachments].flatMap(attachment => [...attachment.listeners])]) {
|
|
793
1017
|
try {
|
|
794
1018
|
listener();
|
|
795
1019
|
}
|
|
796
1020
|
catch { }
|
|
797
1021
|
}
|
|
798
1022
|
}
|
|
1023
|
+
#cancelTimers(context) {
|
|
1024
|
+
this.#cancelScheduledRun(context);
|
|
1025
|
+
if (context.recoveryTimer)
|
|
1026
|
+
clearTimeout(context.recoveryTimer);
|
|
1027
|
+
if (context.writerTimer)
|
|
1028
|
+
clearTimeout(context.writerTimer);
|
|
1029
|
+
context.recoveryTimer = undefined;
|
|
1030
|
+
context.writerTimer = undefined;
|
|
1031
|
+
context.writerBlocked = false;
|
|
1032
|
+
}
|
|
799
1033
|
#cancelScheduledRun(context) {
|
|
800
1034
|
if (context.timer)
|
|
801
1035
|
clearTimeout(context.timer);
|
|
@@ -843,6 +1077,9 @@ function allowedSourceIds(snapshot, manifest) {
|
|
|
843
1077
|
...manifest.sessions.map(({ checkpoint }) => checkpoint.sessionId),
|
|
844
1078
|
])];
|
|
845
1079
|
}
|
|
1080
|
+
function droppedCheckpointsMessage(count) {
|
|
1081
|
+
return `Dropped ${count} memory checkpoint${count === 1 ? '' : 's'} that could not be read`;
|
|
1082
|
+
}
|
|
846
1083
|
function safeProcessingMessage(error) {
|
|
847
1084
|
if (error instanceof MemoryModelUnavailableError) {
|
|
848
1085
|
return 'Low-tier memory model is unavailable; evidence remains pending';
|