@felan-ai/ext-memory 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -7
- package/dist/extension.d.ts.map +1 -1
- package/dist/extension.js +30 -9
- package/dist/extension.js.map +1 -1
- package/dist/hydration.d.ts.map +1 -1
- package/dist/hydration.js +2 -5
- package/dist/hydration.js.map +1 -1
- package/dist/local/control.d.ts +3 -2
- package/dist/local/control.d.ts.map +1 -1
- package/dist/local/control.js +2 -2
- package/dist/local/control.js.map +1 -1
- package/dist/local/coordinator.d.ts +28 -51
- package/dist/local/coordinator.d.ts.map +1 -1
- package/dist/local/coordinator.js +446 -259
- package/dist/local/coordinator.js.map +1 -1
- package/dist/local/dreamer.d.ts +3 -1
- package/dist/local/dreamer.d.ts.map +1 -1
- package/dist/local/dreamer.js +3 -6
- package/dist/local/dreamer.js.map +1 -1
- package/dist/local/history.d.ts.map +1 -1
- package/dist/local/history.js.map +1 -1
- package/dist/local/lease.d.ts.map +1 -1
- package/dist/local/lease.js +12 -7
- package/dist/local/lease.js.map +1 -1
- package/dist/local/run.d.ts.map +1 -1
- package/dist/local/run.js.map +1 -1
- package/dist/local/shared.d.ts +7 -0
- package/dist/local/shared.d.ts.map +1 -0
- package/dist/local/shared.js +30 -0
- package/dist/local/shared.js.map +1 -0
- package/dist/local/store.d.ts +12 -0
- package/dist/local/store.d.ts.map +1 -1
- package/dist/local/store.js +35 -4
- package/dist/local/store.js.map +1 -1
- package/package.json +3 -3
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { removeIfPresent } from './context.js';
|
|
2
1
|
import { createDefaultLocalMemoryDreamRunner, materializeMemoryInput } from './dreamer.js';
|
|
3
2
|
import { LocalMemoryRun } from './run.js';
|
|
4
3
|
import { readMemorySessionCheckpoint } from '../session-history.js';
|
|
@@ -12,48 +11,73 @@ import {} from '@felan-ai/agent-core';
|
|
|
12
11
|
import { MemoryModelUnavailableError } from './dreamer.js';
|
|
13
12
|
import { LocalMemoryControlError } from './store.js';
|
|
14
13
|
import { sanitizeMemoryDiagnostic } from '../diagnostics.js';
|
|
15
|
-
const MEMORY_INPUT_BLOCKED_MESSAGE = '
|
|
14
|
+
const MEMORY_INPUT_BLOCKED_MESSAGE = 'Memory checkpoints are paused after a failed run; evidence remains pending';
|
|
16
15
|
const MEMORY_AUTOMATIC_UPDATE_THRESHOLD = 5;
|
|
17
16
|
const MEMORY_AUTOMATIC_INTERVAL_MS = 24 * 60 * 60 * 1_000;
|
|
18
17
|
const MEMORY_DREAM_INPUT_MAX_BYTES = 10 * 1024 * 1024;
|
|
19
18
|
const MEMORY_MAX_BATCH_SESSIONS = 8;
|
|
20
19
|
export class LocalMemoryCoordinator {
|
|
21
|
-
environment;
|
|
22
20
|
#options;
|
|
23
21
|
#contexts = new Map();
|
|
24
|
-
#
|
|
22
|
+
#attachments = new Set();
|
|
25
23
|
#statusListeners = new Set();
|
|
26
|
-
#dreamRunner;
|
|
27
24
|
#batchSize;
|
|
28
25
|
#enabled;
|
|
29
|
-
#selectedModel;
|
|
30
|
-
#scopedModels;
|
|
31
|
-
#classifier;
|
|
32
26
|
#disposed = false;
|
|
27
|
+
#disposal;
|
|
33
28
|
#draining = false;
|
|
34
|
-
#monitor;
|
|
35
|
-
#monitoring;
|
|
36
29
|
#enableVersion = 0;
|
|
37
|
-
constructor(
|
|
38
|
-
this.environment = environment;
|
|
30
|
+
constructor(options) {
|
|
39
31
|
this.#options = options;
|
|
40
32
|
this.#batchSize = boundedMemoryBatchSize(options.batchSize);
|
|
41
33
|
this.#enabled = options.enabled !== false;
|
|
42
|
-
this.#selectedModel = options.selectedModel;
|
|
43
|
-
this.#scopedModels = options.scopedModels === undefined ? undefined : [...options.scopedModels];
|
|
44
|
-
this.#classifier = options.classifier;
|
|
45
|
-
this.#dreamRunner = options.dreamRunner ?? createDefaultLocalMemoryDreamRunner(this.environment);
|
|
46
|
-
if (this.#enabled)
|
|
47
|
-
this.#startMonitor();
|
|
48
34
|
}
|
|
49
|
-
|
|
50
|
-
|
|
35
|
+
attach(options) {
|
|
36
|
+
if (this.#disposed)
|
|
37
|
+
throw new Error('Local memory coordinator is closed');
|
|
38
|
+
const state = {
|
|
39
|
+
...options,
|
|
40
|
+
selectedModel: options.selectedModel,
|
|
41
|
+
scopedModels: options.scopedModels === undefined ? undefined : [...options.scopedModels],
|
|
42
|
+
classifier: options.classifier,
|
|
43
|
+
projections: new Map(),
|
|
44
|
+
listeners: new Set(),
|
|
45
|
+
draining: false,
|
|
46
|
+
closing: false,
|
|
47
|
+
detached: false,
|
|
48
|
+
detaching: undefined,
|
|
49
|
+
};
|
|
50
|
+
this.#attachments.add(state);
|
|
51
|
+
const attachment = {
|
|
52
|
+
createSessionHost: options => new LocalMemorySessionHost(attachment, options),
|
|
53
|
+
readCurrent: (cwd, root) => this.#readCurrent(state, cwd, root),
|
|
54
|
+
recordCheckpoint: (cwd, checkpoint) => this.#recordCheckpoint(state, cwd, checkpoint),
|
|
55
|
+
status: cwd => this.#status(state, cwd),
|
|
56
|
+
runNow: cwd => this.#runNow(state, cwd),
|
|
57
|
+
retryProject: cwd => this.#retryProject(state, cwd),
|
|
58
|
+
canonicalDirectory: async (cwd) => this.#store(await this.#context(state, cwd), state.environment).currentDirectory,
|
|
59
|
+
setSelectedModel: model => this.#setModelSelection(state, model, state.scopedModels, state.classifier),
|
|
60
|
+
setModelSelection: (model, scopedModels, classifier) => this.#setModelSelection(state, model, scopedModels, classifier),
|
|
61
|
+
subscribeStatusChanges: listener => {
|
|
62
|
+
if (state.detached || this.#disposed)
|
|
63
|
+
return () => { };
|
|
64
|
+
state.listeners.add(listener);
|
|
65
|
+
return () => { state.listeners.delete(listener); };
|
|
66
|
+
},
|
|
67
|
+
busy: () => [...this.#contexts.values()].some(context => context.driver === state || context.recoveryDriver === state),
|
|
68
|
+
settle: mode => this.#settleAttachment(state, mode),
|
|
69
|
+
detach: () => {
|
|
70
|
+
state.detaching ??= this.#detach(state);
|
|
71
|
+
return state.detaching;
|
|
72
|
+
},
|
|
73
|
+
};
|
|
74
|
+
return attachment;
|
|
51
75
|
}
|
|
52
76
|
isEnabled() {
|
|
53
77
|
return this.#enabled;
|
|
54
78
|
}
|
|
55
79
|
hasActiveWork() {
|
|
56
|
-
return
|
|
80
|
+
return [...this.#contexts.values()].some(context => !!context.run || !!context.ready || !!context.recovery || context.followUps.size > 0);
|
|
57
81
|
}
|
|
58
82
|
subscribeStatusChanges(listener) {
|
|
59
83
|
if (this.#disposed)
|
|
@@ -61,34 +85,28 @@ export class LocalMemoryCoordinator {
|
|
|
61
85
|
this.#statusListeners.add(listener);
|
|
62
86
|
return () => this.#statusListeners.delete(listener);
|
|
63
87
|
}
|
|
64
|
-
setModelSelection(model, scopedModels, classifier) {
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
this.#selectedModel = model;
|
|
88
|
+
#setModelSelection(attachment, model, scopedModels, classifier) {
|
|
89
|
+
if (attachment.detached || this.#disposed)
|
|
90
|
+
return;
|
|
91
|
+
attachment.selectedModel = model;
|
|
92
|
+
attachment.scopedModels = scopedModels === undefined ? undefined : [...scopedModels];
|
|
93
|
+
attachment.classifier = classifier;
|
|
71
94
|
for (const context of this.#contexts.values()) {
|
|
95
|
+
if (!context.attachments.has(attachment))
|
|
96
|
+
continue;
|
|
72
97
|
context.blocked.clear();
|
|
73
98
|
if (context.initialized)
|
|
74
|
-
void this.#trackScheduleIfPending(context);
|
|
99
|
+
void this.#trackScheduleIfPending(context, attachment.environment);
|
|
75
100
|
}
|
|
76
101
|
}
|
|
77
102
|
setEnabled(enabled) {
|
|
78
103
|
this.#enableVersion += 1;
|
|
79
|
-
enabled = enabled && !this.#options.unavailableReason;
|
|
80
104
|
this.#enabled = enabled;
|
|
81
|
-
if (enabled)
|
|
82
|
-
this.#startMonitor();
|
|
83
|
-
else if (this.#monitor) {
|
|
84
|
-
clearInterval(this.#monitor);
|
|
85
|
-
this.#monitor = undefined;
|
|
86
|
-
}
|
|
87
105
|
for (const context of this.#contexts.values()) {
|
|
88
106
|
if (!enabled)
|
|
89
107
|
context.recoveryAbort?.abort();
|
|
90
108
|
if (!enabled)
|
|
91
|
-
this.#
|
|
109
|
+
this.#cancelTimers(context);
|
|
92
110
|
if (!enabled)
|
|
93
111
|
this.#cancelContext(context, 'Memory processing was disabled', true);
|
|
94
112
|
if (enabled) {
|
|
@@ -108,26 +126,27 @@ export class LocalMemoryCoordinator {
|
|
|
108
126
|
}
|
|
109
127
|
this.#emitStatusChange();
|
|
110
128
|
}
|
|
111
|
-
async status(cwd) {
|
|
129
|
+
async #status(attachment, cwd) {
|
|
112
130
|
try {
|
|
113
|
-
const context = await this.#context(cwd);
|
|
114
|
-
const
|
|
115
|
-
const
|
|
131
|
+
const context = await this.#context(attachment, cwd);
|
|
132
|
+
const store = this.#store(context, attachment.environment);
|
|
133
|
+
const state = await store.status();
|
|
134
|
+
const control = await store.readControl();
|
|
116
135
|
const processed = Object.values(state.processed);
|
|
117
136
|
const lastProcessedAt = processed
|
|
118
137
|
.map((entry) => entry.processedAt)
|
|
119
138
|
.sort()
|
|
120
139
|
.at(-1);
|
|
121
140
|
return {
|
|
122
|
-
enabled: this.#enabled && !control.autoDisabled,
|
|
123
|
-
state: !this.#enabled || control.autoDisabled ? 'disabled'
|
|
141
|
+
enabled: this.#enabled && attachment.scheduling && !control.autoDisabled,
|
|
142
|
+
state: !this.#enabled || !attachment.scheduling || control.autoDisabled ? 'disabled'
|
|
124
143
|
: control.nextRetryAt && Date.parse(control.nextRetryAt) > this.#now() ? 'scheduled'
|
|
125
144
|
: context.state === 'error' ? 'error'
|
|
126
145
|
: control.activeAttempt ? 'processing' : context.state,
|
|
127
146
|
pendingCheckpoints: Object.keys(state.pending).length,
|
|
128
147
|
memoryFingerprint: state.memoryFingerprint,
|
|
129
148
|
...(lastProcessedAt === undefined ? {} : { lastProcessedAt }),
|
|
130
|
-
...(context.message ||
|
|
149
|
+
...(context.message || attachment.unavailableReason ? { message: context.message ?? attachment.unavailableReason } : {}),
|
|
131
150
|
consecutiveFailures: control.consecutiveFailures,
|
|
132
151
|
...(control.nextRetryAt ? { nextRetryAt: control.nextRetryAt } : {}),
|
|
133
152
|
...(control.autoDisabled ? { autoDisabled: control.autoDisabled } : {}),
|
|
@@ -138,37 +157,39 @@ export class LocalMemoryCoordinator {
|
|
|
138
157
|
}
|
|
139
158
|
catch (error) {
|
|
140
159
|
return {
|
|
141
|
-
enabled: this.#enabled,
|
|
142
|
-
state: this.#enabled ? 'error' : 'disabled',
|
|
160
|
+
enabled: this.#enabled && attachment.scheduling,
|
|
161
|
+
state: this.#enabled && attachment.scheduling ? 'error' : 'disabled',
|
|
143
162
|
pendingCheckpoints: 0,
|
|
144
163
|
message: error instanceof LocalMemoryControlError ? error.message : 'Local memory storage is unavailable',
|
|
145
164
|
};
|
|
146
165
|
}
|
|
147
166
|
}
|
|
148
|
-
async readCurrent(cwd, sessionStorageRoot) {
|
|
149
|
-
const context = await this.#context(cwd);
|
|
150
|
-
const
|
|
167
|
+
async #readCurrent(attachment, cwd, sessionStorageRoot) {
|
|
168
|
+
const context = await this.#context(attachment, cwd);
|
|
169
|
+
const store = this.#store(context, attachment.environment);
|
|
170
|
+
const snapshot = await store.readCurrent();
|
|
151
171
|
const projectionKey = `${context.project.key}:${sessionStorageRoot}`;
|
|
152
|
-
const memoryPath = join(sessionStorageRoot,
|
|
153
|
-
if (
|
|
154
|
-
const projection = await
|
|
155
|
-
|
|
172
|
+
const memoryPath = join(sessionStorageRoot, store.projectionName);
|
|
173
|
+
if (attachment.projections.get(projectionKey) !== snapshot.fingerprint) {
|
|
174
|
+
const projection = await store.projectTo(sessionStorageRoot, snapshot);
|
|
175
|
+
if (!attachment.detached)
|
|
176
|
+
attachment.projections.set(projectionKey, snapshot.fingerprint);
|
|
156
177
|
return projection;
|
|
157
178
|
}
|
|
158
179
|
return createMemoryProjectionSnapshot(snapshot, memoryPath);
|
|
159
180
|
}
|
|
160
|
-
async recordCheckpoint(cwd, checkpoint) {
|
|
161
|
-
const context = await this.#context(cwd);
|
|
162
|
-
if (await context.
|
|
181
|
+
async #recordCheckpoint(attachment, cwd, checkpoint) {
|
|
182
|
+
const context = await this.#context(attachment, cwd);
|
|
183
|
+
if (await this.#store(context, attachment.environment).recordCheckpoint(checkpoint)) {
|
|
163
184
|
context.blocked.delete(checkpoint.sessionId);
|
|
164
|
-
await this.#trackScheduleIfPending(context);
|
|
185
|
+
await this.#trackScheduleIfPending(context, attachment.environment);
|
|
165
186
|
this.#emitStatusChange();
|
|
166
187
|
}
|
|
167
188
|
}
|
|
168
|
-
async runNow(cwd) {
|
|
169
|
-
if (!this.#
|
|
170
|
-
return this
|
|
171
|
-
const context = await this.#context(cwd);
|
|
189
|
+
async #runNow(attachment, cwd) {
|
|
190
|
+
if (!this.#eligible(attachment))
|
|
191
|
+
return this.#status(attachment, cwd);
|
|
192
|
+
const context = await this.#context(attachment, cwd);
|
|
172
193
|
this.#cancelScheduledRun(context);
|
|
173
194
|
while (context.run || context.followUps.size > 0) {
|
|
174
195
|
await Promise.allSettled([
|
|
@@ -178,61 +199,114 @@ export class LocalMemoryCoordinator {
|
|
|
178
199
|
this.#cancelScheduledRun(context);
|
|
179
200
|
}
|
|
180
201
|
context.blocked.clear();
|
|
181
|
-
|
|
182
|
-
context.
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
}
|
|
186
|
-
finally {
|
|
187
|
-
if (context.run === run)
|
|
188
|
-
context.run = undefined;
|
|
189
|
-
await this.#trackScheduleIfPending(context);
|
|
190
|
-
}
|
|
191
|
-
return this.status(cwd);
|
|
202
|
+
context.storageBlocked = false;
|
|
203
|
+
context.message = undefined;
|
|
204
|
+
await this.#launchRun(context, true, attachment);
|
|
205
|
+
return this.#status(attachment, cwd);
|
|
192
206
|
}
|
|
193
|
-
async retryProject(cwd) {
|
|
194
|
-
if (!this.#
|
|
195
|
-
return this
|
|
196
|
-
const context = await this.#context(cwd);
|
|
197
|
-
if (!this.#
|
|
198
|
-
return this
|
|
207
|
+
async #retryProject(attachment, cwd) {
|
|
208
|
+
if (!this.#eligible(attachment))
|
|
209
|
+
return this.#status(attachment, cwd);
|
|
210
|
+
const context = await this.#context(attachment, cwd);
|
|
211
|
+
if (!this.#eligible(attachment))
|
|
212
|
+
return this.#status(attachment, cwd);
|
|
199
213
|
const enableVersion = this.#enableVersion;
|
|
200
214
|
this.#cancelScheduledRun(context);
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
}
|
|
204
|
-
catch (error) {
|
|
205
|
-
throw error;
|
|
206
|
-
}
|
|
215
|
+
await this.#store(context, attachment.environment).resetProjectControl(this.#now(), () => this.#eligible(attachment) && this.#enableVersion === enableVersion);
|
|
216
|
+
this.#emitStatusChange();
|
|
207
217
|
context.abort?.abort(new Error('Memory processing was explicitly retried'));
|
|
208
218
|
if (context.run)
|
|
209
219
|
await context.run;
|
|
210
220
|
context.blocked.clear();
|
|
211
221
|
context.storageBlocked = false;
|
|
212
222
|
context.message = undefined;
|
|
213
|
-
await this
|
|
214
|
-
return this
|
|
223
|
+
await this.#runNow(attachment, cwd);
|
|
224
|
+
return this.#status(attachment, cwd);
|
|
225
|
+
}
|
|
226
|
+
#eligible(attachment) {
|
|
227
|
+
return this.#enabled && !this.#disposed && !this.#draining
|
|
228
|
+
&& attachment.scheduling && !attachment.draining && !attachment.closing && !attachment.detached;
|
|
229
|
+
}
|
|
230
|
+
#driver(context, preferred) {
|
|
231
|
+
if (preferred && context.attachments.has(preferred) && this.#eligible(preferred))
|
|
232
|
+
return preferred;
|
|
233
|
+
return [...this.#attachments].reverse().find(attachment => context.attachments.has(attachment) && this.#eligible(attachment));
|
|
234
|
+
}
|
|
235
|
+
#liveMember(context) {
|
|
236
|
+
return [...this.#attachments].reverse().find(attachment => context.attachments.has(attachment)
|
|
237
|
+
&& attachment.scheduling && !attachment.draining && !attachment.closing && !attachment.detached);
|
|
238
|
+
}
|
|
239
|
+
#runEnvironment(context, driver) {
|
|
240
|
+
// The daemon seals closing runtimes outside their own cleanup async context.
|
|
241
|
+
const current = () => (driver.closing || driver.detached ? this.#liveMember(context) ?? driver : driver).environment;
|
|
242
|
+
const environment = {
|
|
243
|
+
get runtime() { return current().runtime; },
|
|
244
|
+
get sessions() { return current().sessions; },
|
|
245
|
+
};
|
|
246
|
+
Object.defineProperty(environment, 'openWorker', { get: () => current().openWorker });
|
|
247
|
+
return environment;
|
|
248
|
+
}
|
|
249
|
+
async #settleAttachment(attachment, mode) {
|
|
250
|
+
attachment.draining = true;
|
|
251
|
+
if (mode === 'force')
|
|
252
|
+
attachment.closing = true;
|
|
253
|
+
for (const context of this.#contexts.values()) {
|
|
254
|
+
if (context.recoveryDriver === attachment)
|
|
255
|
+
context.recoveryAbort?.abort();
|
|
256
|
+
if (mode === 'force' && context.driver === attachment) {
|
|
257
|
+
this.#cancelContext(context, 'Memory processing was cancelled during shutdown', false);
|
|
258
|
+
}
|
|
259
|
+
if (!this.#driver(context))
|
|
260
|
+
this.#cancelTimers(context);
|
|
261
|
+
}
|
|
262
|
+
while (true) {
|
|
263
|
+
const work = [...this.#contexts.values()].flatMap(context => [
|
|
264
|
+
...(context.driver === attachment ? [...(context.run ? [context.run] : []), ...context.followUps] : []),
|
|
265
|
+
...(context.recoveryDriver === attachment && context.recovery ? [context.recovery] : []),
|
|
266
|
+
]);
|
|
267
|
+
if (!work.length)
|
|
268
|
+
break;
|
|
269
|
+
await Promise.allSettled(work);
|
|
270
|
+
}
|
|
271
|
+
for (const context of this.#contexts.values()) {
|
|
272
|
+
if (mode === 'force' && context.failedDriver === attachment) {
|
|
273
|
+
context.storageBlocked = false;
|
|
274
|
+
context.blocked.clear();
|
|
275
|
+
if (context.state === 'error' || context.state === 'blocked')
|
|
276
|
+
context.state = 'idle';
|
|
277
|
+
context.message = undefined;
|
|
278
|
+
context.failedDriver = undefined;
|
|
279
|
+
}
|
|
280
|
+
if (!context.attachments.has(attachment) || !context.initialized)
|
|
281
|
+
continue;
|
|
282
|
+
this.#startRecovery(context);
|
|
283
|
+
void this.#trackScheduleIfPending(context);
|
|
284
|
+
}
|
|
215
285
|
}
|
|
216
|
-
async
|
|
217
|
-
|
|
218
|
-
|
|
286
|
+
async #detach(attachment) {
|
|
287
|
+
attachment.closing = true;
|
|
288
|
+
await this.#settleAttachment(attachment, 'force');
|
|
289
|
+
attachment.detached = true;
|
|
290
|
+
this.#attachments.delete(attachment);
|
|
291
|
+
attachment.listeners.clear();
|
|
292
|
+
attachment.projections.clear();
|
|
293
|
+
for (const context of this.#contexts.values()) {
|
|
294
|
+
context.attachments.delete(attachment);
|
|
295
|
+
if (!this.#driver(context))
|
|
296
|
+
this.#cancelTimers(context);
|
|
297
|
+
}
|
|
219
298
|
}
|
|
220
299
|
async drain() {
|
|
221
300
|
this.#draining = true;
|
|
222
|
-
if (this.#monitor)
|
|
223
|
-
clearInterval(this.#monitor);
|
|
224
|
-
this.#monitor = undefined;
|
|
225
301
|
for (const context of this.#contexts.values()) {
|
|
226
302
|
context.recoveryAbort?.abort();
|
|
227
|
-
this.#
|
|
303
|
+
this.#cancelTimers(context);
|
|
228
304
|
}
|
|
229
305
|
while (true) {
|
|
230
306
|
const work = [...this.#contexts.values()].flatMap(context => [
|
|
231
307
|
...(context.run ? [context.run] : []), ...(context.ready ? [context.ready] : []),
|
|
232
308
|
...(context.recovery ? [context.recovery] : []), ...context.followUps,
|
|
233
309
|
]);
|
|
234
|
-
if (this.#monitoring)
|
|
235
|
-
work.push(this.#monitoring);
|
|
236
310
|
if (!work.length)
|
|
237
311
|
return;
|
|
238
312
|
const results = await Promise.allSettled(work);
|
|
@@ -241,103 +315,121 @@ export class LocalMemoryCoordinator {
|
|
|
241
315
|
throw new AggregateError(failures, 'Memory drain could not settle all work.');
|
|
242
316
|
}
|
|
243
317
|
}
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
318
|
+
dispose() {
|
|
319
|
+
this.#disposal ??= this.#dispose();
|
|
320
|
+
return this.#disposal;
|
|
321
|
+
}
|
|
322
|
+
async #dispose() {
|
|
247
323
|
this.#disposed = true;
|
|
248
|
-
if (this.#monitor)
|
|
249
|
-
clearInterval(this.#monitor);
|
|
250
|
-
this.#monitor = undefined;
|
|
251
324
|
for (const context of this.#contexts.values()) {
|
|
252
325
|
context.recoveryAbort?.abort();
|
|
253
|
-
this.#
|
|
326
|
+
this.#cancelTimers(context);
|
|
254
327
|
this.#cancelContext(context, 'Memory processing was cancelled during shutdown', false);
|
|
255
328
|
}
|
|
256
329
|
while (true) {
|
|
257
|
-
const work = [...this.#contexts.values()].flatMap(
|
|
330
|
+
const work = [...this.#contexts.values()].flatMap(context => [
|
|
258
331
|
...(context.run ? [context.run] : []),
|
|
259
332
|
...(context.ready ? [context.ready] : []),
|
|
260
333
|
...(context.recovery ? [context.recovery] : []),
|
|
261
334
|
...context.followUps,
|
|
262
335
|
]);
|
|
263
|
-
if (
|
|
264
|
-
work.push(this.#monitoring);
|
|
265
|
-
if (work.length === 0)
|
|
336
|
+
if (!work.length)
|
|
266
337
|
break;
|
|
267
338
|
await Promise.allSettled(work);
|
|
268
339
|
}
|
|
340
|
+
for (const attachment of this.#attachments) {
|
|
341
|
+
attachment.detached = true;
|
|
342
|
+
attachment.listeners.clear();
|
|
343
|
+
attachment.projections.clear();
|
|
344
|
+
}
|
|
345
|
+
this.#attachments.clear();
|
|
269
346
|
this.#statusListeners.clear();
|
|
270
347
|
}
|
|
271
|
-
async #context(cwd) {
|
|
272
|
-
if (this.#disposed)
|
|
348
|
+
async #context(attachment, cwd) {
|
|
349
|
+
if (this.#disposed || attachment.detached)
|
|
273
350
|
throw new Error('Local memory coordinator is closed');
|
|
274
|
-
const project = await resolveLocalMemoryProject(
|
|
275
|
-
if (this.#disposed)
|
|
351
|
+
const project = await resolveLocalMemoryProject(attachment.environment, cwd);
|
|
352
|
+
if (this.#disposed || attachment.detached)
|
|
276
353
|
throw new Error('Local memory coordinator is closed');
|
|
277
354
|
const existing = this.#contexts.get(project.key);
|
|
278
355
|
if (existing) {
|
|
279
|
-
|
|
356
|
+
existing.attachments.add(attachment);
|
|
357
|
+
await this.#initialize(existing, attachment);
|
|
280
358
|
return existing;
|
|
281
359
|
}
|
|
282
|
-
const store = new LocalMemoryStore(this.environment, this.#options.agentDir, project, {
|
|
283
|
-
...(this.#options.retryDelaysMs ? { retryDelaysMs: this.#options.retryDelaysMs } : {}),
|
|
284
|
-
});
|
|
285
360
|
const context = {
|
|
286
361
|
project,
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
362
|
+
stores: new WeakMap(),
|
|
363
|
+
attachments: new Set([attachment]),
|
|
364
|
+
driver: undefined,
|
|
365
|
+
failedDriver: undefined,
|
|
366
|
+
recoveryDriver: undefined,
|
|
290
367
|
ready: undefined,
|
|
291
368
|
initialized: false,
|
|
292
369
|
recovery: undefined,
|
|
293
370
|
recoveryAbort: undefined,
|
|
294
371
|
recoveryRetryAt: undefined,
|
|
372
|
+
recoveryTimer: undefined,
|
|
295
373
|
recoveryCompleted: false,
|
|
296
374
|
diagnosticsReconciled: false,
|
|
297
375
|
timer: undefined,
|
|
298
376
|
timerDueAt: undefined,
|
|
299
377
|
run: undefined,
|
|
378
|
+
runEnvironment: undefined,
|
|
300
379
|
followUps: new Set(),
|
|
301
380
|
abort: undefined,
|
|
302
381
|
attempt: undefined,
|
|
303
382
|
storageBlocked: false,
|
|
304
383
|
writerBlocked: false,
|
|
305
|
-
|
|
384
|
+
writerTimer: undefined,
|
|
306
385
|
blocked: new Map(),
|
|
307
386
|
state: this.#enabled ? 'idle' : 'disabled',
|
|
308
387
|
message: undefined,
|
|
309
388
|
};
|
|
310
389
|
this.#contexts.set(project.key, context);
|
|
311
|
-
await this.#initialize(context);
|
|
390
|
+
await this.#initialize(context, attachment);
|
|
312
391
|
return context;
|
|
313
392
|
}
|
|
314
|
-
|
|
393
|
+
#store(context, environment) {
|
|
394
|
+
let store = context.stores.get(environment);
|
|
395
|
+
if (!store) {
|
|
396
|
+
store = new LocalMemoryStore(environment, this.#options.agentDir, context.project, {
|
|
397
|
+
...(this.#options.retryDelaysMs ? { retryDelaysMs: this.#options.retryDelaysMs } : {}),
|
|
398
|
+
});
|
|
399
|
+
context.stores.set(environment, store);
|
|
400
|
+
}
|
|
401
|
+
return store;
|
|
402
|
+
}
|
|
403
|
+
async #initialize(context, attachment) {
|
|
315
404
|
if (context.ready) {
|
|
316
405
|
await context.ready;
|
|
317
406
|
return;
|
|
318
407
|
}
|
|
319
408
|
if (context.initialized && context.diagnosticsReconciled) {
|
|
320
409
|
this.#startRecovery(context);
|
|
410
|
+
void this.#trackScheduleIfPending(context, attachment.environment);
|
|
321
411
|
return;
|
|
322
412
|
}
|
|
323
|
-
const ready = this.#initializeContext(context).finally(() => {
|
|
413
|
+
const ready = this.#initializeContext(context, attachment).finally(() => {
|
|
324
414
|
if (context.ready === ready)
|
|
325
415
|
context.ready = undefined;
|
|
326
416
|
});
|
|
327
417
|
context.ready = ready;
|
|
328
418
|
await ready;
|
|
329
419
|
}
|
|
330
|
-
async #initializeContext(context) {
|
|
420
|
+
async #initializeContext(context, attachment) {
|
|
421
|
+
const environment = attachment.environment;
|
|
422
|
+
const store = this.#store(context, environment);
|
|
331
423
|
if (!context.initialized) {
|
|
332
|
-
await
|
|
424
|
+
await store.initialize(this.#now()).then(() => {
|
|
333
425
|
context.initialized = true;
|
|
334
426
|
context.state = this.#enabled ? 'idle' : 'disabled';
|
|
335
427
|
context.message = undefined;
|
|
336
428
|
}).catch(async (error) => {
|
|
337
429
|
if (!(error instanceof LocalMemoryControlError))
|
|
338
430
|
throw error;
|
|
339
|
-
const current = await
|
|
340
|
-
if (current.fingerprint !== (await
|
|
431
|
+
const current = await store.readCurrent();
|
|
432
|
+
if (current.fingerprint !== (await store.status()).memoryFingerprint)
|
|
341
433
|
throw error;
|
|
342
434
|
context.message = error.message;
|
|
343
435
|
});
|
|
@@ -346,11 +438,16 @@ export class LocalMemoryCoordinator {
|
|
|
346
438
|
return;
|
|
347
439
|
if (!context.diagnosticsReconciled) {
|
|
348
440
|
try {
|
|
349
|
-
const lease = await acquireLocalMemoryLease(
|
|
441
|
+
const lease = await acquireLocalMemoryLease(environment, store.projectDirectory, this.#options.leaseOptions);
|
|
350
442
|
if (lease) {
|
|
351
443
|
try {
|
|
352
|
-
await this.#reconcileRunDiagnostics(context, lease);
|
|
444
|
+
await this.#reconcileRunDiagnostics(context, environment, lease);
|
|
353
445
|
context.diagnosticsReconciled = true;
|
|
446
|
+
if (context.storageBlocked) {
|
|
447
|
+
context.storageBlocked = false;
|
|
448
|
+
context.failedDriver = undefined;
|
|
449
|
+
context.message = undefined;
|
|
450
|
+
}
|
|
354
451
|
}
|
|
355
452
|
finally {
|
|
356
453
|
await lease.release();
|
|
@@ -358,24 +455,34 @@ export class LocalMemoryCoordinator {
|
|
|
358
455
|
}
|
|
359
456
|
}
|
|
360
457
|
catch (error) {
|
|
361
|
-
context.diagnosticsReconciled = true;
|
|
362
458
|
context.storageBlocked = true;
|
|
459
|
+
context.failedDriver = attachment;
|
|
363
460
|
context.message = `Memory diagnostics recovery is blocked: ${sanitizeMemoryDiagnostic(error)}`;
|
|
364
461
|
}
|
|
365
462
|
}
|
|
366
463
|
if (this.#disposed)
|
|
367
464
|
return;
|
|
368
465
|
if (this.#enabled)
|
|
369
|
-
void this.#trackScheduleIfPending(context);
|
|
466
|
+
void this.#trackScheduleIfPending(context, environment);
|
|
370
467
|
this.#startRecovery(context);
|
|
371
468
|
}
|
|
372
469
|
#startRecovery(context) {
|
|
373
|
-
if (this.#
|
|
374
|
-
|
|
375
|
-
|
|
470
|
+
if (this.#options.recover === false || context.recoveryCompleted || context.recovery)
|
|
471
|
+
return;
|
|
472
|
+
const driver = this.#driver(context);
|
|
473
|
+
if (!driver)
|
|
474
|
+
return;
|
|
475
|
+
if (context.recoveryRetryAt !== undefined && this.#now() < context.recoveryRetryAt) {
|
|
476
|
+
this.#scheduleRecoveryRetry(context);
|
|
376
477
|
return;
|
|
478
|
+
}
|
|
479
|
+
if (context.recoveryTimer)
|
|
480
|
+
clearTimeout(context.recoveryTimer);
|
|
481
|
+
context.recoveryTimer = undefined;
|
|
482
|
+
context.recoveryDriver = driver;
|
|
377
483
|
const abort = new AbortController();
|
|
378
|
-
|
|
484
|
+
context.recoveryAbort = abort;
|
|
485
|
+
const recovery = this.#recover(context, driver, abort.signal).then(completed => {
|
|
379
486
|
if (completed)
|
|
380
487
|
context.recoveryCompleted = true;
|
|
381
488
|
}).catch(() => {
|
|
@@ -384,71 +491,70 @@ export class LocalMemoryCoordinator {
|
|
|
384
491
|
if (context.recovery === recovery) {
|
|
385
492
|
context.recovery = undefined;
|
|
386
493
|
context.recoveryAbort = undefined;
|
|
494
|
+
context.recoveryDriver = undefined;
|
|
387
495
|
}
|
|
496
|
+
if (abort.signal.aborted)
|
|
497
|
+
this.#startRecovery(context);
|
|
498
|
+
else
|
|
499
|
+
this.#scheduleRecoveryRetry(context);
|
|
500
|
+
this.#emitStatusChange();
|
|
388
501
|
});
|
|
389
502
|
context.recovery = recovery;
|
|
390
|
-
context.recoveryAbort = abort;
|
|
391
503
|
}
|
|
392
|
-
#
|
|
393
|
-
|
|
394
|
-
if (this.#disposed || this.#draining || this.#monitor)
|
|
504
|
+
#scheduleRecoveryRetry(context) {
|
|
505
|
+
if (context.recoveryTimer || context.recoveryCompleted || context.recoveryRetryAt === undefined || !this.#driver(context))
|
|
395
506
|
return;
|
|
396
|
-
const
|
|
397
|
-
|
|
398
|
-
throw new Error('Memory monitor interval must be positive');
|
|
399
|
-
this.#monitor = setInterval(() => {
|
|
400
|
-
if (this.#monitoring)
|
|
507
|
+
const timer = setTimeout(() => {
|
|
508
|
+
if (context.recoveryTimer !== timer)
|
|
401
509
|
return;
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
this.#
|
|
510
|
+
context.recoveryTimer = undefined;
|
|
511
|
+
this.#startRecovery(context);
|
|
512
|
+
}, Math.max(0, context.recoveryRetryAt - this.#now()));
|
|
513
|
+
context.recoveryTimer = timer;
|
|
514
|
+
timer.unref?.();
|
|
405
515
|
}
|
|
406
|
-
|
|
407
|
-
if (
|
|
516
|
+
#scheduleWriterRetry(context) {
|
|
517
|
+
if (context.writerTimer || !this.#driver(context))
|
|
408
518
|
return;
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
}
|
|
419
|
-
context.writerBlocked = false;
|
|
420
|
-
if (context.initialized)
|
|
421
|
-
await this.#trackScheduleIfPending(context);
|
|
422
|
-
const signature = JSON.stringify([this.#enabled, context.state, context.message, control.updatedAt]);
|
|
423
|
-
if (signature !== context.statusSignature) {
|
|
424
|
-
context.statusSignature = signature;
|
|
425
|
-
this.#emitStatusChange();
|
|
426
|
-
}
|
|
427
|
-
}
|
|
428
|
-
catch (error) {
|
|
429
|
-
context.state = 'error';
|
|
430
|
-
context.message = sanitizeMemoryDiagnostic(error);
|
|
431
|
-
context.abort?.abort(error);
|
|
432
|
-
this.#emitStatusChange();
|
|
519
|
+
context.writerBlocked = true;
|
|
520
|
+
const timer = setTimeout(() => {
|
|
521
|
+
if (context.writerTimer !== timer)
|
|
522
|
+
return;
|
|
523
|
+
context.writerTimer = undefined;
|
|
524
|
+
context.writerBlocked = false;
|
|
525
|
+
if (context.run) {
|
|
526
|
+
void this.#trackScheduleIfPending(context);
|
|
527
|
+
return;
|
|
433
528
|
}
|
|
434
|
-
|
|
529
|
+
context.diagnosticsReconciled = false;
|
|
530
|
+
const driver = this.#driver(context);
|
|
531
|
+
if (driver)
|
|
532
|
+
void this.#initialize(context, driver).catch(() => { });
|
|
533
|
+
}, this.#options.writerRetryMs ?? 30_000);
|
|
534
|
+
context.writerTimer = timer;
|
|
535
|
+
timer.unref?.();
|
|
435
536
|
}
|
|
537
|
+
#now() { return this.#options.now?.() ?? Date.now(); }
|
|
436
538
|
#cancelContext(context, reason, cancelPersisted) {
|
|
437
539
|
const alreadyAborted = context.abort?.signal.aborted ?? false;
|
|
438
540
|
context.abort?.abort(new Error(reason));
|
|
439
541
|
if (alreadyAborted || !context.initialized || (!cancelPersisted && !context.attempt))
|
|
440
542
|
return;
|
|
543
|
+
const environment = context.runEnvironment ?? this.#liveMember(context)?.environment;
|
|
544
|
+
if (!environment)
|
|
545
|
+
return;
|
|
546
|
+
const store = this.#store(context, environment);
|
|
441
547
|
const cancellation = cancelPersisted
|
|
442
|
-
?
|
|
443
|
-
:
|
|
548
|
+
? store.cancelActiveAttempt(this.#now())
|
|
549
|
+
: store.finishAttempt(undefined, context.attempt, { status: 'cancelled', reason }, this.#now());
|
|
444
550
|
const followUp = cancellation
|
|
445
|
-
.then(() => { })
|
|
551
|
+
.then(() => { this.#emitStatusChange(); })
|
|
446
552
|
.catch(() => { })
|
|
447
553
|
.finally(() => { context.followUps.delete(followUp); });
|
|
448
554
|
context.followUps.add(followUp);
|
|
449
555
|
}
|
|
450
556
|
async #schedule(context, dueAt) {
|
|
451
|
-
if (
|
|
557
|
+
if (!this.#driver(context) || context.run || context.storageBlocked)
|
|
452
558
|
return;
|
|
453
559
|
if (context.timer && context.timerDueAt === dueAt)
|
|
454
560
|
return;
|
|
@@ -460,28 +566,26 @@ export class LocalMemoryCoordinator {
|
|
|
460
566
|
return;
|
|
461
567
|
context.timer = undefined;
|
|
462
568
|
context.timerDueAt = undefined;
|
|
463
|
-
|
|
464
|
-
context.state = 'error';
|
|
465
|
-
context.storageBlocked = true;
|
|
466
|
-
context.message = sanitizeMemoryDiagnostic(error);
|
|
467
|
-
this.#emitStatusChange();
|
|
468
|
-
}).finally(async () => {
|
|
469
|
-
context.run = undefined;
|
|
470
|
-
await this.#trackScheduleIfPending(context);
|
|
471
|
-
});
|
|
472
|
-
context.run = run;
|
|
569
|
+
void this.#launchRun(context)?.catch(() => { });
|
|
473
570
|
}, delay);
|
|
474
571
|
context.timer = timer;
|
|
475
572
|
context.timerDueAt = dueAt;
|
|
476
573
|
timer.unref?.();
|
|
477
574
|
}
|
|
478
|
-
async #scheduleIfPending(context) {
|
|
479
|
-
|
|
575
|
+
async #scheduleIfPending(context, environment) {
|
|
576
|
+
const driver = this.#driver(context);
|
|
577
|
+
if (!driver) {
|
|
578
|
+
this.#cancelTimers(context);
|
|
480
579
|
return;
|
|
580
|
+
}
|
|
581
|
+
// Callers capture their environment before awaiting; a member sealed since then cannot read the store.
|
|
582
|
+
const owner = environment && [...context.attachments].find(attachment => attachment.environment === environment);
|
|
583
|
+
const source = owner && (owner.closing || owner.detached) ? (this.#liveMember(context) ?? owner).environment : environment;
|
|
481
584
|
try {
|
|
482
|
-
const
|
|
585
|
+
const store = this.#store(context, source ?? driver.environment);
|
|
586
|
+
const control = await store.readControl();
|
|
483
587
|
if (control.autoDisabled) {
|
|
484
|
-
this.#
|
|
588
|
+
this.#cancelTimers(context);
|
|
485
589
|
context.state = 'disabled';
|
|
486
590
|
return;
|
|
487
591
|
}
|
|
@@ -489,13 +593,21 @@ export class LocalMemoryCoordinator {
|
|
|
489
593
|
return;
|
|
490
594
|
if (context.writerBlocked)
|
|
491
595
|
return;
|
|
492
|
-
const state = await
|
|
596
|
+
const state = await store.status();
|
|
597
|
+
if (context.state === 'error' && context.message === 'Local memory storage is unavailable') {
|
|
598
|
+
context.state = 'idle';
|
|
599
|
+
context.message = undefined;
|
|
600
|
+
}
|
|
493
601
|
const pending = Object.values(state.pending);
|
|
602
|
+
if (!this.#driver(context))
|
|
603
|
+
return;
|
|
494
604
|
if (control.activeAttempt) {
|
|
495
605
|
this.#cancelScheduledRun(context);
|
|
606
|
+
if (!context.run)
|
|
607
|
+
this.#scheduleWriterRetry(context);
|
|
496
608
|
}
|
|
497
609
|
else if (pending.some(({ checkpoint }) => !isBlockedCheckpoint(context, checkpoint))) {
|
|
498
|
-
const acceptedUpdates = await
|
|
610
|
+
const acceptedUpdates = await store.pendingAcceptedUpdates((checkpoint) => !isBlockedCheckpoint(context, checkpoint));
|
|
499
611
|
if (acceptedUpdates < MEMORY_AUTOMATIC_UPDATE_THRESHOLD) {
|
|
500
612
|
this.#cancelScheduledRun(context);
|
|
501
613
|
if (context.state === 'scheduled')
|
|
@@ -530,37 +642,84 @@ export class LocalMemoryCoordinator {
|
|
|
530
642
|
context.message = 'Local memory storage is unavailable';
|
|
531
643
|
}
|
|
532
644
|
}
|
|
533
|
-
#trackScheduleIfPending(context) {
|
|
534
|
-
const followUp = this.#scheduleIfPending(context).finally(() => {
|
|
645
|
+
#trackScheduleIfPending(context, environment) {
|
|
646
|
+
const followUp = this.#scheduleIfPending(context, environment).finally(() => {
|
|
535
647
|
context.followUps.delete(followUp);
|
|
536
648
|
});
|
|
537
649
|
context.followUps.add(followUp);
|
|
538
650
|
return followUp;
|
|
539
651
|
}
|
|
540
|
-
|
|
652
|
+
#launchRun(context, bypassRetryAt = false, preferred) {
|
|
653
|
+
if (context.run || context.storageBlocked || preferred && !this.#eligible(preferred))
|
|
654
|
+
return;
|
|
655
|
+
const driver = this.#driver(context, preferred);
|
|
656
|
+
if (!driver)
|
|
657
|
+
return;
|
|
658
|
+
context.driver = driver;
|
|
659
|
+
context.failedDriver = undefined;
|
|
660
|
+
const environment = this.#runEnvironment(context, driver);
|
|
661
|
+
context.runEnvironment = environment;
|
|
662
|
+
const run = this.#run(context, driver, environment, bypassRetryAt).catch((error) => {
|
|
663
|
+
context.state = 'error';
|
|
664
|
+
context.failedDriver = driver;
|
|
665
|
+
if (!bypassRetryAt && !driver.closing && !driver.detached)
|
|
666
|
+
context.storageBlocked = true;
|
|
667
|
+
context.message = sanitizeMemoryDiagnostic(error);
|
|
668
|
+
throw error;
|
|
669
|
+
}).finally(async () => {
|
|
670
|
+
if (context.run === settled) {
|
|
671
|
+
context.run = undefined;
|
|
672
|
+
context.driver = undefined;
|
|
673
|
+
context.runEnvironment = undefined;
|
|
674
|
+
context.abort = undefined;
|
|
675
|
+
}
|
|
676
|
+
await this.#trackScheduleIfPending(context);
|
|
677
|
+
this.#emitStatusChange();
|
|
678
|
+
});
|
|
679
|
+
const settled = run.catch(() => { });
|
|
680
|
+
context.run = settled;
|
|
681
|
+
return run;
|
|
682
|
+
}
|
|
683
|
+
async #run(context, driver, environment, bypassRetryAt) {
|
|
541
684
|
if (this.#disposed || this.#draining || !this.#enabled || context.storageBlocked)
|
|
542
685
|
return;
|
|
543
686
|
context.state = 'processing';
|
|
544
687
|
context.message = undefined;
|
|
688
|
+
const abort = new AbortController();
|
|
689
|
+
context.abort = abort;
|
|
690
|
+
const store = this.#store(context, environment);
|
|
545
691
|
let lease;
|
|
692
|
+
const cancelled = () => driver.closing || driver.detached || !this.#enabled || this.#disposed
|
|
693
|
+
|| abort.signal.aborted && !lease?.compromised.aborted;
|
|
546
694
|
try {
|
|
547
|
-
lease = await acquireLocalMemoryLease(
|
|
695
|
+
lease = await acquireLocalMemoryLease(environment, store.projectDirectory, this.#options.leaseOptions);
|
|
548
696
|
}
|
|
549
697
|
catch (error) {
|
|
550
|
-
|
|
698
|
+
if (cancelled()) {
|
|
699
|
+
context.state = 'idle';
|
|
700
|
+
context.message = driver.closing || driver.detached || this.#disposed
|
|
701
|
+
? 'Memory processing was cancelled during shutdown' : 'Memory processing was cancelled';
|
|
702
|
+
return;
|
|
703
|
+
}
|
|
704
|
+
this.#scheduleWriterRetry(context);
|
|
551
705
|
context.state = 'error';
|
|
706
|
+
context.failedDriver = driver;
|
|
552
707
|
context.message = safeProcessingMessage(error);
|
|
553
708
|
return;
|
|
554
709
|
}
|
|
555
710
|
if (!lease) {
|
|
556
|
-
context
|
|
711
|
+
this.#scheduleWriterRetry(context);
|
|
557
712
|
context.state = 'blocked';
|
|
713
|
+
context.failedDriver = driver;
|
|
558
714
|
context.message = 'Another Felan Code process owns the memory writer';
|
|
559
715
|
return;
|
|
560
716
|
}
|
|
561
717
|
context.writerBlocked = false;
|
|
562
|
-
|
|
563
|
-
|
|
718
|
+
if (context.failedDriver === driver)
|
|
719
|
+
context.failedDriver = undefined;
|
|
720
|
+
if (context.writerTimer)
|
|
721
|
+
clearTimeout(context.writerTimer);
|
|
722
|
+
context.writerTimer = undefined;
|
|
564
723
|
const abortOnLeaseCompromise = () => {
|
|
565
724
|
abort.abort(new Error('Memory writer lease was lost'));
|
|
566
725
|
};
|
|
@@ -570,7 +729,9 @@ export class LocalMemoryCoordinator {
|
|
|
570
729
|
let published = false;
|
|
571
730
|
let attempt;
|
|
572
731
|
try {
|
|
573
|
-
|
|
732
|
+
if (abort.signal.aborted || !this.#enabled || this.#disposed)
|
|
733
|
+
return;
|
|
734
|
+
const control = await this.#reconcileRunDiagnostics(context, environment, lease);
|
|
574
735
|
if (control.autoDisabled) {
|
|
575
736
|
context.state = 'disabled';
|
|
576
737
|
return;
|
|
@@ -579,24 +740,25 @@ export class LocalMemoryCoordinator {
|
|
|
579
740
|
context.state = 'scheduled';
|
|
580
741
|
return;
|
|
581
742
|
}
|
|
582
|
-
|
|
743
|
+
// Every pending checkpoint is a candidate so unreadable ones cannot occupy the batch.
|
|
744
|
+
const snapshot = await store.processingSnapshot(Number.POSITIVE_INFINITY, (checkpoint) => !isBlockedCheckpoint(context, checkpoint));
|
|
583
745
|
if (snapshot.checkpoints.length === 0) {
|
|
584
746
|
context.state = 'idle';
|
|
585
747
|
return;
|
|
586
748
|
}
|
|
587
|
-
retainedRun = await LocalMemoryRun.create(
|
|
588
|
-
projectDirectory:
|
|
749
|
+
retainedRun = await LocalMemoryRun.create(environment, {
|
|
750
|
+
projectDirectory: store.projectDirectory,
|
|
589
751
|
sessionDirectory: join(this.#options.sessionDir ?? join(this.#options.agentDir, 'sessions'), 'memory'),
|
|
590
752
|
projectKey: context.project.key,
|
|
591
753
|
projectRoot: context.project.canonicalRoot,
|
|
592
|
-
checkpoints: snapshot.checkpoints.map(({ checkpoint }) => checkpoint),
|
|
754
|
+
checkpoints: snapshot.checkpoints.slice(0, this.#batchSize).map(({ checkpoint }) => checkpoint),
|
|
593
755
|
baseFingerprint: snapshot.fingerprint,
|
|
594
756
|
...(control.lastDisabledRunId ? { protectedRunIds: [control.lastDisabledRunId] } : {}),
|
|
595
757
|
});
|
|
596
758
|
staging = retainedRun.workspace;
|
|
597
759
|
const memoryDirectory = join(staging, '.memory');
|
|
598
|
-
await hydrateMemoryDirectory(
|
|
599
|
-
const manifest = await materializeMemoryInput(
|
|
760
|
+
await hydrateMemoryDirectory(environment, snapshot.artifact, memoryDirectory, { memoryPath: '.memory', mode: 'read' });
|
|
761
|
+
const manifest = await materializeMemoryInput(environment, {
|
|
600
762
|
stagingDirectory: staging,
|
|
601
763
|
checkpoints: snapshot.checkpoints.map(({ checkpoint }) => checkpoint),
|
|
602
764
|
previousCheckpoints: Object.fromEntries(Object.entries(snapshot.state.processed).map(([sessionId, entry]) => [sessionId, entry.checkpoint])),
|
|
@@ -605,26 +767,29 @@ export class LocalMemoryCoordinator {
|
|
|
605
767
|
...(this.#options.maxTranscriptBytes === undefined
|
|
606
768
|
? {}
|
|
607
769
|
: { maxTranscriptBytes: this.#options.maxTranscriptBytes }),
|
|
770
|
+
maxSessions: this.#batchSize,
|
|
608
771
|
signal: abort.signal,
|
|
609
772
|
});
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
773
|
+
// A cancelled read reports as a failure; it must not drop evidence.
|
|
774
|
+
if (abort.signal.aborted || !this.#enabled || this.#disposed)
|
|
775
|
+
throw new Error('Memory processing was cancelled');
|
|
776
|
+
// A reachable Sessions service shows the failures belong to the checkpoints, not to the service.
|
|
777
|
+
if (manifest.failures.length > 0)
|
|
778
|
+
await environment.sessions.list({ scope: 'project' }, { limit: 1, signal: abort.signal });
|
|
779
|
+
const dropped = manifest.failures.length === 0 ? 0 : await store.dropCheckpoints(lease, manifest.failures.map(({ checkpoint, code }) => ({ checkpoint, reason: code })), this.#now());
|
|
780
|
+
const droppedMessage = dropped === 0 ? undefined : droppedCheckpointsMessage(dropped);
|
|
616
781
|
const processedCheckpoints = snapshot.checkpoints.filter(({ checkpoint }) => manifest.sessions.some((session) => session.checkpoint.sessionId === checkpoint.sessionId
|
|
617
782
|
&& sameCheckpointCursor(session.checkpoint, checkpoint)));
|
|
618
783
|
if (processedCheckpoints.length === 0) {
|
|
619
|
-
context.state = '
|
|
620
|
-
context.message =
|
|
621
|
-
await retainedRun.finish('blocked',
|
|
784
|
+
context.state = 'idle';
|
|
785
|
+
context.message = droppedMessage;
|
|
786
|
+
await retainedRun.finish('blocked', droppedMessage ?? 'Memory checkpoints changed before they could be read');
|
|
622
787
|
return;
|
|
623
788
|
}
|
|
624
789
|
await retainedRun.record({ phase: 'model', checkpoints: processedCheckpoints.map(({ checkpoint }) => checkpoint) });
|
|
625
790
|
if (abort.signal.aborted || !this.#enabled || this.#disposed)
|
|
626
791
|
throw new Error('Memory processing was cancelled');
|
|
627
|
-
attempt = await
|
|
792
|
+
attempt = await store.beginAttempt(lease, {
|
|
628
793
|
runId: retainedRun.metadata.sessionId,
|
|
629
794
|
baseFingerprint: snapshot.fingerprint,
|
|
630
795
|
checkpoints: processedCheckpoints,
|
|
@@ -636,16 +801,16 @@ export class LocalMemoryCoordinator {
|
|
|
636
801
|
}
|
|
637
802
|
context.attempt = attempt;
|
|
638
803
|
this.#emitStatusChange();
|
|
639
|
-
const result = await this.#dreamRunner({
|
|
804
|
+
const result = await (this.#options.dreamRunner ?? createDefaultLocalMemoryDreamRunner(environment))({
|
|
640
805
|
stagingDirectory: staging,
|
|
641
806
|
memoryDirectory,
|
|
642
807
|
inputDirectory: join(staging, '.dreaming', 'input'),
|
|
643
808
|
baseSnapshot: createMemorySnapshot(snapshot.artifact, '.memory', { mode: 'read' }),
|
|
644
809
|
manifest,
|
|
645
|
-
models:
|
|
646
|
-
...(
|
|
647
|
-
...(
|
|
648
|
-
...(
|
|
810
|
+
models: driver.models,
|
|
811
|
+
...(driver.selectedModel === undefined ? {} : { selectedModel: driver.selectedModel }),
|
|
812
|
+
...(driver.scopedModels === undefined ? {} : { scopedModels: driver.scopedModels }),
|
|
813
|
+
...(driver.classifier === undefined ? {} : { classifier: driver.classifier }),
|
|
649
814
|
sessionDirectory: join(this.#options.sessionDir ?? join(this.#options.agentDir, 'sessions'), 'memory'),
|
|
650
815
|
run: retainedRun,
|
|
651
816
|
signal: abort.signal,
|
|
@@ -653,9 +818,9 @@ export class LocalMemoryCoordinator {
|
|
|
653
818
|
if (abort.signal.aborted || !this.#enabled || this.#disposed)
|
|
654
819
|
throw new Error('Memory processing was cancelled');
|
|
655
820
|
if (result)
|
|
656
|
-
await hydrateMemoryDirectory(
|
|
821
|
+
await hydrateMemoryDirectory(environment, result, memoryDirectory, { replace: true, memoryPath: '.memory' });
|
|
657
822
|
await retainedRun.record({ phase: 'validate' });
|
|
658
|
-
const artifact = await readMemoryDirectory(
|
|
823
|
+
const artifact = await readMemoryDirectory(environment, memoryDirectory, {
|
|
659
824
|
memoryPath: '.memory',
|
|
660
825
|
sourceSessionIds: allowedSourceIds(snapshot, manifest),
|
|
661
826
|
requireSources: true,
|
|
@@ -664,42 +829,45 @@ export class LocalMemoryCoordinator {
|
|
|
664
829
|
phase: 'publish',
|
|
665
830
|
outputFingerprint: createMemorySnapshot(artifact, '.memory').fingerprint,
|
|
666
831
|
});
|
|
667
|
-
await
|
|
832
|
+
await store.commit(lease, snapshot.fingerprint, artifact, processedCheckpoints, attempt);
|
|
668
833
|
published = true;
|
|
669
|
-
await
|
|
834
|
+
await store.finishAttempt(lease, attempt, { status: 'success' }, this.#now());
|
|
670
835
|
await retainedRun.finish('completed');
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
}
|
|
675
|
-
else {
|
|
676
|
-
context.state = 'idle';
|
|
677
|
-
}
|
|
678
|
-
this.#emitStatusChange();
|
|
836
|
+
context.state = 'idle';
|
|
837
|
+
context.failedDriver = undefined;
|
|
838
|
+
context.message = droppedMessage;
|
|
679
839
|
}
|
|
680
840
|
catch (error) {
|
|
681
|
-
const leaseValid = await lease.verify();
|
|
682
|
-
const explicitlyCancelled =
|
|
683
|
-
|| (abort.signal.aborted && !lease.compromised.aborted);
|
|
841
|
+
const leaseValid = await lease.verify().catch(() => false);
|
|
842
|
+
const explicitlyCancelled = cancelled();
|
|
684
843
|
const unavailableModel = error instanceof MemoryModelUnavailableError;
|
|
685
844
|
const blockedBeforeInference = retainedRun !== undefined && attempt === undefined && !explicitlyCancelled;
|
|
686
845
|
if (attempt && (leaseValid || explicitlyCancelled)) {
|
|
687
846
|
try {
|
|
688
|
-
const control = await
|
|
847
|
+
const control = await store.finishAttempt(explicitlyCancelled || unavailableModel ? undefined : lease, attempt, explicitlyCancelled || unavailableModel ? { status: 'cancelled' }
|
|
689
848
|
: { status: 'failure', reason: sanitizeMemoryDiagnostic(error) }, this.#now());
|
|
690
849
|
published ||= control.lastOutcome?.runId === attempt.runId && control.lastOutcome.status === 'success';
|
|
691
850
|
}
|
|
692
851
|
catch {
|
|
693
|
-
|
|
852
|
+
if (!explicitlyCancelled)
|
|
853
|
+
context.storageBlocked = true;
|
|
694
854
|
}
|
|
695
855
|
}
|
|
856
|
+
if (explicitlyCancelled) {
|
|
857
|
+
context.state = 'idle';
|
|
858
|
+
context.message = driver.closing || driver.detached || this.#disposed
|
|
859
|
+
? 'Memory processing was cancelled during shutdown' : 'Memory processing was cancelled';
|
|
860
|
+
await retainedRun?.finish('cancelled', error).catch(() => { });
|
|
861
|
+
return;
|
|
862
|
+
}
|
|
696
863
|
if (published) {
|
|
697
864
|
context.state = 'idle';
|
|
865
|
+
context.failedDriver = context.storageBlocked ? driver : undefined;
|
|
698
866
|
context.message = 'Memory was published, but its diagnostics could not be fully saved';
|
|
699
|
-
this.#emitStatusChange();
|
|
700
867
|
return;
|
|
701
868
|
}
|
|
702
869
|
context.state = 'error';
|
|
870
|
+
context.failedDriver = driver;
|
|
703
871
|
context.message = !leaseValid
|
|
704
872
|
? safeProcessingMessage(new Error('Memory writer lease was lost'))
|
|
705
873
|
: safeProcessingMessage(error);
|
|
@@ -713,62 +881,68 @@ export class LocalMemoryCoordinator {
|
|
|
713
881
|
context.storageBlocked = true;
|
|
714
882
|
context.message = `Memory processing is paused: ${sanitizeMemoryDiagnostic(error)}`;
|
|
715
883
|
}
|
|
716
|
-
await retainedRun?.finish(
|
|
717
|
-
:
|
|
718
|
-
: !leaseValid ? 'interrupted' : 'failed', error).catch(() => {
|
|
884
|
+
await retainedRun?.finish(unavailableModel || blockedBeforeInference ? 'blocked'
|
|
885
|
+
: !leaseValid ? 'interrupted' : 'failed', error).catch(() => {
|
|
719
886
|
context.storageBlocked = true;
|
|
720
887
|
context.message = 'Memory diagnostics could not be saved; evidence remains pending';
|
|
721
888
|
});
|
|
722
|
-
this.#emitStatusChange();
|
|
723
889
|
}
|
|
724
890
|
finally {
|
|
725
891
|
lease.compromised.removeEventListener('abort', abortOnLeaseCompromise);
|
|
726
|
-
context.abort = undefined;
|
|
727
892
|
context.attempt = undefined;
|
|
728
|
-
await lease.release()
|
|
893
|
+
await lease.release().catch(error => {
|
|
894
|
+
if (!cancelled())
|
|
895
|
+
throw error;
|
|
896
|
+
});
|
|
729
897
|
}
|
|
730
898
|
}
|
|
731
|
-
async #reconcileRunDiagnostics(context, lease) {
|
|
732
|
-
|
|
733
|
-
|
|
899
|
+
async #reconcileRunDiagnostics(context, environment, lease) {
|
|
900
|
+
const store = this.#store(context, environment);
|
|
901
|
+
const previous = await store.readControl();
|
|
902
|
+
const control = await store.reconcileAbandonedAttempt(lease, this.#now());
|
|
903
|
+
if (previous.updatedAt !== control.updatedAt)
|
|
904
|
+
this.#emitStatusChange();
|
|
734
905
|
const options = {
|
|
735
|
-
projectDirectory:
|
|
906
|
+
projectDirectory: store.projectDirectory,
|
|
736
907
|
projectKey: context.project.key,
|
|
737
908
|
sessionDirectory: join(this.#options.sessionDir ?? join(this.#options.agentDir, 'sessions'), 'memory'),
|
|
738
909
|
...(control.lastDisabledRunId ? { protectedRunIds: [control.lastDisabledRunId] } : {}),
|
|
739
910
|
};
|
|
740
911
|
if (control.lastOutcome)
|
|
741
|
-
await LocalMemoryRun.reconcile(
|
|
742
|
-
await LocalMemoryRun.reconcileAbandoned(
|
|
912
|
+
await LocalMemoryRun.reconcile(environment, options, control.lastOutcome);
|
|
913
|
+
await LocalMemoryRun.reconcileAbandoned(environment, options, [
|
|
743
914
|
...(control.activeAttempt ? [control.activeAttempt.runId] : []),
|
|
744
915
|
...(control.lastOutcome ? [control.lastOutcome.runId] : []),
|
|
745
916
|
], this.#now());
|
|
746
917
|
return control;
|
|
747
918
|
}
|
|
748
|
-
async #recover(context, signal) {
|
|
919
|
+
async #recover(context, driver, signal) {
|
|
749
920
|
let cursor;
|
|
750
921
|
let recovered = false;
|
|
751
922
|
const stableAfter = this.#options.recoveryStableMs ?? 30_000;
|
|
923
|
+
const store = this.#store(context, driver.environment);
|
|
752
924
|
const stopped = () => this.#disposed || this.#draining || !this.#enabled || signal.aborted;
|
|
753
925
|
try {
|
|
754
926
|
do {
|
|
755
|
-
const page = await
|
|
927
|
+
const page = await driver.environment.sessions.list({ scope: 'project' }, { ...(cursor ? { cursor } : {}), signal });
|
|
756
928
|
for (const info of page.items) {
|
|
757
929
|
if (stopped())
|
|
758
930
|
return false;
|
|
759
931
|
if (info.parent || this.#now() - info.updatedAt < stableAfter)
|
|
760
932
|
continue;
|
|
933
|
+
let checkpoint;
|
|
761
934
|
try {
|
|
762
|
-
|
|
763
|
-
if (stopped())
|
|
764
|
-
return false;
|
|
765
|
-
recovered = (await context.store.recordCheckpoint(checkpoint)) || recovered;
|
|
935
|
+
checkpoint = await readMemorySessionCheckpoint(driver.environment.sessions, info.id, signal);
|
|
766
936
|
}
|
|
767
937
|
catch {
|
|
768
938
|
if (stopped())
|
|
769
939
|
return false;
|
|
770
940
|
// A session that cannot be read now is skipped; its live checkpoint or a later run records it.
|
|
941
|
+
continue;
|
|
771
942
|
}
|
|
943
|
+
if (stopped())
|
|
944
|
+
return false;
|
|
945
|
+
recovered = (await store.recordCheckpoint(checkpoint)) || recovered;
|
|
772
946
|
}
|
|
773
947
|
cursor = page.nextCursor;
|
|
774
948
|
} while (cursor);
|
|
@@ -790,13 +964,23 @@ export class LocalMemoryCoordinator {
|
|
|
790
964
|
return true;
|
|
791
965
|
}
|
|
792
966
|
#emitStatusChange() {
|
|
793
|
-
for (const listener of [...this.#statusListeners]) {
|
|
967
|
+
for (const listener of [...this.#statusListeners, ...[...this.#attachments].flatMap(attachment => [...attachment.listeners])]) {
|
|
794
968
|
try {
|
|
795
969
|
listener();
|
|
796
970
|
}
|
|
797
971
|
catch { }
|
|
798
972
|
}
|
|
799
973
|
}
|
|
974
|
+
#cancelTimers(context) {
|
|
975
|
+
this.#cancelScheduledRun(context);
|
|
976
|
+
if (context.recoveryTimer)
|
|
977
|
+
clearTimeout(context.recoveryTimer);
|
|
978
|
+
if (context.writerTimer)
|
|
979
|
+
clearTimeout(context.writerTimer);
|
|
980
|
+
context.recoveryTimer = undefined;
|
|
981
|
+
context.writerTimer = undefined;
|
|
982
|
+
context.writerBlocked = false;
|
|
983
|
+
}
|
|
800
984
|
#cancelScheduledRun(context) {
|
|
801
985
|
if (context.timer)
|
|
802
986
|
clearTimeout(context.timer);
|
|
@@ -844,6 +1028,9 @@ function allowedSourceIds(snapshot, manifest) {
|
|
|
844
1028
|
...manifest.sessions.map(({ checkpoint }) => checkpoint.sessionId),
|
|
845
1029
|
])];
|
|
846
1030
|
}
|
|
1031
|
+
function droppedCheckpointsMessage(count) {
|
|
1032
|
+
return `Dropped ${count} memory checkpoint${count === 1 ? '' : 's'} that could not be read`;
|
|
1033
|
+
}
|
|
847
1034
|
function safeProcessingMessage(error) {
|
|
848
1035
|
if (error instanceof MemoryModelUnavailableError) {
|
|
849
1036
|
return 'Low-tier memory model is unavailable; evidence remains pending';
|