@nowcrew/daemon 0.6.30 → 0.6.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-ability/materializer.js +4 -4
- package/dist/config.js +3 -3
- package/dist/execution-runner.js +17 -13
- package/dist/host-execution-coordinator.js +169 -52
- package/dist/local-executor.js +107 -34
- package/dist/project-skills/reconciler.js +5 -2
- package/dist/runner.js +6 -1
- package/dist/runtime-startup-gate.js +47 -23
- package/dist/runtimes/codex-app-server-runner.js +6 -0
- package/dist/serve.js +55 -3
- package/dist/shared-execution-slots.js +75 -16
- package/package.json +1 -1
|
@@ -8,7 +8,7 @@ import { abilityWorkspaceProjectionPath, DaemonAbilityLockSchema, DaemonAbilityM
|
|
|
8
8
|
import { loadAgentAbilityRuntimeContext } from "./runtime-context.js";
|
|
9
9
|
import { parseSkillFrontmatter } from "../skill-frontmatter.js";
|
|
10
10
|
import { isProjectSkillName, MAX_PROJECT_SKILL_DESCRIPTION_LENGTH, MAX_PROJECT_SKILL_FILE_BYTES, } from "../project-skills/types.js";
|
|
11
|
-
import { createAgentProjectionCoordinator,
|
|
11
|
+
import { createAgentProjectionCoordinator, } from "../project-skills/agent-projection-coordinator.js";
|
|
12
12
|
const exists = (path) => lstat(path).then(() => true, () => false);
|
|
13
13
|
const isRealDirectory = (path) => lstat(path)
|
|
14
14
|
.then((info) => info.isDirectory() && !info.isSymbolicLink(), () => false);
|
|
@@ -135,7 +135,7 @@ const projectNativeSkills = async (agentRoot, trainingRoot, skills) => {
|
|
|
135
135
|
const names = await copyExistingSkills(target, staging, managedRoots);
|
|
136
136
|
for (const skill of nativeSkills) {
|
|
137
137
|
if (names.has(skill.name))
|
|
138
|
-
|
|
138
|
+
continue;
|
|
139
139
|
await symlink(skill.source, join(staging, skill.name), "dir");
|
|
140
140
|
}
|
|
141
141
|
await switchDirectory(target, staging, backup);
|
|
@@ -555,7 +555,7 @@ export function createAgentAbilityMaterializer(expectedAgentsRoot, coordinator =
|
|
|
555
555
|
prepareAndLaunch(agentsRoot, handle, ability, launch) {
|
|
556
556
|
if (agentsRoot !== expectedAgentsRoot)
|
|
557
557
|
return Promise.reject(new Error("ability_agents_root_mismatch"));
|
|
558
|
-
return coordinator.
|
|
558
|
+
return coordinator.runExclusive(expectedAgentsRoot, handle, async () => {
|
|
559
559
|
const started = Date.now();
|
|
560
560
|
try {
|
|
561
561
|
const trainingRoot = await ensureWorkspaceTraining(agentsRoot, handle, ability);
|
|
@@ -575,7 +575,7 @@ export function createAgentAbilityMaterializer(expectedAgentsRoot, coordinator =
|
|
|
575
575
|
});
|
|
576
576
|
throw error;
|
|
577
577
|
}
|
|
578
|
-
}
|
|
578
|
+
});
|
|
579
579
|
},
|
|
580
580
|
};
|
|
581
581
|
}
|
package/dist/config.js
CHANGED
|
@@ -15,9 +15,9 @@ export const DEFAULT_EXECUTION_LIMITS = Object.freeze({
|
|
|
15
15
|
maxQueuedPerAgent: 32,
|
|
16
16
|
maxParallelTotal: 10,
|
|
17
17
|
maxQueuedTotal: 128,
|
|
18
|
-
maxStartingTotal:
|
|
19
|
-
maxStartingPerRuntime:
|
|
20
|
-
startupGapMs:
|
|
18
|
+
maxStartingTotal: 10,
|
|
19
|
+
maxStartingPerRuntime: 10,
|
|
20
|
+
startupGapMs: 500,
|
|
21
21
|
startupTimeoutMs: 120_000,
|
|
22
22
|
});
|
|
23
23
|
// Fits the largest mandatory v1 lifecycle envelope (UUID + timestamps + outcome facts) with margin.
|
package/dist/execution-runner.js
CHANGED
|
@@ -497,22 +497,14 @@ export async function runExecution(config, input, dependencies) {
|
|
|
497
497
|
let consoleSequence = 0;
|
|
498
498
|
let externalOutputSequence = 0;
|
|
499
499
|
const callbacks = {
|
|
500
|
-
|
|
500
|
+
onRuntimeStarting: () => {
|
|
501
|
+
dependencies.onRuntimePhase?.("starting");
|
|
502
|
+
},
|
|
503
|
+
onRuntimeRunning: async () => {
|
|
501
504
|
if (dependencies.cancellation?.isRequested())
|
|
502
505
|
return;
|
|
503
|
-
const ready = await dependencies.journal.markRuntimeReady(spec.executionId, now().toISOString());
|
|
504
|
-
if (ready.runtimeReadyAt === null) {
|
|
505
|
-
throw new Error(`Execution ${spec.executionId} runtime readiness was not persisted`);
|
|
506
|
-
}
|
|
507
|
-
startedAt = ready.runtimeReadyAt;
|
|
508
|
-
await reportBestEffort(dependencies.report, boundExecutionFrame(DaemonToServerExecutionFrameSchema.parse({
|
|
509
|
-
type: "execution:started",
|
|
510
|
-
protocolVersion: 1,
|
|
511
|
-
executionId: spec.executionId,
|
|
512
|
-
at: startedAt,
|
|
513
|
-
}), config.executionLimits.maxEventBytes), bestEffortTimeoutMs);
|
|
514
506
|
const cancel = runtimeCancel;
|
|
515
|
-
if (cancel !== null &&
|
|
507
|
+
if (cancel !== null && timeout === undefined) {
|
|
516
508
|
timeout = setTimeout(() => {
|
|
517
509
|
timedOut = true;
|
|
518
510
|
try {
|
|
@@ -523,6 +515,18 @@ export async function runExecution(config, input, dependencies) {
|
|
|
523
515
|
}
|
|
524
516
|
}, effectiveTimeoutMs);
|
|
525
517
|
}
|
|
518
|
+
dependencies.onRuntimePhase?.("running");
|
|
519
|
+
const ready = await dependencies.journal.markRuntimeReady(spec.executionId, now().toISOString());
|
|
520
|
+
if (ready.runtimeReadyAt === null) {
|
|
521
|
+
throw new Error(`Execution ${spec.executionId} runtime readiness was not persisted`);
|
|
522
|
+
}
|
|
523
|
+
startedAt = ready.runtimeReadyAt;
|
|
524
|
+
await reportBestEffort(dependencies.report, boundExecutionFrame(DaemonToServerExecutionFrameSchema.parse({
|
|
525
|
+
type: "execution:started",
|
|
526
|
+
protocolVersion: 1,
|
|
527
|
+
executionId: spec.executionId,
|
|
528
|
+
at: startedAt,
|
|
529
|
+
}), config.executionLimits.maxEventBytes), bestEffortTimeoutMs);
|
|
526
530
|
},
|
|
527
531
|
...(spec.reporting.streamActivity ? {
|
|
528
532
|
onActivity: (activity) => {
|
|
@@ -4,9 +4,12 @@ import { mkdir, open, readFile, rename, rm, writeFile } from "node:fs/promises";
|
|
|
4
4
|
import { randomUUID } from "node:crypto";
|
|
5
5
|
import { defaultProcessController } from "./execution-journal.js";
|
|
6
6
|
import { createJournalLease, createJournalLeaseRegistry, JournalLockedError, } from "./execution-journal-lock.js";
|
|
7
|
+
import { dslog } from "./slog.js";
|
|
7
8
|
const HOST_EXECUTION_SLOT_COUNT = 10;
|
|
8
|
-
const
|
|
9
|
+
const HOST_STARTUP_SLOT_COUNT = 10;
|
|
10
|
+
const HOST_STARTUP_GAP_MS = 500;
|
|
9
11
|
const RETRY_MS = 50;
|
|
12
|
+
const RELEASE_RETRY_MAX_MS = 5_000;
|
|
10
13
|
export function defaultHostExecutionCoordinatorRoot(userHome = homedir()) {
|
|
11
14
|
// Deliberately independent of CREW_DAEMON_HOME and agentsRoot: every profile
|
|
12
15
|
// owned by this OS user must share the same physical-compute safety boundary.
|
|
@@ -68,11 +71,12 @@ function reservation(prerequisite, acquire) {
|
|
|
68
71
|
const lease = await acquire(() => released);
|
|
69
72
|
if (lease === null)
|
|
70
73
|
return null;
|
|
74
|
+
owned = lease;
|
|
71
75
|
if (released) {
|
|
72
|
-
|
|
76
|
+
// The release path owns cleanup so every close failure gets the same
|
|
77
|
+
// retry, diagnostics, and drain tracking as an already-granted slot.
|
|
73
78
|
return null;
|
|
74
79
|
}
|
|
75
|
-
owned = lease;
|
|
76
80
|
granted = true;
|
|
77
81
|
return lease;
|
|
78
82
|
});
|
|
@@ -87,8 +91,23 @@ function reservation(prerequisite, acquire) {
|
|
|
87
91
|
return releasePromise;
|
|
88
92
|
released = true;
|
|
89
93
|
cancelPrerequisite();
|
|
90
|
-
|
|
91
|
-
|
|
94
|
+
const closeOwned = async () => {
|
|
95
|
+
const lease = owned;
|
|
96
|
+
if (lease === null)
|
|
97
|
+
return;
|
|
98
|
+
await lease.close();
|
|
99
|
+
if (owned === lease)
|
|
100
|
+
owned = null;
|
|
101
|
+
};
|
|
102
|
+
releasePromise = acquisition.then(closeOwned, async () => {
|
|
103
|
+
// Acquisition errors are already exposed through ready. Release only
|
|
104
|
+
// owns cleanup; with no acquired lease there is nothing to retry.
|
|
105
|
+
if (owned !== null)
|
|
106
|
+
await closeOwned();
|
|
107
|
+
}).catch((error) => {
|
|
108
|
+
releasePromise = null;
|
|
109
|
+
throw error;
|
|
110
|
+
});
|
|
92
111
|
void releasePromise.catch(() => undefined);
|
|
93
112
|
return releasePromise;
|
|
94
113
|
},
|
|
@@ -107,90 +126,161 @@ async function writeLaunchTimestamp(path, value) {
|
|
|
107
126
|
export function createHostExecutionCoordinator(options = {}) {
|
|
108
127
|
const root = resolve(options.root ?? defaultHostExecutionCoordinatorRoot());
|
|
109
128
|
const executionSlots = options.executionSlots ?? HOST_EXECUTION_SLOT_COUNT;
|
|
129
|
+
const startupSlots = options.startupSlots ?? HOST_STARTUP_SLOT_COUNT;
|
|
110
130
|
const startupGapMs = options.startupGapMs ?? HOST_STARTUP_GAP_MS;
|
|
111
131
|
const retryMs = options.retryMs ?? RETRY_MS;
|
|
112
132
|
const now = options.now ?? Date.now;
|
|
113
133
|
const wait = options.wait ?? ((milliseconds) => new Promise((resolveWait) => setTimeout(resolveWait, milliseconds)));
|
|
134
|
+
const acquireLease = options.acquireLease ?? leaseFor;
|
|
114
135
|
const pendingReleases = new Set();
|
|
115
136
|
if (!Number.isSafeInteger(executionSlots) || executionSlots <= 0) {
|
|
116
137
|
throw new RangeError("executionSlots must be a positive safe integer");
|
|
117
138
|
}
|
|
139
|
+
if (!Number.isSafeInteger(startupSlots) || startupSlots <= 0) {
|
|
140
|
+
throw new RangeError("startupSlots must be a positive safe integer");
|
|
141
|
+
}
|
|
118
142
|
if (!Number.isFinite(startupGapMs) || startupGapMs < 0) {
|
|
119
143
|
throw new RangeError("startupGapMs must be a nonnegative finite number");
|
|
120
144
|
}
|
|
121
145
|
if (!Number.isFinite(retryMs) || retryMs <= 0) {
|
|
122
146
|
throw new RangeError("retryMs must be a positive finite number");
|
|
123
147
|
}
|
|
124
|
-
const
|
|
148
|
+
const reportReleaseError = options.onReleaseError ?? ((event) => {
|
|
149
|
+
dslog("execution.host_lease_release_retry", "Host execution lease 释放失败,准备重试", {
|
|
150
|
+
level: "ERROR",
|
|
151
|
+
resource: event.resource,
|
|
152
|
+
attempt: event.attempt,
|
|
153
|
+
retry_delay_ms: event.retryDelayMs,
|
|
154
|
+
error_message: event.error instanceof Error ? event.error.message.slice(0, 500) : String(event.error).slice(0, 500),
|
|
155
|
+
});
|
|
156
|
+
});
|
|
157
|
+
const retryRelease = async (resource, release) => {
|
|
158
|
+
let attempt = 0;
|
|
159
|
+
while (true) {
|
|
160
|
+
try {
|
|
161
|
+
await release();
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
catch (error) {
|
|
165
|
+
attempt += 1;
|
|
166
|
+
const retryDelayMs = Math.min(RELEASE_RETRY_MAX_MS, retryMs * 2 ** Math.min(attempt - 1, 6));
|
|
167
|
+
try {
|
|
168
|
+
reportReleaseError({ resource, attempt, retryDelayMs, error });
|
|
169
|
+
}
|
|
170
|
+
catch { /* diagnostics must never stop lease cleanup */ }
|
|
171
|
+
await wait(retryDelayMs);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
};
|
|
175
|
+
const trackRelease = (release) => {
|
|
176
|
+
pendingReleases.add(release);
|
|
177
|
+
void release.finally(() => pendingReleases.delete(release)).catch(() => undefined);
|
|
178
|
+
return release;
|
|
179
|
+
};
|
|
180
|
+
const acquireExecution = async (released, lane) => {
|
|
125
181
|
let offset = 0;
|
|
182
|
+
const slotCount = lane === "memory_prune" ? 1 : executionSlots;
|
|
183
|
+
const slotRoot = lane === "memory_prune" ? "memory-prune-slots" : "slots";
|
|
126
184
|
while (!released()) {
|
|
127
|
-
for (let step = 0; step <
|
|
128
|
-
const slot = (offset + step) %
|
|
129
|
-
const lease = await
|
|
185
|
+
for (let step = 0; step < slotCount; step += 1) {
|
|
186
|
+
const slot = (offset + step) % slotCount;
|
|
187
|
+
const lease = await acquireLease(join(root, slotRoot, String(slot)));
|
|
130
188
|
if (lease !== null)
|
|
131
189
|
return lease;
|
|
132
190
|
}
|
|
133
|
-
offset = (offset + 1) %
|
|
191
|
+
offset = (offset + 1) % slotCount;
|
|
134
192
|
await wait(retryMs);
|
|
135
193
|
}
|
|
136
194
|
return null;
|
|
137
195
|
};
|
|
138
|
-
const acquireStartup = async (released) => {
|
|
196
|
+
const acquireStartup = async (released, lane) => {
|
|
197
|
+
const slotCount = lane === "memory_prune" ? 1 : startupSlots;
|
|
198
|
+
const slotRoot = lane === "memory_prune" ? "memory-prune-startup-slots" : "startup-slots";
|
|
199
|
+
// Keep the legacy directory as the short-lived launch-gap mutex. During a rolling upgrade,
|
|
200
|
+
// older daemons still hold this same lease for their full startup lifetime, so old and new
|
|
201
|
+
// versions remain mutually coordinated instead of producing an unbounded spawn burst.
|
|
202
|
+
const gapDirectory = join(root, lane === "memory_prune" ? "memory-prune-startup" : "startup");
|
|
203
|
+
let offset = 0;
|
|
139
204
|
while (!released()) {
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
205
|
+
let startupSlot = null;
|
|
206
|
+
for (let step = 0; step < slotCount; step += 1) {
|
|
207
|
+
const slot = (offset + step) % slotCount;
|
|
208
|
+
startupSlot = await acquireLease(join(root, slotRoot, String(slot)));
|
|
209
|
+
if (startupSlot !== null)
|
|
210
|
+
break;
|
|
211
|
+
}
|
|
212
|
+
if (startupSlot === null) {
|
|
213
|
+
offset = (offset + 1) % slotCount;
|
|
143
214
|
await wait(retryMs);
|
|
144
215
|
continue;
|
|
145
216
|
}
|
|
146
|
-
|
|
147
|
-
let previous = Number.NEGATIVE_INFINITY;
|
|
217
|
+
let granted = false;
|
|
148
218
|
try {
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
219
|
+
while (!released()) {
|
|
220
|
+
const gapLease = await acquireLease(gapDirectory);
|
|
221
|
+
if (gapLease === null) {
|
|
222
|
+
await wait(retryMs);
|
|
223
|
+
continue;
|
|
224
|
+
}
|
|
225
|
+
try {
|
|
226
|
+
const timestampPath = join(gapDirectory, "last-launch-at");
|
|
227
|
+
let previous = Number.NEGATIVE_INFINITY;
|
|
228
|
+
try {
|
|
229
|
+
previous = Number.parseInt(await readFile(timestampPath, "utf8"), 10);
|
|
230
|
+
if (!Number.isFinite(previous))
|
|
231
|
+
previous = Number.NEGATIVE_INFINITY;
|
|
232
|
+
}
|
|
233
|
+
catch (error) {
|
|
234
|
+
if (!(error instanceof Error && "code" in error && error.code === "ENOENT"))
|
|
235
|
+
throw error;
|
|
236
|
+
}
|
|
237
|
+
const delay = Math.max(0, previous + startupGapMs - now());
|
|
238
|
+
if (delay > 0)
|
|
239
|
+
await wait(delay);
|
|
240
|
+
if (released())
|
|
241
|
+
return null;
|
|
242
|
+
await writeLaunchTimestamp(timestampPath, now());
|
|
243
|
+
}
|
|
244
|
+
finally {
|
|
245
|
+
await retryRelease(`${lane}:startup_gap`, () => gapLease.close());
|
|
246
|
+
}
|
|
247
|
+
granted = true;
|
|
248
|
+
return startupSlot;
|
|
157
249
|
}
|
|
158
|
-
}
|
|
159
|
-
const delay = Math.max(0, previous + startupGapMs - now());
|
|
160
|
-
if (delay > 0)
|
|
161
|
-
await wait(delay);
|
|
162
|
-
if (released()) {
|
|
163
|
-
await lease.close();
|
|
164
250
|
return null;
|
|
165
251
|
}
|
|
166
|
-
|
|
167
|
-
|
|
252
|
+
finally {
|
|
253
|
+
if (!granted)
|
|
254
|
+
await retryRelease(`${lane}:startup_slot`, () => startupSlot.close());
|
|
255
|
+
}
|
|
168
256
|
}
|
|
169
257
|
return null;
|
|
170
258
|
};
|
|
171
|
-
const tracked = (entry) => {
|
|
259
|
+
const tracked = (entry, resource) => {
|
|
172
260
|
let trackedRelease = null;
|
|
173
261
|
return {
|
|
174
262
|
...entry,
|
|
175
263
|
release: () => {
|
|
176
264
|
if (trackedRelease !== null)
|
|
177
265
|
return trackedRelease;
|
|
178
|
-
trackedRelease = entry.release
|
|
179
|
-
pendingReleases.add(trackedRelease);
|
|
180
|
-
void trackedRelease.then(() => pendingReleases.delete(trackedRelease), () => pendingReleases.delete(trackedRelease));
|
|
266
|
+
trackedRelease = trackRelease(retryRelease(resource, entry.release));
|
|
181
267
|
return trackedRelease;
|
|
182
268
|
},
|
|
183
269
|
};
|
|
184
270
|
};
|
|
185
271
|
return {
|
|
186
|
-
reserveExecution: (prerequisite) => tracked(reservation(prerequisite, acquireExecution)),
|
|
187
|
-
reserveStartup: (prerequisite) => tracked(reservation(prerequisite, acquireStartup)),
|
|
272
|
+
reserveExecution: (prerequisite, lane = "foreground") => tracked(reservation(prerequisite, (released) => acquireExecution(released, lane)), `${lane}:execution_slot`),
|
|
273
|
+
reserveStartup: (prerequisite, lane = "foreground") => tracked(reservation(prerequisite, (released) => acquireStartup(released, lane)), `${lane}:startup_slot`),
|
|
188
274
|
tryAcquireExclusiveExecution: async () => {
|
|
189
275
|
const leases = [];
|
|
190
|
-
|
|
191
|
-
|
|
276
|
+
const targets = [
|
|
277
|
+
...Array.from({ length: executionSlots }, (_, slot) => join(root, "slots", String(slot))),
|
|
278
|
+
join(root, "memory-prune-slots", "0"),
|
|
279
|
+
];
|
|
280
|
+
for (const target of targets) {
|
|
281
|
+
const lease = await acquireLease(target);
|
|
192
282
|
if (lease === null) {
|
|
193
|
-
await Promise.all(leases.map((owned) => owned.close()));
|
|
283
|
+
await retryRelease("exclusive_execution_partial", () => Promise.all(leases.map((owned) => owned.close())).then(() => undefined));
|
|
194
284
|
return null;
|
|
195
285
|
}
|
|
196
286
|
leases.push(lease);
|
|
@@ -200,9 +290,7 @@ export function createHostExecutionCoordinator(options = {}) {
|
|
|
200
290
|
release: () => {
|
|
201
291
|
if (releasePromise !== null)
|
|
202
292
|
return releasePromise;
|
|
203
|
-
releasePromise = Promise.all(leases.map((lease) => lease.close())).then(() => undefined);
|
|
204
|
-
pendingReleases.add(releasePromise);
|
|
205
|
-
void releasePromise.then(() => pendingReleases.delete(releasePromise), () => pendingReleases.delete(releasePromise));
|
|
293
|
+
releasePromise = trackRelease(retryRelease("exclusive_execution", () => Promise.all(leases.map((lease) => lease.close())).then(() => undefined)));
|
|
206
294
|
return releasePromise;
|
|
207
295
|
},
|
|
208
296
|
};
|
|
@@ -214,13 +302,20 @@ export function createHostExecutionCoordinator(options = {}) {
|
|
|
214
302
|
};
|
|
215
303
|
}
|
|
216
304
|
export function hostCoordinatedSlotManager(local, host) {
|
|
305
|
+
const coordinated = new Set();
|
|
217
306
|
return {
|
|
218
307
|
reserve: (handle, kind, executionClass) => {
|
|
219
308
|
const localReservation = local.reserve(handle, kind, executionClass);
|
|
220
309
|
if (!localReservation.accepted)
|
|
221
310
|
return localReservation;
|
|
222
|
-
const hostReservation = host.reserveExecution(localReservation.ready);
|
|
223
|
-
|
|
311
|
+
const hostReservation = host.reserveExecution(localReservation.ready, executionClass === "memory_prune" ? "memory_prune" : "foreground");
|
|
312
|
+
const entry = {
|
|
313
|
+
local: localReservation,
|
|
314
|
+
host: hostReservation,
|
|
315
|
+
memoryPrune: executionClass === "memory_prune",
|
|
316
|
+
released: false,
|
|
317
|
+
};
|
|
318
|
+
coordinated.add(entry);
|
|
224
319
|
return {
|
|
225
320
|
...localReservation,
|
|
226
321
|
// File ownership is asynchronous, so accepted work is conservatively
|
|
@@ -229,23 +324,45 @@ export function hostCoordinatedSlotManager(local, host) {
|
|
|
229
324
|
ready: hostReservation.ready,
|
|
230
325
|
isQueued: hostReservation.isQueued,
|
|
231
326
|
release: () => {
|
|
232
|
-
if (released)
|
|
327
|
+
if (entry.released)
|
|
233
328
|
return;
|
|
234
|
-
released = true;
|
|
329
|
+
entry.released = true;
|
|
330
|
+
coordinated.delete(entry);
|
|
235
331
|
void hostReservation.release();
|
|
236
332
|
localReservation.release();
|
|
237
333
|
},
|
|
238
334
|
};
|
|
239
335
|
},
|
|
240
|
-
snapshot:
|
|
336
|
+
snapshot: () => {
|
|
337
|
+
const snapshot = local.snapshot();
|
|
338
|
+
let foregroundHostWaiting = 0;
|
|
339
|
+
let memoryPruneHostWaiting = 0;
|
|
340
|
+
for (const entry of coordinated) {
|
|
341
|
+
if (entry.released || entry.local.isQueued() || !entry.host.isQueued())
|
|
342
|
+
continue;
|
|
343
|
+
if (entry.memoryPrune)
|
|
344
|
+
memoryPruneHostWaiting += 1;
|
|
345
|
+
else
|
|
346
|
+
foregroundHostWaiting += 1;
|
|
347
|
+
}
|
|
348
|
+
return {
|
|
349
|
+
...snapshot,
|
|
350
|
+
queuedTotal: snapshot.queuedTotal + foregroundHostWaiting,
|
|
351
|
+
admittedTotal: Math.max(0, snapshot.admittedTotal - foregroundHostWaiting),
|
|
352
|
+
preparingTotal: Math.max(0, snapshot.preparingTotal - foregroundHostWaiting),
|
|
353
|
+
memoryPruneQueuedTotal: snapshot.memoryPruneQueuedTotal + memoryPruneHostWaiting,
|
|
354
|
+
memoryPruneAdmittedTotal: Math.max(0, snapshot.memoryPruneAdmittedTotal - memoryPruneHostWaiting),
|
|
355
|
+
memoryPrunePreparingTotal: Math.max(0, snapshot.memoryPrunePreparingTotal - memoryPruneHostWaiting),
|
|
356
|
+
};
|
|
357
|
+
},
|
|
241
358
|
tryAcquireExclusive: local.tryAcquireExclusive,
|
|
242
359
|
};
|
|
243
360
|
}
|
|
244
361
|
export function hostCoordinatedStartupGate(local, host) {
|
|
245
362
|
return {
|
|
246
|
-
reserve: (runtime) => {
|
|
247
|
-
const localReservation = local.reserve(runtime);
|
|
248
|
-
const hostReservation = host.reserveStartup(localReservation.ready);
|
|
363
|
+
reserve: (runtime, lane = "foreground") => {
|
|
364
|
+
const localReservation = local.reserve(runtime, lane);
|
|
365
|
+
const hostReservation = host.reserveStartup(localReservation.ready, lane);
|
|
249
366
|
let released = false;
|
|
250
367
|
return {
|
|
251
368
|
ready: hostReservation.ready,
|
package/dist/local-executor.js
CHANGED
|
@@ -19,7 +19,7 @@ import { decodeExternalOutputEvent, extractExternalAnswer, ExternalAnswerDecoder
|
|
|
19
19
|
import { cleanupMaterializedAttachments as cleanupAttachments, executionAttachmentDirectory, materializeAttachments, } from "./attachments.js";
|
|
20
20
|
import { routeRuntimeAttachments, runtimeCapability, } from "./runtime-capabilities.js";
|
|
21
21
|
import { awaitWithCancellation, RuntimeCancelledError, } from "./runtime-cancellation.js";
|
|
22
|
-
import { isRuntimeReadyEvent, } from "./runtime-startup-gate.js";
|
|
22
|
+
import { isRuntimeReadyEvent, isRuntimeRunningEvent, } from "./runtime-startup-gate.js";
|
|
23
23
|
import { dslog } from "./slog.js";
|
|
24
24
|
import { boundedDiagnosticJsonArray } from "./diagnostic-json.js";
|
|
25
25
|
import { ProjectProjectionError, } from "./project-skills/reconciler.js";
|
|
@@ -61,6 +61,7 @@ function logMemoryPruneDiagnosticsFailure(input, traceId, phase, error) {
|
|
|
61
61
|
});
|
|
62
62
|
}
|
|
63
63
|
const STDERR_TAIL_CAP = 2_000;
|
|
64
|
+
const RUNTIME_RUNNING_CALLBACK_TIMEOUT_MS = 10_000;
|
|
64
65
|
const MEMORY_PRUNE_DIAGNOSTICS_TIMEOUT_MS = 2_000;
|
|
65
66
|
const LOCAL_MEMORY_DIAGNOSTICS_TIMEOUT_MS = 250;
|
|
66
67
|
const CLAUDE_INSTRUCTION_WARNING = "claude_additional_directory_instructions_unverified";
|
|
@@ -266,7 +267,7 @@ async function executeLocalUnlocked(input, callbacks, dependencies) {
|
|
|
266
267
|
}
|
|
267
268
|
let materialized = null;
|
|
268
269
|
let knownAttachmentDirectory = null;
|
|
269
|
-
|
|
270
|
+
const startupReservationState = { current: null };
|
|
270
271
|
let memoryPruneTraceId = null;
|
|
271
272
|
let memoryPruneRuntimeExitCode;
|
|
272
273
|
let memoryPruneExecutorCompleted = false;
|
|
@@ -479,31 +480,11 @@ async function executeLocalUnlocked(input, callbacks, dependencies) {
|
|
|
479
480
|
const launchRuntime = dependencies.launchRuntime ?? launchLegacyRuntime;
|
|
480
481
|
if (dependencies.cancellation?.isRequested())
|
|
481
482
|
throw new RuntimeCancelledError();
|
|
482
|
-
startupReservation = dependencies.startupGate?.reserve(runtime.name) ?? null;
|
|
483
|
-
if (startupReservation !== null) {
|
|
484
|
-
const startupQueueEnteredAt = Date.now();
|
|
485
|
-
dslog("runtime.start_queued", "runtime 等待启动许可", {
|
|
486
|
-
execution_id: input.executionId,
|
|
487
|
-
runtime: runtime.name,
|
|
488
|
-
queued: startupReservation.isQueued(),
|
|
489
|
-
});
|
|
490
|
-
try {
|
|
491
|
-
await awaitWithCancellation(startupReservation.ready, dependencies.cancellation);
|
|
492
|
-
dslog("runtime.start_granted", "runtime 获得启动许可", {
|
|
493
|
-
execution_id: input.executionId,
|
|
494
|
-
runtime: runtime.name,
|
|
495
|
-
startup_queue_ms: Date.now() - startupQueueEnteredAt,
|
|
496
|
-
});
|
|
497
|
-
}
|
|
498
|
-
catch (error) {
|
|
499
|
-
startupReservation.release();
|
|
500
|
-
throw error;
|
|
501
|
-
}
|
|
502
|
-
}
|
|
503
|
-
const runtimeLaunchAt = Date.now();
|
|
504
|
-
memoryPruneFailurePhase = "runtime_launch";
|
|
505
483
|
let activeProjectSkillRuntimeRoot;
|
|
506
|
-
const
|
|
484
|
+
const abilityContext = input.abilityRelease !== undefined && dependencies.abilityRelease !== undefined
|
|
485
|
+
? await dependencies.abilityRelease.prepareAndLaunch(input.launch.agentsRoot, input.handle, input.abilityRelease, async (context) => context)
|
|
486
|
+
: undefined;
|
|
487
|
+
const prepareRuntimeLaunch = async (runtimeRoot) => {
|
|
507
488
|
activeProjectSkillRuntimeRoot = runtimeRoot;
|
|
508
489
|
const effectiveSystemPrompt = abilityContext === undefined
|
|
509
490
|
? systemPrompt
|
|
@@ -602,24 +583,55 @@ async function executeLocalUnlocked(input, callbacks, dependencies) {
|
|
|
602
583
|
projectContext: input.projectContext,
|
|
603
584
|
}),
|
|
604
585
|
};
|
|
586
|
+
return launchRequest;
|
|
587
|
+
};
|
|
588
|
+
let runtimeLaunchAt = Date.now();
|
|
589
|
+
const launchPreparedRuntime = async (runtimeRoot) => {
|
|
590
|
+
const launchRequest = await prepareRuntimeLaunch(runtimeRoot);
|
|
591
|
+
if (dependencies.cancellation?.isRequested())
|
|
592
|
+
throw new RuntimeCancelledError();
|
|
593
|
+
const startupLane = memoryPruneTraceId === null ? "foreground" : "memory_prune";
|
|
594
|
+
startupReservationState.current = dependencies.startupGate?.reserve(runtime.name, startupLane) ?? null;
|
|
595
|
+
const startupReservation = startupReservationState.current;
|
|
596
|
+
if (startupReservation !== null) {
|
|
597
|
+
const startupQueueEnteredAt = Date.now();
|
|
598
|
+
dslog("runtime.start_queued", "runtime 等待启动许可", {
|
|
599
|
+
execution_id: input.executionId,
|
|
600
|
+
runtime: runtime.name,
|
|
601
|
+
startup_lane: startupLane,
|
|
602
|
+
queued: startupReservation.isQueued(),
|
|
603
|
+
});
|
|
604
|
+
try {
|
|
605
|
+
await awaitWithCancellation(startupReservation.ready, dependencies.cancellation);
|
|
606
|
+
dslog("runtime.start_granted", "runtime 获得启动许可", {
|
|
607
|
+
execution_id: input.executionId,
|
|
608
|
+
runtime: runtime.name,
|
|
609
|
+
startup_lane: startupLane,
|
|
610
|
+
startup_queue_ms: Date.now() - startupQueueEnteredAt,
|
|
611
|
+
});
|
|
612
|
+
}
|
|
613
|
+
catch (error) {
|
|
614
|
+
startupReservation.release();
|
|
615
|
+
throw error;
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
await callbacks.onRuntimeStarting?.(runtime.name);
|
|
619
|
+
runtimeLaunchAt = Date.now();
|
|
605
620
|
memoryPruneFailurePhase = "runtime_launch";
|
|
606
621
|
return launchRuntime(launchRequest);
|
|
607
622
|
};
|
|
608
|
-
const launchWithAbility = (runtimeRoot) => input.abilityRelease !== undefined && dependencies.abilityRelease !== undefined
|
|
609
|
-
? dependencies.abilityRelease.prepareAndLaunch(input.launch.agentsRoot, input.handle, input.abilityRelease, (abilityContext) => launchPreparedRuntime(runtimeRoot, abilityContext))
|
|
610
|
-
: launchPreparedRuntime(runtimeRoot);
|
|
611
623
|
let child;
|
|
612
624
|
if (input.projectSkills !== undefined && dependencies.projectSkills !== undefined) {
|
|
613
625
|
const launchOwner = dependencies.projectSkillLeaseSafeLaunchOwner;
|
|
614
626
|
if (launchOwner === undefined)
|
|
615
627
|
throw new ProjectProjectionError("skill_projection_failed");
|
|
616
|
-
child = await dependencies.projectSkills.prepareAndLaunch(input.launch.agentsRoot, input.executionId, input.handle, input.projectSkills, launchOwner.claim(
|
|
628
|
+
child = await dependencies.projectSkills.prepareAndLaunch(input.launch.agentsRoot, input.executionId, input.handle, input.projectSkills, launchOwner.claim(launchPreparedRuntime), (warning) => callbacks.onConsole?.({
|
|
617
629
|
stream: "system",
|
|
618
630
|
text: formatProjectSkillRuntimeWarning(warning),
|
|
619
631
|
}));
|
|
620
632
|
}
|
|
621
633
|
else {
|
|
622
|
-
child = await
|
|
634
|
+
child = await launchPreparedRuntime();
|
|
623
635
|
}
|
|
624
636
|
localMemoryTelemetry?.markRuntimeStarted();
|
|
625
637
|
memoryPruneFailurePhase = "runtime_execution";
|
|
@@ -640,15 +652,24 @@ async function executeLocalUnlocked(input, callbacks, dependencies) {
|
|
|
640
652
|
// 每轮独立:tool_use/result 关联状态不能跨 execution 泄漏。
|
|
641
653
|
const consoleFormatter = createConsoleFormatter();
|
|
642
654
|
let runtimeReady = false;
|
|
655
|
+
let runtimeRunning = false;
|
|
643
656
|
let resolveRuntimeReady;
|
|
644
657
|
const runtimeReadySignal = new Promise((resolve) => { resolveRuntimeReady = resolve; });
|
|
645
658
|
let runtimeReadyNotification = Promise.resolve();
|
|
659
|
+
let runtimeRunningNotification = Promise.resolve();
|
|
660
|
+
let rejectRuntimeRunningFailure;
|
|
661
|
+
const runtimeRunningFailure = new Promise((_resolve, reject) => {
|
|
662
|
+
rejectRuntimeRunningFailure = reject;
|
|
663
|
+
});
|
|
664
|
+
// The failure can arrive in the same stdout turn before the exit race is installed.
|
|
665
|
+
// Attach a handler immediately while retaining the original rejection for that race.
|
|
666
|
+
void runtimeRunningFailure.catch(() => undefined);
|
|
646
667
|
const markRuntimeReady = () => {
|
|
647
668
|
if (runtimeReady)
|
|
648
669
|
return;
|
|
649
670
|
runtimeReady = true;
|
|
650
671
|
localMemoryTelemetry?.markRuntimeReady();
|
|
651
|
-
|
|
672
|
+
startupReservationState.current?.release();
|
|
652
673
|
dslog("runtime.start_ready", "runtime 已完成初始化", {
|
|
653
674
|
execution_id: input.executionId,
|
|
654
675
|
runtime: runtime.name,
|
|
@@ -657,6 +678,51 @@ async function executeLocalUnlocked(input, callbacks, dependencies) {
|
|
|
657
678
|
runtimeReadyNotification = Promise.resolve(callbacks.onRuntimeReady?.(runtime.name));
|
|
658
679
|
resolveRuntimeReady();
|
|
659
680
|
};
|
|
681
|
+
const markRuntimeRunning = () => {
|
|
682
|
+
if (runtimeRunning)
|
|
683
|
+
return;
|
|
684
|
+
runtimeRunning = true;
|
|
685
|
+
let callbackTimer;
|
|
686
|
+
runtimeRunningNotification = Promise.race([
|
|
687
|
+
Promise.resolve().then(() => callbacks.onRuntimeRunning?.(runtime.name)),
|
|
688
|
+
new Promise((_resolve, reject) => {
|
|
689
|
+
const callbackTimeoutMs = dependencies.runtimeRunningCallbackTimeoutMs
|
|
690
|
+
?? RUNTIME_RUNNING_CALLBACK_TIMEOUT_MS;
|
|
691
|
+
callbackTimer = setTimeout(() => reject(new Error(`${runtime.name} runtime-running callback timed out after ${callbackTimeoutMs}ms`)), callbackTimeoutMs);
|
|
692
|
+
}),
|
|
693
|
+
]).finally(() => {
|
|
694
|
+
if (callbackTimer !== undefined)
|
|
695
|
+
clearTimeout(callbackTimer);
|
|
696
|
+
});
|
|
697
|
+
void runtimeRunningNotification.catch((error) => {
|
|
698
|
+
// Persisting the running phase is part of execution liveness. Propagate
|
|
699
|
+
// its failure immediately: cancellation is best-effort and must not be
|
|
700
|
+
// allowed to keep the execution or its slots alive indefinitely.
|
|
701
|
+
rejectRuntimeRunningFailure(error);
|
|
702
|
+
try {
|
|
703
|
+
void child.cancel?.().catch((cancelError) => {
|
|
704
|
+
dslog("runtime.running_callback_cancel_failed", "runtime-running 回调失败后的取消也失败", {
|
|
705
|
+
level: "ERROR",
|
|
706
|
+
execution_id: input.executionId,
|
|
707
|
+
runtime: runtime.name,
|
|
708
|
+
error_message: cancelError instanceof Error
|
|
709
|
+
? cancelError.message.slice(0, 500)
|
|
710
|
+
: String(cancelError).slice(0, 500),
|
|
711
|
+
});
|
|
712
|
+
});
|
|
713
|
+
}
|
|
714
|
+
catch (cancelError) {
|
|
715
|
+
dslog("runtime.running_callback_cancel_failed", "runtime-running 回调失败后的取消也失败", {
|
|
716
|
+
level: "ERROR",
|
|
717
|
+
execution_id: input.executionId,
|
|
718
|
+
runtime: runtime.name,
|
|
719
|
+
error_message: cancelError instanceof Error
|
|
720
|
+
? cancelError.message.slice(0, 500)
|
|
721
|
+
: String(cancelError).slice(0, 500),
|
|
722
|
+
});
|
|
723
|
+
}
|
|
724
|
+
});
|
|
725
|
+
};
|
|
660
726
|
const readline = createInterface({ input: child.stdout });
|
|
661
727
|
readline.on("line", (line) => {
|
|
662
728
|
const event = parseLine(line);
|
|
@@ -665,6 +731,11 @@ async function executeLocalUnlocked(input, callbacks, dependencies) {
|
|
|
665
731
|
localMemoryTelemetry?.observe(event);
|
|
666
732
|
if (isRuntimeReadyEvent(runtime.name, event))
|
|
667
733
|
markRuntimeReady();
|
|
734
|
+
if (isRuntimeRunningEvent(runtime.name, event)) {
|
|
735
|
+
if (!runtimeReady)
|
|
736
|
+
markRuntimeReady();
|
|
737
|
+
markRuntimeRunning();
|
|
738
|
+
}
|
|
668
739
|
const meta = extractRunMeta(event);
|
|
669
740
|
if (meta.sessionId)
|
|
670
741
|
sessionId = meta.sessionId;
|
|
@@ -723,6 +794,7 @@ async function executeLocalUnlocked(input, callbacks, dependencies) {
|
|
|
723
794
|
child.stderr.on("end", () => codexStartupStageParser?.finish());
|
|
724
795
|
let runtimeExit;
|
|
725
796
|
try {
|
|
797
|
+
const startupReservation = startupReservationState.current;
|
|
726
798
|
if (startupReservation !== null) {
|
|
727
799
|
let startupTimer;
|
|
728
800
|
let startupOutcome;
|
|
@@ -761,7 +833,8 @@ async function executeLocalUnlocked(input, callbacks, dependencies) {
|
|
|
761
833
|
}
|
|
762
834
|
await runtimeReadyNotification;
|
|
763
835
|
}
|
|
764
|
-
runtimeExit = await awaitWithCancellation(child.exit, dependencies.cancellation);
|
|
836
|
+
runtimeExit = await awaitWithCancellation(Promise.race([child.exit, runtimeRunningFailure]), dependencies.cancellation);
|
|
837
|
+
await runtimeRunningNotification;
|
|
765
838
|
}
|
|
766
839
|
catch (error) {
|
|
767
840
|
if (error instanceof RuntimeCancelledError) {
|
|
@@ -831,7 +904,7 @@ async function executeLocalUnlocked(input, callbacks, dependencies) {
|
|
|
831
904
|
};
|
|
832
905
|
}
|
|
833
906
|
finally {
|
|
834
|
-
|
|
907
|
+
startupReservationState.current?.release();
|
|
835
908
|
try {
|
|
836
909
|
await localMemoryTelemetry?.finish();
|
|
837
910
|
}
|
|
@@ -105,7 +105,7 @@ export const canonicalTrainingSkillLinkTarget = (source, link, trainingSkillsRoo
|
|
|
105
105
|
: null;
|
|
106
106
|
};
|
|
107
107
|
export const isTrainingSkillLink = (source, link, trainingSkillsRoot, platform) => canonicalTrainingSkillLinkTarget(source, link, trainingSkillsRoot, platform) !== null;
|
|
108
|
-
const preserveNonProjectSkills = async (projectionTarget, trainingSkillsRoot, platform) => {
|
|
108
|
+
const preserveNonProjectSkills = async (projectionTarget, trainingSkillsRoot, platform, projectSkillNames) => {
|
|
109
109
|
const names = new Set();
|
|
110
110
|
if (!await exists(projectionTarget.target))
|
|
111
111
|
return names;
|
|
@@ -130,6 +130,8 @@ const preserveNonProjectSkills = async (projectionTarget, trainingSkillsRoot, pl
|
|
|
130
130
|
const canonicalTarget = canonicalTrainingSkillLinkTarget(source, link, trainingSkillsRoot, platform);
|
|
131
131
|
if (canonicalTarget === null)
|
|
132
132
|
continue;
|
|
133
|
+
if (projectSkillNames.has(entry.name))
|
|
134
|
+
continue;
|
|
133
135
|
await symlink(platform === "win32" ? canonicalTarget : link, destination, platform === "win32" ? "junction" : "dir");
|
|
134
136
|
}
|
|
135
137
|
else {
|
|
@@ -194,6 +196,7 @@ export function createProjectSkillsReconciler(deps) {
|
|
|
194
196
|
resolutionRecords.push(Object.freeze({ ...binding, sourcePath: null, mode: "missing" }));
|
|
195
197
|
}
|
|
196
198
|
}
|
|
199
|
+
const linkedNames = new Set(linked.map((item) => item.binding.skillName));
|
|
197
200
|
const id = randomUUID();
|
|
198
201
|
const targets = projectionTargets(agentRoot, id);
|
|
199
202
|
const nextProjectionDiagnostics = [];
|
|
@@ -202,7 +205,7 @@ export function createProjectSkillsReconciler(deps) {
|
|
|
202
205
|
await mkdir(dirname(target.target), { recursive: true });
|
|
203
206
|
await mkdir(target.staging, { mode: 0o700 });
|
|
204
207
|
await markProjectionStaging(target, id);
|
|
205
|
-
const preservedNames = await preserveNonProjectSkills(target, join(agentRoot, "training", "skills"), platform);
|
|
208
|
+
const preservedNames = await preserveNonProjectSkills(target, join(agentRoot, "training", "skills"), platform, linkedNames);
|
|
206
209
|
for (const item of linked) {
|
|
207
210
|
if (preservedNames.has(item.binding.skillName)) {
|
|
208
211
|
throw new ProjectProjectionError("skill_name_conflict");
|
package/dist/runner.js
CHANGED
|
@@ -105,7 +105,12 @@ export async function runAgent(config, input, onActivity = defaultPrint, onConso
|
|
|
105
105
|
softTokens: config.sessionSoftTokens,
|
|
106
106
|
maxTurns: config.sessionMaxTurns,
|
|
107
107
|
},
|
|
108
|
-
}, {
|
|
108
|
+
}, {
|
|
109
|
+
onActivity,
|
|
110
|
+
onConsole,
|
|
111
|
+
onRuntimeStarting: () => dependencies.onRuntimePhase?.("starting"),
|
|
112
|
+
onRuntimeRunning: () => dependencies.onRuntimePhase?.("running"),
|
|
113
|
+
}, {
|
|
109
114
|
launchRuntime: dependencies.launchRuntime ?? ((request) => launchSupervisedRuntime(request, dependencies.startSupervisor, dependencies.cancellation, dependencies.platform)),
|
|
110
115
|
...(dependencies.cancellation === undefined ? {} : { cancellation: dependencies.cancellation }),
|
|
111
116
|
...(dependencies.startupGate === undefined ? {} : { startupGate: dependencies.startupGate }),
|
|
@@ -1,41 +1,54 @@
|
|
|
1
1
|
export function createRuntimeStartupGate(limits, now = Date.now) {
|
|
2
|
-
const
|
|
2
|
+
const activeByRuntimeAndLane = new Map();
|
|
3
3
|
const queue = [];
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
4
|
+
const activeByLane = { foreground: 0, memory_prune: 0 };
|
|
5
|
+
const lastLaunchAt = {
|
|
6
|
+
foreground: Number.NEGATIVE_INFINITY,
|
|
7
|
+
memory_prune: Number.NEGATIVE_INFINITY,
|
|
8
|
+
};
|
|
9
|
+
const activeKey = (runtime, lane) => `${lane}:${runtime}`;
|
|
10
|
+
const maxTotalFor = (lane) => lane === "memory_prune" ? 1 : limits.maxStartingTotal;
|
|
11
|
+
const maxPerRuntimeFor = (lane) => lane === "memory_prune" ? 1 : limits.maxStartingPerRuntime;
|
|
12
|
+
const promoteLane = (lane) => {
|
|
13
|
+
while (activeByLane[lane] < maxTotalFor(lane)) {
|
|
8
14
|
const index = queue.findIndex((entry) => !entry.released
|
|
9
|
-
&&
|
|
15
|
+
&& entry.lane === lane
|
|
16
|
+
&& (activeByRuntimeAndLane.get(activeKey(entry.runtime, lane)) ?? 0) < maxPerRuntimeFor(lane));
|
|
10
17
|
if (index < 0)
|
|
11
18
|
return;
|
|
12
19
|
const [next] = queue.splice(index, 1);
|
|
13
20
|
if (!next || next.released)
|
|
14
21
|
continue;
|
|
15
22
|
next.promoted = true;
|
|
16
|
-
|
|
17
|
-
|
|
23
|
+
activeByLane[lane] += 1;
|
|
24
|
+
const key = activeKey(next.runtime, lane);
|
|
25
|
+
activeByRuntimeAndLane.set(key, (activeByRuntimeAndLane.get(key) ?? 0) + 1);
|
|
18
26
|
const grant = () => {
|
|
19
27
|
next.timer = null;
|
|
20
28
|
if (next.released)
|
|
21
29
|
return;
|
|
22
30
|
next.launchGranted = true;
|
|
23
|
-
lastLaunchAt = now();
|
|
31
|
+
lastLaunchAt[lane] = now();
|
|
24
32
|
next.resolve();
|
|
25
33
|
};
|
|
26
|
-
const delay = Math.max(0, lastLaunchAt + limits.startupGapMs - now());
|
|
34
|
+
const delay = Math.max(0, lastLaunchAt[lane] + limits.startupGapMs - now());
|
|
27
35
|
if (delay === 0)
|
|
28
36
|
grant();
|
|
29
37
|
else
|
|
30
38
|
next.timer = setTimeout(grant, delay);
|
|
31
39
|
}
|
|
32
40
|
};
|
|
41
|
+
const promote = () => {
|
|
42
|
+
promoteLane("foreground");
|
|
43
|
+
promoteLane("memory_prune");
|
|
44
|
+
};
|
|
33
45
|
return {
|
|
34
|
-
reserve: (runtime) => {
|
|
46
|
+
reserve: (runtime, lane = "foreground") => {
|
|
35
47
|
let resolveReady;
|
|
36
48
|
const ready = new Promise((resolve) => { resolveReady = resolve; });
|
|
37
49
|
const entry = {
|
|
38
50
|
runtime,
|
|
51
|
+
lane,
|
|
39
52
|
released: false,
|
|
40
53
|
promoted: false,
|
|
41
54
|
launchGranted: false,
|
|
@@ -54,12 +67,13 @@ export function createRuntimeStartupGate(limits, now = Date.now) {
|
|
|
54
67
|
if (entry.timer !== null)
|
|
55
68
|
clearTimeout(entry.timer);
|
|
56
69
|
if (entry.promoted) {
|
|
57
|
-
|
|
58
|
-
const
|
|
70
|
+
activeByLane[lane] = Math.max(0, activeByLane[lane] - 1);
|
|
71
|
+
const key = activeKey(runtime, lane);
|
|
72
|
+
const nextForRuntime = Math.max(0, (activeByRuntimeAndLane.get(key) ?? 1) - 1);
|
|
59
73
|
if (nextForRuntime === 0)
|
|
60
|
-
|
|
74
|
+
activeByRuntimeAndLane.delete(key);
|
|
61
75
|
else
|
|
62
|
-
|
|
76
|
+
activeByRuntimeAndLane.set(key, nextForRuntime);
|
|
63
77
|
}
|
|
64
78
|
else {
|
|
65
79
|
const index = queue.indexOf(entry);
|
|
@@ -71,15 +85,17 @@ export function createRuntimeStartupGate(limits, now = Date.now) {
|
|
|
71
85
|
};
|
|
72
86
|
},
|
|
73
87
|
snapshot: () => ({
|
|
74
|
-
startingTotal:
|
|
75
|
-
queuedTotal: queue.length,
|
|
88
|
+
startingTotal: activeByLane.foreground,
|
|
89
|
+
queuedTotal: queue.filter((entry) => entry.lane === "foreground").length,
|
|
90
|
+
memoryPruneStartingTotal: activeByLane.memory_prune,
|
|
91
|
+
memoryPruneQueuedTotal: queue.filter((entry) => entry.lane === "memory_prune").length,
|
|
76
92
|
startingByRuntime: {
|
|
77
|
-
claude:
|
|
78
|
-
codex:
|
|
79
|
-
kimi:
|
|
80
|
-
hermes:
|
|
81
|
-
opencode:
|
|
82
|
-
"deepseek-harness":
|
|
93
|
+
claude: activeByRuntimeAndLane.get(activeKey("claude", "foreground")) ?? 0,
|
|
94
|
+
codex: activeByRuntimeAndLane.get(activeKey("codex", "foreground")) ?? 0,
|
|
95
|
+
kimi: activeByRuntimeAndLane.get(activeKey("kimi", "foreground")) ?? 0,
|
|
96
|
+
hermes: activeByRuntimeAndLane.get(activeKey("hermes", "foreground")) ?? 0,
|
|
97
|
+
opencode: activeByRuntimeAndLane.get(activeKey("opencode", "foreground")) ?? 0,
|
|
98
|
+
"deepseek-harness": activeByRuntimeAndLane.get(activeKey("deepseek-harness", "foreground")) ?? 0,
|
|
83
99
|
},
|
|
84
100
|
}),
|
|
85
101
|
};
|
|
@@ -92,3 +108,11 @@ export function isRuntimeReadyEvent(runtime, event) {
|
|
|
92
108
|
return value.type === "system" && value.subtype === "init";
|
|
93
109
|
return value.type === "thread.started";
|
|
94
110
|
}
|
|
111
|
+
export function isRuntimeRunningEvent(runtime, event) {
|
|
112
|
+
if (typeof event !== "object" || event === null)
|
|
113
|
+
return false;
|
|
114
|
+
const value = event;
|
|
115
|
+
if (runtime === "codex")
|
|
116
|
+
return value.type === "turn.started";
|
|
117
|
+
return isRuntimeReadyEvent(runtime, event);
|
|
118
|
+
}
|
|
@@ -209,6 +209,12 @@ export function mapCodexNotification(method, params) {
|
|
|
209
209
|
if (method === "thread/started" && value.thread?.id) {
|
|
210
210
|
return [{ type: "thread.started", thread_id: value.thread.id }];
|
|
211
211
|
}
|
|
212
|
+
if (method === "turn/started") {
|
|
213
|
+
return [{
|
|
214
|
+
type: "turn.started",
|
|
215
|
+
...(value.turn?.id === undefined ? {} : { turn_id: value.turn.id }),
|
|
216
|
+
}];
|
|
217
|
+
}
|
|
212
218
|
const plan = method === "turn/plan/updated" ? boundedCodexPlan(value.plan) : null;
|
|
213
219
|
if (plan !== null && typeof value.threadId === "string" && typeof value.turnId === "string") {
|
|
214
220
|
return [{
|
package/dist/serve.js
CHANGED
|
@@ -80,7 +80,11 @@ export function serve(config, opts = {}) {
|
|
|
80
80
|
let runtimeFacts = null;
|
|
81
81
|
const runtimeProbe = createRuntimeProbeCoordinator();
|
|
82
82
|
const hostCoordinator = opts.execution?.hostCoordinator
|
|
83
|
-
?? createHostExecutionCoordinator({
|
|
83
|
+
?? createHostExecutionCoordinator({
|
|
84
|
+
executionSlots: config.executionLimits.maxParallelTotal,
|
|
85
|
+
startupSlots: config.executionLimits.maxStartingTotal,
|
|
86
|
+
startupGapMs: config.executionLimits.startupGapMs,
|
|
87
|
+
});
|
|
84
88
|
const localSlots = createSharedSlotManager(config.executionLimits);
|
|
85
89
|
const sharedSlots = hostCoordinatedSlotManager(localSlots, hostCoordinator);
|
|
86
90
|
const runtimeStartupGate = hostCoordinatedStartupGate(createRuntimeStartupGate(config.executionLimits), hostCoordinator);
|
|
@@ -502,7 +506,9 @@ export function serve(config, opts = {}) {
|
|
|
502
506
|
task_key: spec.workspace.taskKey,
|
|
503
507
|
});
|
|
504
508
|
}
|
|
505
|
-
const executionClass =
|
|
509
|
+
const executionClass = pruneTraceId !== null
|
|
510
|
+
? "memory_prune"
|
|
511
|
+
: spec.context.scheduledRunId === undefined ? "normal" : "scheduled";
|
|
506
512
|
const reservation = sharedSlots.reserve(spec.agent.handle, "execution", executionClass);
|
|
507
513
|
if (!reservation.accepted) {
|
|
508
514
|
dslog("execution.machine_queue_rejected", "机器执行队列已满", {
|
|
@@ -604,19 +610,49 @@ export function serve(config, opts = {}) {
|
|
|
604
610
|
report: reportExecutionFrame,
|
|
605
611
|
startupGate: runtimeStartupGate,
|
|
606
612
|
startupTimeoutMs: config.executionLimits.startupTimeoutMs,
|
|
613
|
+
onRuntimePhase: (phase) => {
|
|
614
|
+
if (phase === "starting")
|
|
615
|
+
reservation.markStarting();
|
|
616
|
+
else
|
|
617
|
+
reservation.markRunning();
|
|
618
|
+
const snapshot = sharedSlots.snapshot();
|
|
619
|
+
dslog(`execution.runtime_${phase}`, `execution runtime ${phase}`, {
|
|
620
|
+
execution_id: spec.executionId,
|
|
621
|
+
agent_handle: spec.agent.handle,
|
|
622
|
+
execution_class: executionClass,
|
|
623
|
+
admitted_total: snapshot.admittedTotal,
|
|
624
|
+
preparing_total: snapshot.preparingTotal,
|
|
625
|
+
starting_total: snapshot.startingTotal,
|
|
626
|
+
running_total: snapshot.runningTotal,
|
|
627
|
+
queued_total: snapshot.queuedTotal,
|
|
628
|
+
memory_prune_preparing_total: snapshot.memoryPrunePreparingTotal,
|
|
629
|
+
memory_prune_starting_total: snapshot.memoryPruneStartingTotal,
|
|
630
|
+
memory_prune_running_total: snapshot.memoryPruneRunningTotal,
|
|
631
|
+
});
|
|
632
|
+
},
|
|
607
633
|
...(reservation.state === undefined ? {} : {
|
|
608
634
|
slot: {
|
|
609
635
|
state: reservation.state,
|
|
610
636
|
ready: reservation.ready.then(() => {
|
|
611
637
|
markExecutionTaskKeyActive();
|
|
638
|
+
reservation.markPreparing();
|
|
612
639
|
const snapshot = sharedSlots.snapshot();
|
|
613
640
|
dslog("execution.machine_slot_ready", "execution 获得机器执行名额", {
|
|
614
641
|
execution_id: spec.executionId,
|
|
615
642
|
agent_handle: spec.agent.handle,
|
|
616
643
|
execution_class: executionClass,
|
|
617
644
|
queue_ms: Date.now() - machineQueueEnteredAt,
|
|
618
|
-
active_total: snapshot.
|
|
645
|
+
active_total: snapshot.runningTotal,
|
|
646
|
+
admitted_total: snapshot.admittedTotal,
|
|
647
|
+
preparing_total: snapshot.preparingTotal,
|
|
648
|
+
starting_total: snapshot.startingTotal,
|
|
649
|
+
running_total: snapshot.runningTotal,
|
|
619
650
|
queued_total: snapshot.queuedTotal,
|
|
651
|
+
memory_prune_admitted_total: snapshot.memoryPruneAdmittedTotal,
|
|
652
|
+
memory_prune_queued_total: snapshot.memoryPruneQueuedTotal,
|
|
653
|
+
memory_prune_preparing_total: snapshot.memoryPrunePreparingTotal,
|
|
654
|
+
memory_prune_starting_total: snapshot.memoryPruneStartingTotal,
|
|
655
|
+
memory_prune_running_total: snapshot.memoryPruneRunningTotal,
|
|
620
656
|
});
|
|
621
657
|
}),
|
|
622
658
|
},
|
|
@@ -846,6 +882,7 @@ export function serve(config, opts = {}) {
|
|
|
846
882
|
});
|
|
847
883
|
}
|
|
848
884
|
await awaitWithCancellation(legacyReservation.ready, controller.cancellation);
|
|
885
|
+
legacyReservation.markPreparing();
|
|
849
886
|
const queueMs = Date.now() - queueWaitStart;
|
|
850
887
|
const machineSnapshot = sharedSlots.snapshot();
|
|
851
888
|
const threadLabel = threadId ?? null;
|
|
@@ -967,6 +1004,21 @@ export function serve(config, opts = {}) {
|
|
|
967
1004
|
cancellation: controller.cancellation,
|
|
968
1005
|
startupGate: runtimeStartupGate,
|
|
969
1006
|
startupTimeoutMs: config.executionLimits.startupTimeoutMs,
|
|
1007
|
+
onRuntimePhase: (phase) => {
|
|
1008
|
+
if (phase === "starting")
|
|
1009
|
+
legacyReservation?.markStarting();
|
|
1010
|
+
else
|
|
1011
|
+
legacyReservation?.markRunning();
|
|
1012
|
+
const snapshot = sharedSlots.snapshot();
|
|
1013
|
+
dslog(`run.runtime_${phase}`, `legacy runtime ${phase}`, {
|
|
1014
|
+
...runKeys,
|
|
1015
|
+
admitted_total: snapshot.admittedTotal,
|
|
1016
|
+
preparing_total: snapshot.preparingTotal,
|
|
1017
|
+
starting_total: snapshot.startingTotal,
|
|
1018
|
+
running_total: snapshot.runningTotal,
|
|
1019
|
+
queued_total: snapshot.queuedTotal,
|
|
1020
|
+
});
|
|
1021
|
+
},
|
|
970
1022
|
});
|
|
971
1023
|
});
|
|
972
1024
|
const result = mergeRunAgentResults(guarded.results);
|
|
@@ -1,25 +1,44 @@
|
|
|
1
1
|
export function createSharedSlotManager(limits) {
|
|
2
2
|
const activeByHandleAndClass = new Map();
|
|
3
3
|
const queue = [];
|
|
4
|
+
const entries = new Set();
|
|
4
5
|
let activeTotal = 0;
|
|
6
|
+
let activeMemoryPruneTotal = 0;
|
|
5
7
|
let exclusive = false;
|
|
6
|
-
const queuedFor = (handle) => queue.filter((entry) => !entry.released
|
|
8
|
+
const queuedFor = (handle, executionClass) => queue.filter((entry) => !entry.released
|
|
9
|
+
&& entry.handle === handle
|
|
10
|
+
&& (entry.executionClass === "memory_prune") === (executionClass === "memory_prune")).length;
|
|
7
11
|
const activeKey = (handle, executionClass) => `${handle}:${executionClass}`;
|
|
8
12
|
const maxParallelFor = (executionClass) => executionClass === "scheduled"
|
|
9
13
|
? limits.maxParallelScheduledPerAgent
|
|
10
|
-
:
|
|
14
|
+
: executionClass === "memory_prune"
|
|
15
|
+
? 1
|
|
16
|
+
: limits.maxParallelPerAgent;
|
|
11
17
|
const promote = () => {
|
|
12
|
-
while (
|
|
13
|
-
const
|
|
14
|
-
|
|
15
|
-
|
|
18
|
+
while (true) {
|
|
19
|
+
const foregroundIndex = activeTotal < limits.maxParallelTotal
|
|
20
|
+
? queue.findIndex((entry) => !entry.released
|
|
21
|
+
&& entry.executionClass !== "memory_prune"
|
|
22
|
+
&& (activeByHandleAndClass.get(activeKey(entry.handle, entry.executionClass)) ?? 0)
|
|
23
|
+
< maxParallelFor(entry.executionClass))
|
|
24
|
+
: -1;
|
|
25
|
+
const memoryPruneIndex = activeMemoryPruneTotal < 1
|
|
26
|
+
? queue.findIndex((entry) => !entry.released
|
|
27
|
+
&& entry.executionClass === "memory_prune"
|
|
28
|
+
&& (activeByHandleAndClass.get(activeKey(entry.handle, entry.executionClass)) ?? 0) < 1)
|
|
29
|
+
: -1;
|
|
30
|
+
const index = foregroundIndex >= 0 ? foregroundIndex : memoryPruneIndex;
|
|
16
31
|
if (index < 0)
|
|
17
32
|
return;
|
|
18
33
|
const [next] = queue.splice(index, 1);
|
|
19
34
|
if (!next || next.released)
|
|
20
35
|
continue;
|
|
21
36
|
next.promoted = true;
|
|
22
|
-
|
|
37
|
+
next.phase = "admitted";
|
|
38
|
+
if (next.executionClass === "memory_prune")
|
|
39
|
+
activeMemoryPruneTotal += 1;
|
|
40
|
+
else
|
|
41
|
+
activeTotal += 1;
|
|
23
42
|
const key = activeKey(next.handle, next.executionClass);
|
|
24
43
|
activeByHandleAndClass.set(key, (activeByHandleAndClass.get(key) ?? 0) + 1);
|
|
25
44
|
next.resolve();
|
|
@@ -29,12 +48,13 @@ export function createSharedSlotManager(limits) {
|
|
|
29
48
|
reserve: (handle, kind, executionClass = "normal") => {
|
|
30
49
|
const key = activeKey(handle, executionClass);
|
|
31
50
|
const activeForAgent = activeByHandleAndClass.get(key) ?? 0;
|
|
32
|
-
const queuedForAgent = queuedFor(handle);
|
|
51
|
+
const queuedForAgent = queuedFor(handle, executionClass);
|
|
52
|
+
const laneQueuedTotal = queue.filter((entry) => (entry.executionClass === "memory_prune") === (executionClass === "memory_prune")).length;
|
|
33
53
|
const facts = {
|
|
34
54
|
activeForAgent,
|
|
35
55
|
queuedForAgent,
|
|
36
|
-
activeTotal,
|
|
37
|
-
queuedTotal:
|
|
56
|
+
activeTotal: executionClass === "memory_prune" ? activeMemoryPruneTotal : activeTotal,
|
|
57
|
+
queuedTotal: laneQueuedTotal,
|
|
38
58
|
};
|
|
39
59
|
if (exclusive) {
|
|
40
60
|
return {
|
|
@@ -42,19 +62,27 @@ export function createSharedSlotManager(limits) {
|
|
|
42
62
|
facts,
|
|
43
63
|
ready: Promise.resolve(),
|
|
44
64
|
isQueued: () => false,
|
|
65
|
+
markPreparing: () => { },
|
|
66
|
+
markStarting: () => { },
|
|
67
|
+
markRunning: () => { },
|
|
45
68
|
release: () => { },
|
|
46
69
|
};
|
|
47
70
|
}
|
|
48
|
-
const canStartImmediately =
|
|
71
|
+
const canStartImmediately = (executionClass === "memory_prune"
|
|
72
|
+
? activeMemoryPruneTotal < 1
|
|
73
|
+
: activeTotal < limits.maxParallelTotal)
|
|
49
74
|
&& activeForAgent < maxParallelFor(executionClass)
|
|
50
|
-
&&
|
|
75
|
+
&& laneQueuedTotal === 0;
|
|
51
76
|
if (kind === "execution" && !canStartImmediately && (queuedForAgent >= limits.maxQueuedPerAgent
|
|
52
|
-
||
|
|
77
|
+
|| laneQueuedTotal >= limits.maxQueuedTotal)) {
|
|
53
78
|
return {
|
|
54
79
|
accepted: false,
|
|
55
80
|
facts,
|
|
56
81
|
ready: Promise.resolve(),
|
|
57
82
|
isQueued: () => false,
|
|
83
|
+
markPreparing: () => { },
|
|
84
|
+
markStarting: () => { },
|
|
85
|
+
markRunning: () => { },
|
|
58
86
|
release: () => { },
|
|
59
87
|
};
|
|
60
88
|
}
|
|
@@ -65,22 +93,36 @@ export function createSharedSlotManager(limits) {
|
|
|
65
93
|
executionClass,
|
|
66
94
|
released: false,
|
|
67
95
|
promoted: false,
|
|
96
|
+
phase: "queued",
|
|
68
97
|
resolve: resolveReady,
|
|
69
98
|
};
|
|
70
99
|
queue.push(entry);
|
|
100
|
+
entries.add(entry);
|
|
71
101
|
promote();
|
|
102
|
+
const markPhase = (phase) => {
|
|
103
|
+
if (!entry.released && entry.promoted)
|
|
104
|
+
entry.phase = phase;
|
|
105
|
+
};
|
|
72
106
|
return {
|
|
73
107
|
accepted: true,
|
|
74
108
|
facts,
|
|
75
109
|
state: entry.promoted ? "ready" : "queued",
|
|
76
110
|
ready,
|
|
77
111
|
isQueued: () => !entry.promoted && !entry.released,
|
|
112
|
+
markPreparing: () => markPhase("preparing"),
|
|
113
|
+
markStarting: () => markPhase("starting"),
|
|
114
|
+
markRunning: () => markPhase("running"),
|
|
78
115
|
release: () => {
|
|
79
116
|
if (entry.released)
|
|
80
117
|
return;
|
|
81
118
|
entry.released = true;
|
|
119
|
+
entries.delete(entry);
|
|
82
120
|
if (entry.promoted) {
|
|
83
|
-
|
|
121
|
+
if (entry.executionClass === "memory_prune") {
|
|
122
|
+
activeMemoryPruneTotal = Math.max(0, activeMemoryPruneTotal - 1);
|
|
123
|
+
}
|
|
124
|
+
else
|
|
125
|
+
activeTotal = Math.max(0, activeTotal - 1);
|
|
84
126
|
const nextForAgent = Math.max(0, (activeByHandleAndClass.get(key) ?? 1) - 1);
|
|
85
127
|
if (nextForAgent === 0)
|
|
86
128
|
activeByHandleAndClass.delete(key);
|
|
@@ -96,9 +138,26 @@ export function createSharedSlotManager(limits) {
|
|
|
96
138
|
},
|
|
97
139
|
};
|
|
98
140
|
},
|
|
99
|
-
snapshot: () =>
|
|
141
|
+
snapshot: () => {
|
|
142
|
+
const foreground = [...entries].filter((entry) => entry.executionClass !== "memory_prune");
|
|
143
|
+
const memoryPrune = [...entries].filter((entry) => entry.executionClass === "memory_prune");
|
|
144
|
+
const runningTotal = foreground.filter((entry) => entry.phase === "running").length;
|
|
145
|
+
return {
|
|
146
|
+
activeTotal: runningTotal,
|
|
147
|
+
queuedTotal: foreground.filter((entry) => entry.phase === "queued").length,
|
|
148
|
+
admittedTotal: foreground.filter((entry) => entry.promoted).length,
|
|
149
|
+
preparingTotal: foreground.filter((entry) => entry.phase === "preparing" || entry.phase === "admitted").length,
|
|
150
|
+
startingTotal: foreground.filter((entry) => entry.phase === "starting").length,
|
|
151
|
+
runningTotal,
|
|
152
|
+
memoryPruneAdmittedTotal: memoryPrune.filter((entry) => entry.promoted).length,
|
|
153
|
+
memoryPruneQueuedTotal: memoryPrune.filter((entry) => entry.phase === "queued").length,
|
|
154
|
+
memoryPrunePreparingTotal: memoryPrune.filter((entry) => entry.phase === "preparing" || entry.phase === "admitted").length,
|
|
155
|
+
memoryPruneStartingTotal: memoryPrune.filter((entry) => entry.phase === "starting").length,
|
|
156
|
+
memoryPruneRunningTotal: memoryPrune.filter((entry) => entry.phase === "running").length,
|
|
157
|
+
};
|
|
158
|
+
},
|
|
100
159
|
tryAcquireExclusive: () => {
|
|
101
|
-
if (exclusive || activeTotal !== 0 || queue.length !== 0)
|
|
160
|
+
if (exclusive || activeTotal !== 0 || activeMemoryPruneTotal !== 0 || queue.length !== 0)
|
|
102
161
|
return null;
|
|
103
162
|
exclusive = true;
|
|
104
163
|
let released = false;
|