@nowcrew/daemon 0.5.29 → 0.5.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -1
- package/dist/config.js +13 -1
- package/dist/execution-journal.js +96 -5
- package/dist/execution-protocol.js +1 -0
- package/dist/execution-runner.js +52 -29
- package/dist/host-execution-coordinator.js +241 -0
- package/dist/local-executor.js +78 -0
- package/dist/machine-info.js +16 -1
- package/dist/remote/claude-bridge.js +402 -0
- package/dist/remote/claude-channel.js +164 -0
- package/dist/remote/codex-client.js +408 -0
- package/dist/remote/codex-runtime.js +77 -0
- package/dist/remote/config.js +83 -0
- package/dist/remote/gateway.js +572 -0
- package/dist/remote/protocol.js +178 -0
- package/dist/remote/remote-cli.js +233 -0
- package/dist/remote/session-discovery.js +249 -0
- package/dist/remote/wrapper.js +40 -0
- package/dist/runner.js +4 -0
- package/dist/runtime-startup-gate.js +91 -0
- package/dist/runtimes/codex-app-server-runner.js +226 -29
- package/dist/serve.js +70 -8
- package/dist/shared-execution-slots.js +48 -33
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -75,6 +75,10 @@ The server selects the machine and sends a validated `execution:start` containin
|
|
|
75
75
|
- requested permission and channel/thread identifiers;
|
|
76
76
|
- reporting flags for final text, activity, and console streams.
|
|
77
77
|
|
|
78
|
+
An Agent bound to a computer has a hard machine constraint: if that computer is offline or incompatible,
|
|
79
|
+
dispatch reports unavailable instead of running on another compatible daemon. An unbound Agent may use
|
|
80
|
+
the global compatible-machine pool.
|
|
81
|
+
|
|
78
82
|
The daemon intersects requested permission with local policy, checks advertised resource limits, prepares
|
|
79
83
|
the local workspace/environment, and launches only a built-in runtime adapter (`claude`, `codex`, or
|
|
80
84
|
`kimi`). It does not decide collaboration rules, scheduled output policy, fallback delivery, or thread
|
|
@@ -134,12 +138,23 @@ Useful environment controls:
|
|
|
134
138
|
```text
|
|
135
139
|
CREW_RUNTIME_SAFE=1
|
|
136
140
|
CREW_EXECUTION_MAX_PROMPT_BYTES=256000
|
|
137
|
-
CREW_EXECUTION_MAX_TIMEOUT_MS=
|
|
141
|
+
CREW_EXECUTION_MAX_TIMEOUT_MS=10800000
|
|
138
142
|
CREW_EXECUTION_MAX_EVENT_BYTES=64000
|
|
139
143
|
CREW_MAX_PARALLEL=4
|
|
140
144
|
CREW_EXECUTION_MAX_QUEUED_PER_AGENT=32
|
|
145
|
+
CREW_EXECUTION_MAX_PARALLEL_TOTAL=4
|
|
146
|
+
CREW_EXECUTION_MAX_QUEUED_TOTAL=128
|
|
147
|
+
CREW_EXECUTION_MAX_STARTING_TOTAL=1
|
|
148
|
+
CREW_EXECUTION_MAX_STARTING_PER_RUNTIME=1
|
|
149
|
+
CREW_EXECUTION_START_GAP_MS=3000
|
|
150
|
+
CREW_EXECUTION_STARTUP_TIMEOUT_MS=60000
|
|
141
151
|
```
|
|
142
152
|
|
|
153
|
+
`MAX_PARALLEL_TOTAL` is the machine-wide process cap shared by protocol-v1 and legacy work.
|
|
154
|
+
Runtime startup is a separate FIFO gate: by default only one Claude, Codex, or Kimi process may be
|
|
155
|
+
initializing at a time, and launches are spaced by three seconds. A process leaves the startup gate
|
|
156
|
+
only after its runtime-specific ready event; startup timeout cancels the owned process tree.
|
|
157
|
+
|
|
143
158
|
Local policy can reduce server-requested access and limits; it cannot grant more access than requested.
|
|
144
159
|
Provider credentials and configured environment are prepared locally and never carried in execution
|
|
145
160
|
control frames.
|
|
@@ -148,6 +163,7 @@ control frames.
|
|
|
148
163
|
|
|
149
164
|
- `src/serve.ts`: connection, negotiation, routing, sync, and legacy boundary.
|
|
150
165
|
- `src/execution-runner.ts`: spec admission and lifecycle reporting.
|
|
166
|
+
- `src/shared-execution-slots.ts` and `src/runtime-startup-gate.ts`: machine concurrency and startup FIFO.
|
|
151
167
|
- `src/local-executor.ts` and `src/execution-supervisor.ts`: runtime process boundary.
|
|
152
168
|
- `src/execution-journal.ts`: crash-safe local execution facts.
|
|
153
169
|
- `src/runner.ts`: protocol-0 compatibility runner.
|
package/dist/config.js
CHANGED
|
@@ -7,10 +7,16 @@ import { detectDaemonLang, translateDaemon } from "./i18n.js";
|
|
|
7
7
|
import { resolveAgentsRoot } from "./computer-profile.js";
|
|
8
8
|
export const DEFAULT_EXECUTION_LIMITS = Object.freeze({
|
|
9
9
|
maxPromptBytes: 256_000,
|
|
10
|
-
maxTimeoutMs:
|
|
10
|
+
maxTimeoutMs: 3 * 60 * 60_000,
|
|
11
11
|
maxEventBytes: 64_000,
|
|
12
12
|
maxParallelPerAgent: 4,
|
|
13
13
|
maxQueuedPerAgent: 32,
|
|
14
|
+
maxParallelTotal: 4,
|
|
15
|
+
maxQueuedTotal: 128,
|
|
16
|
+
maxStartingTotal: 1,
|
|
17
|
+
maxStartingPerRuntime: 1,
|
|
18
|
+
startupGapMs: 3_000,
|
|
19
|
+
startupTimeoutMs: 60_000,
|
|
14
20
|
});
|
|
15
21
|
// Fits the largest mandatory v1 lifecycle envelope (UUID + timestamps + outcome facts) with margin.
|
|
16
22
|
export const MIN_EXECUTION_EVENT_BYTES = 512;
|
|
@@ -62,6 +68,12 @@ export function loadConfig(env = process.env) {
|
|
|
62
68
|
maxEventBytes: integerEnvAtLeast(env, "CREW_EXECUTION_MAX_EVENT_BYTES", DEFAULT_EXECUTION_LIMITS.maxEventBytes, MIN_EXECUTION_EVENT_BYTES),
|
|
63
69
|
maxParallelPerAgent: positiveIntegerEnv(env, "CREW_MAX_PARALLEL", DEFAULT_EXECUTION_LIMITS.maxParallelPerAgent),
|
|
64
70
|
maxQueuedPerAgent: positiveIntegerEnv(env, "CREW_EXECUTION_MAX_QUEUED_PER_AGENT", DEFAULT_EXECUTION_LIMITS.maxQueuedPerAgent),
|
|
71
|
+
maxParallelTotal: positiveIntegerEnv(env, "CREW_EXECUTION_MAX_PARALLEL_TOTAL", DEFAULT_EXECUTION_LIMITS.maxParallelTotal),
|
|
72
|
+
maxQueuedTotal: positiveIntegerEnv(env, "CREW_EXECUTION_MAX_QUEUED_TOTAL", DEFAULT_EXECUTION_LIMITS.maxQueuedTotal),
|
|
73
|
+
maxStartingTotal: positiveIntegerEnv(env, "CREW_EXECUTION_MAX_STARTING_TOTAL", DEFAULT_EXECUTION_LIMITS.maxStartingTotal),
|
|
74
|
+
maxStartingPerRuntime: positiveIntegerEnv(env, "CREW_EXECUTION_MAX_STARTING_PER_RUNTIME", DEFAULT_EXECUTION_LIMITS.maxStartingPerRuntime),
|
|
75
|
+
startupGapMs: positiveIntegerEnv(env, "CREW_EXECUTION_START_GAP_MS", DEFAULT_EXECUTION_LIMITS.startupGapMs),
|
|
76
|
+
startupTimeoutMs: positiveIntegerEnv(env, "CREW_EXECUTION_STARTUP_TIMEOUT_MS", DEFAULT_EXECUTION_LIMITS.startupTimeoutMs),
|
|
65
77
|
});
|
|
66
78
|
return {
|
|
67
79
|
serverUrl,
|
|
@@ -8,6 +8,7 @@ export { JournalLockedError, JournalLockCorruptionError } from "./execution-jour
|
|
|
8
8
|
const ExecutionIdSchema = z.string().uuid();
|
|
9
9
|
const TimestampSchema = z.string().datetime({ offset: true });
|
|
10
10
|
const RuntimeSchema = z.enum(["claude", "codex", "kimi"]);
|
|
11
|
+
const RUNTIME_READY_SUFFIX = ".runtime-ready";
|
|
11
12
|
const RawJournalEntrySchema = z.object({
|
|
12
13
|
executionId: ExecutionIdSchema,
|
|
13
14
|
specHash: z.string().min(1),
|
|
@@ -21,6 +22,8 @@ const RawJournalEntrySchema = z.object({
|
|
|
21
22
|
resumed: z.boolean(),
|
|
22
23
|
acceptedAt: TimestampSchema,
|
|
23
24
|
processStartedAt: TimestampSchema.nullable(),
|
|
25
|
+
// Stored in a non-JSON sidecar so older strict journal readers can still roll back.
|
|
26
|
+
runtimeReadyAt: TimestampSchema.nullable().default(null),
|
|
24
27
|
processIdentity: z.string().min(1).refine((value) => value.trim().length > 0).nullable(),
|
|
25
28
|
// Optional for journals written before execution permission persistence was introduced.
|
|
26
29
|
effectivePermission: EffectivePermissionSchema.nullable().optional(),
|
|
@@ -35,6 +38,9 @@ export const JournalEntrySchema = RawJournalEntrySchema.superRefine((entry, ctx)
|
|
|
35
38
|
if (entry.processStartedAt !== null) {
|
|
36
39
|
issue("accepted entry cannot have processStartedAt", "processStartedAt");
|
|
37
40
|
}
|
|
41
|
+
if (entry.runtimeReadyAt !== null) {
|
|
42
|
+
issue("accepted entry cannot have runtimeReadyAt", "runtimeReadyAt");
|
|
43
|
+
}
|
|
38
44
|
if (entry.processIdentity !== null)
|
|
39
45
|
issue("accepted entry cannot have processIdentity", "processIdentity");
|
|
40
46
|
if (entry.completion !== null)
|
|
@@ -290,6 +296,7 @@ export function createExecutionJournal(agentsRoot, options = {}) {
|
|
|
290
296
|
throw new RangeError("maxEntries must be an integer");
|
|
291
297
|
let initialized = false;
|
|
292
298
|
const recordPath = (executionId) => join(directory, `${executionId}.json`);
|
|
299
|
+
const runtimeReadyPath = (executionId) => join(directory, `${executionId}${RUNTIME_READY_SUFFIX}`);
|
|
293
300
|
const parseRecord = (path, raw) => {
|
|
294
301
|
try {
|
|
295
302
|
const entry = JournalEntrySchema.parse(JSON.parse(raw));
|
|
@@ -303,10 +310,30 @@ export function createExecutionJournal(agentsRoot, options = {}) {
|
|
|
303
310
|
throw new JournalCorruptionError(path, error);
|
|
304
311
|
}
|
|
305
312
|
};
|
|
313
|
+
const readRuntimeReadyAt = async (executionId, path) => {
|
|
314
|
+
try {
|
|
315
|
+
return TimestampSchema.parse((await readFile(runtimeReadyPath(executionId), "utf8")).trim());
|
|
316
|
+
}
|
|
317
|
+
catch (error) {
|
|
318
|
+
if (isMissingFile(error))
|
|
319
|
+
return null;
|
|
320
|
+
throw new JournalCorruptionError(path, error);
|
|
321
|
+
}
|
|
322
|
+
};
|
|
323
|
+
const readRecordPath = async (path) => {
|
|
324
|
+
const entry = parseRecord(path, await readFile(path, "utf8"));
|
|
325
|
+
const runtimeReadyAt = await readRuntimeReadyAt(entry.executionId, path);
|
|
326
|
+
try {
|
|
327
|
+
return JournalEntrySchema.parse({ ...entry, runtimeReadyAt });
|
|
328
|
+
}
|
|
329
|
+
catch (error) {
|
|
330
|
+
throw new JournalCorruptionError(path, error);
|
|
331
|
+
}
|
|
332
|
+
};
|
|
306
333
|
const readRecord = async (executionId) => {
|
|
307
334
|
const path = recordPath(executionId);
|
|
308
335
|
try {
|
|
309
|
-
return
|
|
336
|
+
return await readRecordPath(path);
|
|
310
337
|
}
|
|
311
338
|
catch (error) {
|
|
312
339
|
if (isMissingFile(error))
|
|
@@ -321,7 +348,7 @@ export function createExecutionJournal(agentsRoot, options = {}) {
|
|
|
321
348
|
const entries = [];
|
|
322
349
|
for (const name of names) {
|
|
323
350
|
const path = join(directory, name);
|
|
324
|
-
entries.push(
|
|
351
|
+
entries.push(await readRecordPath(path));
|
|
325
352
|
}
|
|
326
353
|
return entries;
|
|
327
354
|
};
|
|
@@ -385,13 +412,49 @@ export function createExecutionJournal(agentsRoot, options = {}) {
|
|
|
385
412
|
await fsyncDirectory(directory);
|
|
386
413
|
return diskEntry;
|
|
387
414
|
};
|
|
415
|
+
const writeRuntimeReadyAt = async (executionId, runtimeReadyAt) => {
|
|
416
|
+
const finalPath = runtimeReadyPath(executionId);
|
|
417
|
+
if (runtimeReadyAt === null) {
|
|
418
|
+
try {
|
|
419
|
+
await unlink(finalPath);
|
|
420
|
+
await fsyncDirectory(directory);
|
|
421
|
+
}
|
|
422
|
+
catch (error) {
|
|
423
|
+
if (!isMissingFile(error))
|
|
424
|
+
throw error;
|
|
425
|
+
}
|
|
426
|
+
return;
|
|
427
|
+
}
|
|
428
|
+
const temporaryPath = `${finalPath}.tmp`;
|
|
429
|
+
let handle = null;
|
|
430
|
+
try {
|
|
431
|
+
handle = await open(temporaryPath, "w", 0o600);
|
|
432
|
+
await handle.writeFile(`${runtimeReadyAt}\n`, "utf8");
|
|
433
|
+
await handle.sync();
|
|
434
|
+
await handle.close();
|
|
435
|
+
handle = null;
|
|
436
|
+
await rename(temporaryPath, finalPath);
|
|
437
|
+
await fsyncFinalFile(finalPath);
|
|
438
|
+
await fsyncDirectory(directory);
|
|
439
|
+
}
|
|
440
|
+
catch (error) {
|
|
441
|
+
if (handle !== null)
|
|
442
|
+
await handle.close().catch(() => undefined);
|
|
443
|
+
await rm(temporaryPath, { force: true }).catch(() => undefined);
|
|
444
|
+
throw error;
|
|
445
|
+
}
|
|
446
|
+
};
|
|
388
447
|
const writeRecord = async (entry) => {
|
|
389
448
|
const validated = JournalEntrySchema.parse(entry);
|
|
449
|
+
const { runtimeReadyAt, ...diskEntry } = validated;
|
|
390
450
|
const temporaryPath = join(directory, `${entry.executionId}.tmp`);
|
|
391
451
|
let handle = null;
|
|
392
452
|
try {
|
|
453
|
+
// Persist the sidecar first. A crash before the JSON rename can expose a
|
|
454
|
+
// ready timestamp early, but can never lose an already-observed ready event.
|
|
455
|
+
await writeRuntimeReadyAt(entry.executionId, runtimeReadyAt);
|
|
393
456
|
handle = await open(temporaryPath, "w", 0o600);
|
|
394
|
-
await handle.writeFile(`${JSON.stringify(
|
|
457
|
+
await handle.writeFile(`${JSON.stringify(diskEntry, null, 2)}\n`, "utf8");
|
|
395
458
|
await handle.sync();
|
|
396
459
|
await handle.close();
|
|
397
460
|
handle = null;
|
|
@@ -420,6 +483,7 @@ export function createExecutionJournal(agentsRoot, options = {}) {
|
|
|
420
483
|
const toDelete = new Set([...expired, ...overLimit].map((entry) => entry.executionId));
|
|
421
484
|
for (const executionId of toDelete) {
|
|
422
485
|
await unlink(recordPath(executionId));
|
|
486
|
+
await rm(runtimeReadyPath(executionId), { force: true });
|
|
423
487
|
}
|
|
424
488
|
if (toDelete.size > 0)
|
|
425
489
|
await fsyncDirectory(directory);
|
|
@@ -433,6 +497,13 @@ export function createExecutionJournal(agentsRoot, options = {}) {
|
|
|
433
497
|
for (const name of names.filter((candidate) => candidate.endsWith(".tmp"))) {
|
|
434
498
|
await rm(join(directory, name), { force: true });
|
|
435
499
|
}
|
|
500
|
+
const records = new Set(names.filter((name) => name.endsWith(".json"))
|
|
501
|
+
.map((name) => basename(name, ".json")));
|
|
502
|
+
for (const name of names.filter((candidate) => candidate.endsWith(RUNTIME_READY_SUFFIX))) {
|
|
503
|
+
const executionId = name.slice(0, -RUNTIME_READY_SUFFIX.length);
|
|
504
|
+
if (!records.has(executionId))
|
|
505
|
+
await rm(join(directory, name), { force: true });
|
|
506
|
+
}
|
|
436
507
|
await pruneInternal();
|
|
437
508
|
initialized = true;
|
|
438
509
|
};
|
|
@@ -448,8 +519,8 @@ export function createExecutionJournal(agentsRoot, options = {}) {
|
|
|
448
519
|
return entry;
|
|
449
520
|
};
|
|
450
521
|
const interruptedCompletion = (entry) => {
|
|
451
|
-
const startedAt = entry.state === "running" && entry.
|
|
452
|
-
? entry.
|
|
522
|
+
const startedAt = entry.state === "running" && entry.runtimeReadyAt !== null
|
|
523
|
+
? entry.runtimeReadyAt
|
|
453
524
|
: entry.acceptedAt;
|
|
454
525
|
return ExecutionCompletedSchema.parse({
|
|
455
526
|
type: "execution:completed",
|
|
@@ -551,6 +622,7 @@ export function createExecutionJournal(agentsRoot, options = {}) {
|
|
|
551
622
|
resumed: recoveryFacts.resumed ?? false,
|
|
552
623
|
acceptedAt: timestamp,
|
|
553
624
|
processStartedAt: null,
|
|
625
|
+
runtimeReadyAt: null,
|
|
554
626
|
processIdentity: null,
|
|
555
627
|
effectivePermission: recoveryFacts.effectivePermission ?? null,
|
|
556
628
|
});
|
|
@@ -618,6 +690,25 @@ export function createExecutionJournal(agentsRoot, options = {}) {
|
|
|
618
690
|
}
|
|
619
691
|
});
|
|
620
692
|
},
|
|
693
|
+
markRuntimeReady: async (executionId, runtimeReadyAt) => {
|
|
694
|
+
validateExecutionId(executionId);
|
|
695
|
+
TimestampSchema.parse(runtimeReadyAt);
|
|
696
|
+
return serialized(async () => {
|
|
697
|
+
const entry = await requireRecord(executionId);
|
|
698
|
+
if (entry.state !== "running") {
|
|
699
|
+
throw new JournalTransitionError(`Cannot mark ${entry.state} execution runtime-ready`);
|
|
700
|
+
}
|
|
701
|
+
if (entry.runtimeReadyAt !== null)
|
|
702
|
+
return confirmDurable(entry);
|
|
703
|
+
const updated = JournalEntrySchema.parse({
|
|
704
|
+
...entry,
|
|
705
|
+
runtimeReadyAt,
|
|
706
|
+
updatedAt: now().toISOString(),
|
|
707
|
+
});
|
|
708
|
+
await writeRecord(updated);
|
|
709
|
+
return updated;
|
|
710
|
+
});
|
|
711
|
+
},
|
|
621
712
|
complete: async (executionId, completionInput) => {
|
|
622
713
|
validateExecutionId(executionId);
|
|
623
714
|
const completion = ExecutionCompletedSchema.parse(completionInput);
|
|
@@ -121,6 +121,7 @@ export const ExecutionStartSchema = z.object({
|
|
|
121
121
|
channelId: z.string().min(1),
|
|
122
122
|
threadId: z.string().min(1).optional(),
|
|
123
123
|
wakeMessageId: z.string().min(1).optional(),
|
|
124
|
+
scheduledRunId: ExecutionIdSchema.optional(),
|
|
124
125
|
externalResponseSessionId: ExecutionIdSchema.optional(),
|
|
125
126
|
answerStream: z.boolean().optional(),
|
|
126
127
|
attachments: z.array(ExecutionAttachmentSchema).max(20).optional(),
|
package/dist/execution-runner.js
CHANGED
|
@@ -217,18 +217,26 @@ function admission(spec, config, dependencies, at) {
|
|
|
217
217
|
}
|
|
218
218
|
if (!Number.isInteger(dependencies.facts.activeForAgent)
|
|
219
219
|
|| !Number.isInteger(dependencies.facts.queuedForAgent)
|
|
220
|
+
|| !Number.isInteger(dependencies.facts.activeTotal)
|
|
221
|
+
|| !Number.isInteger(dependencies.facts.queuedTotal)
|
|
220
222
|
|| dependencies.facts.activeForAgent < 0
|
|
221
|
-
|| dependencies.facts.queuedForAgent < 0
|
|
223
|
+
|| dependencies.facts.queuedForAgent < 0
|
|
224
|
+
|| dependencies.facts.activeTotal < 0
|
|
225
|
+
|| dependencies.facts.queuedTotal < 0) {
|
|
222
226
|
return { rejected: rejection(spec.executionId, "invalid_spec", "Invalid local resource facts", at) };
|
|
223
227
|
}
|
|
228
|
+
const agentAtCapacity = dependencies.facts.activeForAgent >= config.executionLimits.maxParallelPerAgent;
|
|
229
|
+
const machineAtCapacity = dependencies.facts.activeTotal >= config.executionLimits.maxParallelTotal;
|
|
224
230
|
const slotInvalid = dependencies.slot?.state === "ready"
|
|
225
|
-
?
|
|
231
|
+
? agentAtCapacity || machineAtCapacity
|
|
226
232
|
: dependencies.slot?.state === "queued"
|
|
227
|
-
? dependencies.facts.
|
|
228
|
-
|| dependencies.facts.
|
|
233
|
+
? dependencies.facts.queuedForAgent >= config.executionLimits.maxQueuedPerAgent
|
|
234
|
+
|| dependencies.facts.queuedTotal >= config.executionLimits.maxQueuedTotal
|
|
229
235
|
: false;
|
|
230
|
-
if (slotInvalid || (dependencies.slot === undefined && (
|
|
231
|
-
||
|
|
236
|
+
if (slotInvalid || (dependencies.slot === undefined && (agentAtCapacity
|
|
237
|
+
|| machineAtCapacity
|
|
238
|
+
|| dependencies.facts.queuedForAgent >= config.executionLimits.maxQueuedPerAgent
|
|
239
|
+
|| dependencies.facts.queuedTotal >= config.executionLimits.maxQueuedTotal))) {
|
|
232
240
|
return { rejected: rejection(spec.executionId, "resource_limit", "Local execution capacity is exhausted", at) };
|
|
233
241
|
}
|
|
234
242
|
const permission = effectivePermission(spec, config);
|
|
@@ -306,12 +314,12 @@ export async function runExecution(config, input, dependencies) {
|
|
|
306
314
|
effectivePermission: replayPermission,
|
|
307
315
|
at: replay.acceptedAt,
|
|
308
316
|
}), config.executionLimits.maxEventBytes), bestEffortTimeoutMs);
|
|
309
|
-
if (replay.state === "running" && replay.
|
|
317
|
+
if (replay.state === "running" && replay.runtimeReadyAt !== null) {
|
|
310
318
|
await reportBestEffort(dependencies.report, boundExecutionFrame(DaemonToServerExecutionFrameSchema.parse({
|
|
311
319
|
type: "execution:started",
|
|
312
320
|
protocolVersion: 1,
|
|
313
321
|
executionId: spec.executionId,
|
|
314
|
-
at: replay.
|
|
322
|
+
at: replay.runtimeReadyAt,
|
|
315
323
|
}), config.executionLimits.maxEventBytes), bestEffortTimeoutMs);
|
|
316
324
|
}
|
|
317
325
|
return { kind: "existing", entry: replay };
|
|
@@ -344,12 +352,12 @@ export async function runExecution(config, input, dependencies) {
|
|
|
344
352
|
effectivePermission: accepted.effectivePermission ?? permission,
|
|
345
353
|
at: accepted.acceptedAt,
|
|
346
354
|
}), config.executionLimits.maxEventBytes), bestEffortTimeoutMs);
|
|
347
|
-
if (accepted.state === "running" && accepted.
|
|
355
|
+
if (accepted.state === "running" && accepted.runtimeReadyAt !== null) {
|
|
348
356
|
await reportBestEffort(dependencies.report, boundExecutionFrame(DaemonToServerExecutionFrameSchema.parse({
|
|
349
357
|
type: "execution:started",
|
|
350
358
|
protocolVersion: 1,
|
|
351
359
|
executionId: spec.executionId,
|
|
352
|
-
at: accepted.
|
|
360
|
+
at: accepted.runtimeReadyAt,
|
|
353
361
|
}), config.executionLimits.maxEventBytes), bestEffortTimeoutMs);
|
|
354
362
|
}
|
|
355
363
|
return { kind: "existing", entry: accepted };
|
|
@@ -376,6 +384,7 @@ export async function runExecution(config, input, dependencies) {
|
|
|
376
384
|
await Promise.all([...launchAttempts]);
|
|
377
385
|
};
|
|
378
386
|
let startedAt = accepted.acceptedAt;
|
|
387
|
+
let runtimeCancel = null;
|
|
379
388
|
let timeout;
|
|
380
389
|
let timedOut = false;
|
|
381
390
|
let completion;
|
|
@@ -396,6 +405,33 @@ export async function runExecution(config, input, dependencies) {
|
|
|
396
405
|
let consoleSequence = 0;
|
|
397
406
|
let externalOutputSequence = 0;
|
|
398
407
|
const callbacks = {
|
|
408
|
+
onRuntimeReady: async () => {
|
|
409
|
+
if (dependencies.cancellation?.isRequested())
|
|
410
|
+
return;
|
|
411
|
+
const ready = await dependencies.journal.markRuntimeReady(spec.executionId, now().toISOString());
|
|
412
|
+
if (ready.runtimeReadyAt === null) {
|
|
413
|
+
throw new Error(`Execution ${spec.executionId} runtime readiness was not persisted`);
|
|
414
|
+
}
|
|
415
|
+
startedAt = ready.runtimeReadyAt;
|
|
416
|
+
await reportBestEffort(dependencies.report, boundExecutionFrame(DaemonToServerExecutionFrameSchema.parse({
|
|
417
|
+
type: "execution:started",
|
|
418
|
+
protocolVersion: 1,
|
|
419
|
+
executionId: spec.executionId,
|
|
420
|
+
at: startedAt,
|
|
421
|
+
}), config.executionLimits.maxEventBytes), bestEffortTimeoutMs);
|
|
422
|
+
const cancel = runtimeCancel;
|
|
423
|
+
if (cancel !== null && !dependencies.cancellation?.isRequested()) {
|
|
424
|
+
timeout = setTimeout(() => {
|
|
425
|
+
timedOut = true;
|
|
426
|
+
try {
|
|
427
|
+
void cancel().catch(rejectCancellationFailure);
|
|
428
|
+
}
|
|
429
|
+
catch (error) {
|
|
430
|
+
rejectCancellationFailure(error);
|
|
431
|
+
}
|
|
432
|
+
}, effectiveTimeoutMs);
|
|
433
|
+
}
|
|
434
|
+
},
|
|
399
435
|
...(spec.reporting.streamActivity ? {
|
|
400
436
|
onActivity: (activity) => {
|
|
401
437
|
const frame = DaemonToServerExecutionFrameSchema.parse({
|
|
@@ -453,6 +489,10 @@ export async function runExecution(config, input, dependencies) {
|
|
|
453
489
|
};
|
|
454
490
|
const localDependencies = {
|
|
455
491
|
...(dependencies.cancellation === undefined ? {} : { cancellation: dependencies.cancellation }),
|
|
492
|
+
...(dependencies.startupGate === undefined ? {} : { startupGate: dependencies.startupGate }),
|
|
493
|
+
...(dependencies.startupTimeoutMs === undefined
|
|
494
|
+
? {}
|
|
495
|
+
: { startupTimeoutMs: dependencies.startupTimeoutMs }),
|
|
456
496
|
launchRuntime: async (request) => {
|
|
457
497
|
if (launchClosed || dependencies.cancellation?.isRequested())
|
|
458
498
|
throw new ExecutionCancelledError();
|
|
@@ -463,7 +503,7 @@ export async function runExecution(config, input, dependencies) {
|
|
|
463
503
|
const processStartedAt = now().toISOString();
|
|
464
504
|
const launchControl = { cancel: null };
|
|
465
505
|
const guarded = await dependencies.journal.startGuarded(spec.executionId, processStartedAt, () => startSupervisor(supervisorLaunch(request)), {
|
|
466
|
-
beforeRelease: ({
|
|
506
|
+
beforeRelease: ({ handle, abort }) => {
|
|
467
507
|
supervisorState.active = handle;
|
|
468
508
|
let stopPromise = null;
|
|
469
509
|
let releaseStarted = false;
|
|
@@ -481,7 +521,6 @@ export async function runExecution(config, input, dependencies) {
|
|
|
481
521
|
launchControl.cancel = () => stopOnce(releaseStarted ? handle.cancel : abort);
|
|
482
522
|
supervisorState.abortOnce = () => stopOnce(abort);
|
|
483
523
|
dependencies.cancellation?.register(launchControl.cancel);
|
|
484
|
-
startedAt = entry.processStartedAt ?? processStartedAt;
|
|
485
524
|
if (dependencies.cancellation?.isRequested()) {
|
|
486
525
|
return dependencies.cancellation.waitForStop().then(() => {
|
|
487
526
|
throw new ExecutionCancelledError();
|
|
@@ -497,23 +536,7 @@ export async function runExecution(config, input, dependencies) {
|
|
|
497
536
|
throw new Error("Execution launch cancellation gate was not installed");
|
|
498
537
|
}
|
|
499
538
|
const installedCancel = launchControl.cancel;
|
|
500
|
-
|
|
501
|
-
type: "execution:started",
|
|
502
|
-
protocolVersion: 1,
|
|
503
|
-
executionId: spec.executionId,
|
|
504
|
-
at: startedAt,
|
|
505
|
-
}), config.executionLimits.maxEventBytes), bestEffortTimeoutMs);
|
|
506
|
-
if (!dependencies.cancellation?.isRequested()) {
|
|
507
|
-
timeout = setTimeout(() => {
|
|
508
|
-
timedOut = true;
|
|
509
|
-
try {
|
|
510
|
-
void installedCancel().catch(rejectCancellationFailure);
|
|
511
|
-
}
|
|
512
|
-
catch (error) {
|
|
513
|
-
rejectCancellationFailure(error);
|
|
514
|
-
}
|
|
515
|
-
}, effectiveTimeoutMs);
|
|
516
|
-
}
|
|
539
|
+
runtimeCancel = installedCancel;
|
|
517
540
|
return { ...guarded.handle, cancel: installedCancel };
|
|
518
541
|
}
|
|
519
542
|
finally {
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
import { homedir } from "node:os";
|
|
2
|
+
import { join, resolve } from "node:path";
|
|
3
|
+
import { mkdir, open, readFile, rename, rm, writeFile } from "node:fs/promises";
|
|
4
|
+
import { randomUUID } from "node:crypto";
|
|
5
|
+
import { defaultProcessController } from "./execution-journal.js";
|
|
6
|
+
import { createJournalLease, createJournalLeaseRegistry, JournalLockedError, } from "./execution-journal-lock.js";
|
|
7
|
+
const HOST_EXECUTION_SLOT_COUNT = 4;
|
|
8
|
+
const HOST_STARTUP_GAP_MS = 3_000;
|
|
9
|
+
const RETRY_MS = 50;
|
|
10
|
+
export function defaultHostExecutionCoordinatorRoot(userHome = homedir()) {
|
|
11
|
+
// Deliberately independent of CREW_DAEMON_HOME and agentsRoot: every profile
|
|
12
|
+
// owned by this OS user must share the same physical-compute safety boundary.
|
|
13
|
+
return resolve(userHome, ".crew", "daemon", "compute-execution");
|
|
14
|
+
}
|
|
15
|
+
async function syncDirectory(path) {
|
|
16
|
+
if (process.platform === "win32")
|
|
17
|
+
return;
|
|
18
|
+
const handle = await open(path, "r");
|
|
19
|
+
try {
|
|
20
|
+
await handle.sync();
|
|
21
|
+
}
|
|
22
|
+
finally {
|
|
23
|
+
await handle.close();
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
async function leaseFor(directory) {
|
|
27
|
+
await mkdir(directory, { recursive: true, mode: 0o700 });
|
|
28
|
+
const lease = createJournalLease({
|
|
29
|
+
directory,
|
|
30
|
+
currentPid: process.pid,
|
|
31
|
+
captureCurrentIdentity: async () => {
|
|
32
|
+
const identity = await defaultProcessController.inspectIdentity(process.pid);
|
|
33
|
+
if (identity === null)
|
|
34
|
+
throw new Error("current process identity is absent");
|
|
35
|
+
return identity;
|
|
36
|
+
},
|
|
37
|
+
inspectIdentity: (pid) => defaultProcessController.inspectIdentity(pid),
|
|
38
|
+
syncDirectory,
|
|
39
|
+
// A separate registry is intentional: reservations in one process must
|
|
40
|
+
// compete for the same slot instead of attaching to one re-entrant lease.
|
|
41
|
+
registry: createJournalLeaseRegistry(),
|
|
42
|
+
});
|
|
43
|
+
try {
|
|
44
|
+
await lease.acquire();
|
|
45
|
+
return lease;
|
|
46
|
+
}
|
|
47
|
+
catch (error) {
|
|
48
|
+
if (error instanceof JournalLockedError)
|
|
49
|
+
return null;
|
|
50
|
+
throw error;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
function reservation(prerequisite, acquire) {
|
|
54
|
+
let released = false;
|
|
55
|
+
let granted = false;
|
|
56
|
+
let owned = null;
|
|
57
|
+
let releasePromise = null;
|
|
58
|
+
let cancelPrerequisite;
|
|
59
|
+
const prerequisiteCancelled = new Promise((resolveCancelled) => {
|
|
60
|
+
cancelPrerequisite = () => resolveCancelled(false);
|
|
61
|
+
});
|
|
62
|
+
const acquisition = Promise.race([
|
|
63
|
+
prerequisite.then(() => true),
|
|
64
|
+
prerequisiteCancelled,
|
|
65
|
+
]).then(async (shouldAcquire) => {
|
|
66
|
+
if (!shouldAcquire)
|
|
67
|
+
return null;
|
|
68
|
+
const lease = await acquire(() => released);
|
|
69
|
+
if (lease === null)
|
|
70
|
+
return null;
|
|
71
|
+
if (released) {
|
|
72
|
+
await lease.close();
|
|
73
|
+
return null;
|
|
74
|
+
}
|
|
75
|
+
owned = lease;
|
|
76
|
+
granted = true;
|
|
77
|
+
return lease;
|
|
78
|
+
});
|
|
79
|
+
const neverReady = new Promise(() => { });
|
|
80
|
+
const ready = acquisition.then((lease) => lease === null ? neverReady : undefined);
|
|
81
|
+
void ready.catch(() => undefined);
|
|
82
|
+
return {
|
|
83
|
+
ready,
|
|
84
|
+
isQueued: () => !granted && !released,
|
|
85
|
+
release: () => {
|
|
86
|
+
if (releasePromise !== null)
|
|
87
|
+
return releasePromise;
|
|
88
|
+
released = true;
|
|
89
|
+
cancelPrerequisite();
|
|
90
|
+
releasePromise = acquisition.then(async () => { if (owned !== null)
|
|
91
|
+
await owned.close(); });
|
|
92
|
+
void releasePromise.catch(() => undefined);
|
|
93
|
+
return releasePromise;
|
|
94
|
+
},
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
async function writeLaunchTimestamp(path, value) {
|
|
98
|
+
const temporary = `${path}.${randomUUID()}.tmp`;
|
|
99
|
+
try {
|
|
100
|
+
await writeFile(temporary, `${value}\n`, { encoding: "utf8", mode: 0o600, flag: "wx" });
|
|
101
|
+
await rename(temporary, path);
|
|
102
|
+
}
|
|
103
|
+
finally {
|
|
104
|
+
await rm(temporary, { force: true });
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
export function createHostExecutionCoordinator(options = {}) {
|
|
108
|
+
const root = resolve(options.root ?? defaultHostExecutionCoordinatorRoot());
|
|
109
|
+
const executionSlots = options.executionSlots ?? HOST_EXECUTION_SLOT_COUNT;
|
|
110
|
+
const startupGapMs = options.startupGapMs ?? HOST_STARTUP_GAP_MS;
|
|
111
|
+
const retryMs = options.retryMs ?? RETRY_MS;
|
|
112
|
+
const now = options.now ?? Date.now;
|
|
113
|
+
const wait = options.wait ?? ((milliseconds) => new Promise((resolveWait) => setTimeout(resolveWait, milliseconds)));
|
|
114
|
+
const pendingReleases = new Set();
|
|
115
|
+
if (!Number.isSafeInteger(executionSlots) || executionSlots <= 0) {
|
|
116
|
+
throw new RangeError("executionSlots must be a positive safe integer");
|
|
117
|
+
}
|
|
118
|
+
if (!Number.isFinite(startupGapMs) || startupGapMs < 0) {
|
|
119
|
+
throw new RangeError("startupGapMs must be a nonnegative finite number");
|
|
120
|
+
}
|
|
121
|
+
if (!Number.isFinite(retryMs) || retryMs <= 0) {
|
|
122
|
+
throw new RangeError("retryMs must be a positive finite number");
|
|
123
|
+
}
|
|
124
|
+
const acquireExecution = async (released) => {
|
|
125
|
+
let offset = 0;
|
|
126
|
+
while (!released()) {
|
|
127
|
+
for (let step = 0; step < executionSlots; step += 1) {
|
|
128
|
+
const slot = (offset + step) % executionSlots;
|
|
129
|
+
const lease = await leaseFor(join(root, "slots", String(slot)));
|
|
130
|
+
if (lease !== null)
|
|
131
|
+
return lease;
|
|
132
|
+
}
|
|
133
|
+
offset = (offset + 1) % executionSlots;
|
|
134
|
+
await wait(retryMs);
|
|
135
|
+
}
|
|
136
|
+
return null;
|
|
137
|
+
};
|
|
138
|
+
const acquireStartup = async (released) => {
|
|
139
|
+
while (!released()) {
|
|
140
|
+
const directory = join(root, "startup");
|
|
141
|
+
const lease = await leaseFor(directory);
|
|
142
|
+
if (lease === null) {
|
|
143
|
+
await wait(retryMs);
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
const timestampPath = join(directory, "last-launch-at");
|
|
147
|
+
let previous = Number.NEGATIVE_INFINITY;
|
|
148
|
+
try {
|
|
149
|
+
previous = Number.parseInt(await readFile(timestampPath, "utf8"), 10);
|
|
150
|
+
if (!Number.isFinite(previous))
|
|
151
|
+
previous = Number.NEGATIVE_INFINITY;
|
|
152
|
+
}
|
|
153
|
+
catch (error) {
|
|
154
|
+
if (!(error instanceof Error && "code" in error && error.code === "ENOENT")) {
|
|
155
|
+
await lease.close();
|
|
156
|
+
throw error;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
const delay = Math.max(0, previous + startupGapMs - now());
|
|
160
|
+
if (delay > 0)
|
|
161
|
+
await wait(delay);
|
|
162
|
+
if (released()) {
|
|
163
|
+
await lease.close();
|
|
164
|
+
return null;
|
|
165
|
+
}
|
|
166
|
+
await writeLaunchTimestamp(timestampPath, now());
|
|
167
|
+
return lease;
|
|
168
|
+
}
|
|
169
|
+
return null;
|
|
170
|
+
};
|
|
171
|
+
const tracked = (entry) => {
|
|
172
|
+
let trackedRelease = null;
|
|
173
|
+
return {
|
|
174
|
+
...entry,
|
|
175
|
+
release: () => {
|
|
176
|
+
if (trackedRelease !== null)
|
|
177
|
+
return trackedRelease;
|
|
178
|
+
trackedRelease = entry.release();
|
|
179
|
+
pendingReleases.add(trackedRelease);
|
|
180
|
+
void trackedRelease.then(() => pendingReleases.delete(trackedRelease), () => pendingReleases.delete(trackedRelease));
|
|
181
|
+
return trackedRelease;
|
|
182
|
+
},
|
|
183
|
+
};
|
|
184
|
+
};
|
|
185
|
+
return {
|
|
186
|
+
reserveExecution: (prerequisite) => tracked(reservation(prerequisite, acquireExecution)),
|
|
187
|
+
reserveStartup: (prerequisite) => tracked(reservation(prerequisite, acquireStartup)),
|
|
188
|
+
drain: async () => {
|
|
189
|
+
while (pendingReleases.size > 0)
|
|
190
|
+
await Promise.all([...pendingReleases]);
|
|
191
|
+
},
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
export function hostCoordinatedSlotManager(local, host) {
|
|
195
|
+
return {
|
|
196
|
+
reserve: (handle, kind) => {
|
|
197
|
+
const localReservation = local.reserve(handle, kind);
|
|
198
|
+
if (!localReservation.accepted)
|
|
199
|
+
return localReservation;
|
|
200
|
+
const hostReservation = host.reserveExecution(localReservation.ready);
|
|
201
|
+
let released = false;
|
|
202
|
+
return {
|
|
203
|
+
...localReservation,
|
|
204
|
+
// File ownership is asynchronous, so accepted work is conservatively
|
|
205
|
+
// reported queued until it owns both its local and host-wide slot.
|
|
206
|
+
state: "queued",
|
|
207
|
+
ready: hostReservation.ready,
|
|
208
|
+
isQueued: hostReservation.isQueued,
|
|
209
|
+
release: () => {
|
|
210
|
+
if (released)
|
|
211
|
+
return;
|
|
212
|
+
released = true;
|
|
213
|
+
void hostReservation.release();
|
|
214
|
+
localReservation.release();
|
|
215
|
+
},
|
|
216
|
+
};
|
|
217
|
+
},
|
|
218
|
+
snapshot: local.snapshot,
|
|
219
|
+
};
|
|
220
|
+
}
|
|
221
|
+
export function hostCoordinatedStartupGate(local, host) {
|
|
222
|
+
return {
|
|
223
|
+
reserve: (runtime) => {
|
|
224
|
+
const localReservation = local.reserve(runtime);
|
|
225
|
+
const hostReservation = host.reserveStartup(localReservation.ready);
|
|
226
|
+
let released = false;
|
|
227
|
+
return {
|
|
228
|
+
ready: hostReservation.ready,
|
|
229
|
+
isQueued: hostReservation.isQueued,
|
|
230
|
+
release: () => {
|
|
231
|
+
if (released)
|
|
232
|
+
return;
|
|
233
|
+
released = true;
|
|
234
|
+
void hostReservation.release();
|
|
235
|
+
localReservation.release();
|
|
236
|
+
},
|
|
237
|
+
};
|
|
238
|
+
},
|
|
239
|
+
snapshot: local.snapshot,
|
|
240
|
+
};
|
|
241
|
+
}
|