@nowcrew/daemon 0.6.45 → 0.6.46

package/README.md

@@ -376,18 +376,29 @@ CREW_SCHEDULED_MAX_PARALLEL=4
 CREW_EXECUTION_MAX_QUEUED_PER_AGENT=32
 CREW_EXECUTION_MAX_PARALLEL_TOTAL=10
 CREW_EXECUTION_MAX_QUEUED_TOTAL=128
-CREW_EXECUTION_MAX_STARTING_TOTAL=1
-CREW_EXECUTION_MAX_STARTING_PER_RUNTIME=1
-CREW_EXECUTION_START_GAP_MS=3000
+CREW_EXECUTION_MAX_STARTING_TOTAL=10
+CREW_EXECUTION_MAX_STARTING_PER_RUNTIME=10
+CREW_EXECUTION_START_GAP_MS=500
 CREW_EXECUTION_STARTUP_TIMEOUT_MS=120000
+CREW_EXECUTION_MAX_QUEUE_WAIT_MS=300000
+CREW_EXECUTION_MAX_FIRST_OUTPUT_WAIT_MS=120000
+CREW_EXECUTION_FIRST_OUTPUT_GRACE_MS=180000
 ```
 
 `CREW_MAX_PARALLEL` limits normal executions per Agent. `CREW_SCHEDULED_MAX_PARALLEL` separately
 limits scheduled executions per Agent. `MAX_PARALLEL_TOTAL` is the machine-wide process cap shared by
 protocol-v1 and legacy work.
-Runtime startup is a separate FIFO gate: by default only one Claude, Codex, or Kimi process may be
-initializing at a time, and launches are spaced by three seconds. A process leaves the startup gate
+Runtime startup is a separate FIFO gate: by default up to ten Claude, Codex, or Kimi processes may be
+initializing at a time, and launches are spaced by 500ms. A process leaves the startup gate
 only after its runtime-specific ready event; startup timeout cancels the owned process tree.
+An accepted execution may wait up to five minutes for a local and host slot. After runtime readiness,
+the daemon waits up to two minutes for the first model event, with a three-minute grace window for
+slow providers. When one of these local liveness limits expires, the supervisor is stopped and the
+terminal result is written to the execution journal before it is reported, so the execution remains
+replayable instead of being silently dropped. Queue and first-output timeouts are retried locally at
+most twice (three total attempts). Each retry releases its current machine/host slot and reserves a
+new one, which puts it behind work already waiting in the queue. A third timeout is terminal and is
+not requeued; other failures are not automatically retried.
 
 Local policy can reduce server-requested access and limits; it cannot grant more access than requested.
 Provider credentials and configured environment are prepared locally and never carried in execution

package/dist/config.js

@@ -15,10 +15,13 @@ export const DEFAULT_EXECUTION_LIMITS = Object.freeze({
     maxQueuedPerAgent: 32,
     maxParallelTotal: 10,
     maxQueuedTotal: 128,
-    maxStartingTotal: 1,
-    maxStartingPerRuntime: 1,
-    startupGapMs: 3_000,
+    maxStartingTotal: 10,
+    maxStartingPerRuntime: 10,
+    startupGapMs: 500,
     startupTimeoutMs: 120_000,
+    maxQueueWaitMs: 5 * 60_000,
+    maxFirstOutputWaitMs: 2 * 60_000,
+    firstOutputGraceMs: 3 * 60_000,
 });
 // Fits the largest mandatory v1 lifecycle envelope (UUID + timestamps + outcome facts) with margin.
 export const MIN_EXECUTION_EVENT_BYTES = 512;
@@ -85,6 +88,9 @@ export function loadConfig(env = process.env) {
         maxStartingPerRuntime: positiveIntegerEnv(env, "CREW_EXECUTION_MAX_STARTING_PER_RUNTIME", DEFAULT_EXECUTION_LIMITS.maxStartingPerRuntime),
         startupGapMs: positiveIntegerEnv(env, "CREW_EXECUTION_START_GAP_MS", DEFAULT_EXECUTION_LIMITS.startupGapMs),
         startupTimeoutMs: positiveIntegerEnv(env, "CREW_EXECUTION_STARTUP_TIMEOUT_MS", DEFAULT_EXECUTION_LIMITS.startupTimeoutMs),
+        maxQueueWaitMs: positiveIntegerEnv(env, "CREW_EXECUTION_MAX_QUEUE_WAIT_MS", DEFAULT_EXECUTION_LIMITS.maxQueueWaitMs),
+        maxFirstOutputWaitMs: positiveIntegerEnv(env, "CREW_EXECUTION_MAX_FIRST_OUTPUT_WAIT_MS", DEFAULT_EXECUTION_LIMITS.maxFirstOutputWaitMs),
+        firstOutputGraceMs: positiveIntegerEnv(env, "CREW_EXECUTION_FIRST_OUTPUT_GRACE_MS", DEFAULT_EXECUTION_LIMITS.firstOutputGraceMs),
     });
     return {
         serverUrl,

package/dist/execution-journal.js

@@ -748,6 +748,30 @@ export function createExecutionJournal(agentsRoot, options = {}) {
                 return { kind: "created", ...entry };
             });
         },
+        retry: async (executionId) => {
+            validateExecutionId(executionId);
+            return serializedExecution(executionId, "retry", async () => {
+                const entry = await requireRecord(executionId);
+                if (entry.state !== "accepted" && entry.state !== "running") {
+                    throw new JournalTransitionError(`Cannot retry ${entry.state} execution`);
+                }
+                const timestamp = now().toISOString();
+                const updated = JournalEntrySchema.parse({
+                    ...entry,
+                    state: "accepted",
+                    pid: null,
+                    completion: null,
+                    completionAcknowledged: false,
+                    acceptedAt: timestamp,
+                    processStartedAt: null,
+                    runtimeReadyAt: null,
+                    processIdentity: null,
+                    updatedAt: timestamp,
+                });
+                await writeRecord(updated);
+                return updated;
+            });
+        },
         startGuarded: async (executionId, processStartedAt, startDormant, hooks) => {
             validateExecutionId(executionId);
             TimestampSchema.parse(processStartedAt);

package/dist/execution-runner.js

@@ -4,7 +4,7 @@ import { DaemonToServerExecutionFrameSchema, ExecutionCompletedSchema, Execution
 import { JournalConflictError } from "./execution-journal.js";
 import { boundExecutionFrame } from "./execution-event-limit.js";
 import { mintAgentToken } from "./token.js";
-import { executeLocal, withLocalExecutionFacts, } from "./local-executor.js";
+import { executeLocal, RuntimeRunningCallbackTimeoutError, withLocalExecutionFacts, } from "./local-executor.js";
 import { startDormantSupervisor, } from "./execution-supervisor.js";
 import { CLAUDE_EFFORT_LEVELS } from "./runtimes/claude.js";
 import { CODEX_EFFORT_LEVELS } from "./runtimes/codex.js";
@@ -25,6 +25,7 @@ import { PROJECT_SKILL_PROJECTION_V2_CAPABILITY } from "./project-skills/types.j
 import { dslog } from "./slog.js";
 import { identityRosterPromptFacts } from "./identity-roster-telemetry.js";
 export { supervisorLaunch } from "./supervised-runtime.js";
+const MAX_LOCAL_EXECUTION_ATTEMPTS = 3;
 const ACTIVITY_KIND = {
     init: "working",
     text: "thinking",
@@ -298,6 +299,12 @@ class ExecutionCancelledError extends Error {
         this.name = "ExecutionCancelledError";
     }
 }
+class ExecutionQueueTimeoutError extends Error {
+    constructor(timeoutMs) {
+        super(`Execution queue wait exceeded ${timeoutMs}ms`);
+        this.name = "ExecutionQueueTimeoutError";
+    }
+}
 async function cancellable(promise, cancellation) {
     if (cancellation === undefined)
         return promise;
@@ -308,9 +315,24 @@ async function cancellable(promise, cancellation) {
         cancellation.requested.then(() => { throw new ExecutionCancelledError(); }),
     ]);
 }
+async function cancellableWithTimeout(promise, timeoutMs, cancellation, timeoutError) {
+    let timer;
+    try {
+        return await Promise.race([
+            cancellable(promise, cancellation),
+            new Promise((_resolve, reject) => {
+                timer = setTimeout(() => reject(timeoutError), timeoutMs);
+            }),
+        ]);
+    }
+    finally {
+        if (timer !== undefined)
+            clearTimeout(timer);
+    }
+}
 function failedCompletion(spec, error, startedAt, finishedAt) {
     const message = error instanceof Error ? error.message : String(error);
-    const errorCode = error instanceof HostReservationCancelledError
+    const errorCode = error instanceof ExecutionQueueTimeoutError || error instanceof HostReservationCancelledError
         ? "queue_timeout"
         : error instanceof ProjectContextUnavailableError
             ? error.code
@@ -331,6 +353,10 @@ function failedCompletion(spec, error, startedAt, finishedAt) {
 }
 export async function runExecution(config, input, dependencies) {
     const now = dependencies.now ?? (() => new Date());
+    const retryAttempt = dependencies.retryAttempt ?? 1;
+    if (!Number.isInteger(retryAttempt) || retryAttempt < 1 || retryAttempt > MAX_LOCAL_EXECUTION_ATTEMPTS) {
+        throw new RangeError(`retryAttempt must be between 1 and ${MAX_LOCAL_EXECUTION_ATTEMPTS}`);
+    }
     const initialAt = now().toISOString();
     const parsed = ExecutionStartSchema.safeParse(input);
     if (!parsed.success) {
@@ -373,31 +399,38 @@ export async function runExecution(config, input, dependencies) {
             await dependencies.report(boundExecutionFrame(replay.completion, config.executionLimits.maxEventBytes));
             return { kind: "existing", entry: replay };
         }
-        await reportBestEffort(dependencies.report, boundExecutionFrame(DaemonToServerExecutionFrameSchema.parse({
-            type: "execution:accepted",
-            protocolVersion: 1,
-            executionId: spec.executionId,
-            state: dependencies.slot?.state ?? "ready",
-            effectivePermission: replayPermission,
-            at: replay.acceptedAt,
-        }), config.executionLimits.maxEventBytes), bestEffortTimeoutMs);
-        if (replay.state === "running" && replay.runtimeReadyAt !== null) {
+        if (retryAttempt === 1) {
             await reportBestEffort(dependencies.report, boundExecutionFrame(DaemonToServerExecutionFrameSchema.parse({
-                type: "execution:started",
+                type: "execution:accepted",
                 protocolVersion: 1,
                 executionId: spec.executionId,
-                at: replay.runtimeReadyAt,
+                state: dependencies.slot?.state ?? "ready",
+                effectivePermission: replayPermission,
+                at: replay.acceptedAt,
             }), config.executionLimits.maxEventBytes), bestEffortTimeoutMs);
+            if (replay.state === "running" && replay.runtimeReadyAt !== null) {
+                await reportBestEffort(dependencies.report, boundExecutionFrame(DaemonToServerExecutionFrameSchema.parse({
+                    type: "execution:started",
+                    protocolVersion: 1,
+                    executionId: spec.executionId,
+                    at: replay.runtimeReadyAt,
+                }), config.executionLimits.maxEventBytes), bestEffortTimeoutMs);
+            }
+            return { kind: "existing", entry: replay };
         }
-        return { kind: "existing", entry: replay };
     }
-    const checked = admission(spec, config, dependencies, initialAt);
+    // A retry already owns a freshly admitted slot. Re-running the initial capacity
+    // admission against the first attempt's stale facts would reject a valid tail retry.
+    const checked = retryAttempt > 1
+        ? { permission: effectivePermission(spec, config) }
+        : admission(spec, config, dependencies, initialAt);
     if ("rejected" in checked) {
         const frame = ExecutionRejectedSchema.parse(boundExecutionFrame(checked.rejected, config.executionLimits.maxEventBytes));
         await dependencies.report(frame);
         return { kind: "rejected", frame };
     }
     const { permission } = checked;
+    const effectiveTimeoutMs = spec.runtime.timeoutMs ?? config.executionLimits.maxTimeoutMs;
     const accepted = await dependencies.journal.accept(spec.executionId, specHash, {
         runtime: spec.runtime.name,
         ...(spec.runtime.model === undefined ? {} : { model: spec.runtime.model }),
@@ -410,32 +443,36 @@ export async function runExecution(config, input, dependencies) {
             await dependencies.report(boundExecutionFrame(accepted.completion, config.executionLimits.maxEventBytes));
             return { kind: "existing", entry: accepted };
         }
+        if (retryAttempt === 1) {
+            await reportBestEffort(dependencies.report, boundExecutionFrame(DaemonToServerExecutionFrameSchema.parse({
+                type: "execution:accepted",
+                protocolVersion: 1,
+                executionId: spec.executionId,
+                state: dependencies.slot?.state ?? "ready",
+                effectivePermission: accepted.effectivePermission ?? permission,
+                at: accepted.acceptedAt,
+            }), config.executionLimits.maxEventBytes), bestEffortTimeoutMs);
+            if (accepted.state === "running" && accepted.runtimeReadyAt !== null) {
+                await reportBestEffort(dependencies.report, boundExecutionFrame(DaemonToServerExecutionFrameSchema.parse({
+                    type: "execution:started",
+                    protocolVersion: 1,
+                    executionId: spec.executionId,
+                    at: accepted.runtimeReadyAt,
+                }), config.executionLimits.maxEventBytes), bestEffortTimeoutMs);
+            }
+            return { kind: "existing", entry: accepted };
+        }
+    }
+    if (retryAttempt === 1) {
         await reportBestEffort(dependencies.report, boundExecutionFrame(DaemonToServerExecutionFrameSchema.parse({
             type: "execution:accepted",
             protocolVersion: 1,
             executionId: spec.executionId,
             state: dependencies.slot?.state ?? "ready",
-            effectivePermission: accepted.effectivePermission ?? permission,
+            effectivePermission: permission,
             at: accepted.acceptedAt,
         }), config.executionLimits.maxEventBytes), bestEffortTimeoutMs);
-        if (accepted.state === "running" && accepted.runtimeReadyAt !== null) {
-            await reportBestEffort(dependencies.report, boundExecutionFrame(DaemonToServerExecutionFrameSchema.parse({
-                type: "execution:started",
-                protocolVersion: 1,
-                executionId: spec.executionId,
-                at: accepted.runtimeReadyAt,
-            }), config.executionLimits.maxEventBytes), bestEffortTimeoutMs);
-        }
-        return { kind: "existing", entry: accepted };
     }
-    await reportBestEffort(dependencies.report, boundExecutionFrame(DaemonToServerExecutionFrameSchema.parse({
-        type: "execution:accepted",
-        protocolVersion: 1,
-        executionId: spec.executionId,
-        state: dependencies.slot?.state ?? "ready",
-        effectivePermission: permission,
-        at: accepted.acceptedAt,
-    }), config.executionLimits.maxEventBytes), bestEffortTimeoutMs);
     const mint = dependencies.mintAgentToken ?? mintAgentToken;
     const execute = dependencies.executeLocal ?? executeLocal;
     const startSupervisor = dependencies.startSupervisor ?? startDormantSupervisor;
@@ -466,6 +503,11 @@ export async function runExecution(config, input, dependencies) {
     };
     let startedAt = accepted.acceptedAt;
     let runtimeCancel = null;
+    let timeout;
+    let timedOut = false;
+    let firstOutputTimer;
+    let firstOutputTimedOut = false;
+    let abandonedRuntimeRunningCompletion = null;
     let completion;
     let memoryCaptureFinalText = null;
     let boundImDecision = spec.reporting.allowBoundImDecision
@@ -477,7 +519,7 @@ export async function runExecution(config, input, dependencies) {
     });
     try {
         if (dependencies.slot !== undefined) {
-            await cancellable(dependencies.slot.ready, dependencies.cancellation);
+            await cancellableWithTimeout(dependencies.slot.ready, config.executionLimits.maxQueueWaitMs, dependencies.cancellation, new ExecutionQueueTimeoutError(config.executionLimits.maxQueueWaitMs));
         }
         let projectContext;
         let sessionContextFingerprint;
@@ -508,23 +550,90 @@ export async function runExecution(config, input, dependencies) {
         let consoleSequence = 0;
         let externalOutputSequence = 0;
         const callbacks = {
-            onRuntimeReady: async () => {
+            onRuntimeStarting: () => {
+                dependencies.onRuntimePhase?.("starting", spec.runtime.name);
+            },
+            onRuntimeRunning: async () => {
+                const callbackStartedAt = Date.now();
+                let callbackStage = "journal";
                 if (dependencies.cancellation?.isRequested())
                     return;
-                const ready = await dependencies.journal.markRuntimeReady(spec.executionId, now().toISOString());
-                if (ready.runtimeReadyAt === null) {
-                    throw new Error(`Execution ${spec.executionId} runtime readiness was not persisted`);
+                const cancel = runtimeCancel;
+                if (cancel !== null && timeout === undefined && !dependencies.cancellation?.isRequested()) {
+                    const firstOutputDeadline = config.executionLimits.maxFirstOutputWaitMs
+                        + config.executionLimits.firstOutputGraceMs;
+                    if (effectiveTimeoutMs > firstOutputDeadline) {
+                        firstOutputTimer = setTimeout(() => {
+                            firstOutputTimedOut = true;
+                            try {
+                                void cancel().catch(rejectCancellationFailure);
+                            }
+                            catch (error) {
+                                rejectCancellationFailure(error);
+                            }
+                        }, firstOutputDeadline);
+                    }
+                    timeout = setTimeout(() => {
+                        timedOut = true;
+                        try {
+                            void cancel().catch(rejectCancellationFailure);
+                        }
+                        catch (error) {
+                            rejectCancellationFailure(error);
+                        }
+                    }, effectiveTimeoutMs);
+                }
+                dependencies.onRuntimePhase?.("running", spec.runtime.name);
+                const journalStartedAt = Date.now();
+                try {
+                    const ready = await dependencies.journal.markRuntimeReady(spec.executionId, now().toISOString());
+                    const journalMs = Date.now() - journalStartedAt;
+                    if (ready.runtimeReadyAt === null) {
+                        throw new Error(`Execution ${spec.executionId} runtime readiness was not persisted`);
+                    }
+                    if (abandonedRuntimeRunningCompletion !== null) {
+                        await dependencies.journal.complete(spec.executionId, abandonedRuntimeRunningCompletion);
+                        await dependencies.report(abandonedRuntimeRunningCompletion);
+                        return;
+                    }
+                    startedAt = ready.runtimeReadyAt;
+                    callbackStage = "started_report";
+                    const reportStartedAt = Date.now();
+                    const reportDelivered = await reportBestEffort(dependencies.report, boundExecutionFrame(DaemonToServerExecutionFrameSchema.parse({
+                        type: "execution:started",
+                        protocolVersion: 1,
+                        executionId: spec.executionId,
+                        at: startedAt,
+                    }), config.executionLimits.maxEventBytes), bestEffortTimeoutMs);
+                    dslog("execution.runtime_running_callback", "runtime-running callback 已完成", {
+                        execution_id: spec.executionId,
+                        runtime: spec.runtime.name,
+                        outcome: "succeeded",
+                        journal_ms: journalMs,
+                        started_report_ms: Date.now() - reportStartedAt,
+                        started_report_delivered: reportDelivered,
+                        callback_total_ms: Date.now() - callbackStartedAt,
+                    });
+                }
+                catch (error) {
+                    dslog("execution.runtime_running_callback", "runtime-running callback 失败", {
+                        level: "ERROR",
+                        execution_id: spec.executionId,
+                        runtime: spec.runtime.name,
+                        outcome: "failed",
+                        failed_stage: callbackStage,
+                        callback_total_ms: Date.now() - callbackStartedAt,
+                        error_message: error instanceof Error ? error.message.slice(0, 500) : String(error).slice(0, 500),
+                    });
+                    throw error;
                 }
-                startedAt = ready.runtimeReadyAt;
-                await reportBestEffort(dependencies.report, boundExecutionFrame(DaemonToServerExecutionFrameSchema.parse({
-                    type: "execution:started",
-                    protocolVersion: 1,
-                    executionId: spec.executionId,
-                    at: startedAt,
-                }), config.executionLimits.maxEventBytes), bestEffortTimeoutMs);
             },
             ...(spec.reporting.streamActivity ? {
                 onActivity: (activity) => {
+                    if (firstOutputTimer !== undefined) {
+                        clearTimeout(firstOutputTimer);
+                        firstOutputTimer = undefined;
+                    }
                     const frame = DaemonToServerExecutionFrameSchema.parse({
                         type: "execution:activity",
                         protocolVersion: 1,
@@ -543,6 +652,10 @@ export async function runExecution(config, input, dependencies) {
             } : {}),
             ...(spec.reporting.streamConsole ? {
                 onConsole: (chunk) => {
+                    if (firstOutputTimer !== undefined) {
+                        clearTimeout(firstOutputTimer);
+                        firstOutputTimer = undefined;
+                    }
                     const frame = DaemonToServerExecutionFrameSchema.parse({
                         type: "execution:console",
                         protocolVersion: 1,
@@ -562,6 +675,10 @@ export async function runExecution(config, input, dependencies) {
             } : {}),
             ...(spec.context.externalResponseSessionId || spec.context.answerStream ? {
                 onExternalOutput: (text) => {
+                    if (firstOutputTimer !== undefined) {
+                        clearTimeout(firstOutputTimer);
+                        firstOutputTimer = undefined;
+                    }
                     const frame = DaemonToServerExecutionFrameSchema.parse({
                         type: "execution:output",
                         protocolVersion: 1,
@@ -735,6 +852,10 @@ export async function runExecution(config, input, dependencies) {
             execute(localInput, callbacks, localDependencies),
             cancellationFailure,
         ]);
+        if (timeout !== undefined)
+            clearTimeout(timeout);
+        if (firstOutputTimer !== undefined)
+            clearTimeout(firstOutputTimer);
         const finishedAt = now().toISOString();
         if (result.exitCode === 0 && result.finalText?.trim())
             memoryCaptureFinalText = result.finalText;
@@ -744,7 +865,33 @@ export async function runExecution(config, input, dependencies) {
             await resetBoundImDecision(path);
             boundImDecision = selected?.decision ?? "silent";
         }
-        completion = ExecutionCompletedSchema.parse({
+        completion = ExecutionCompletedSchema.parse(firstOutputTimedOut ? {
+            type: "execution:completed",
+            protocolVersion: 1,
+            executionId: spec.executionId,
+            outcome: "cancelled",
+            errorCode: "runtime_no_first_output",
+            errorMessage: "Runtime produced no activity before the first-output deadline",
+            runtime: result.runtime,
+            ...(result.model === null ? {} : { model: result.model }),
+            resumed: result.resumed,
+            ...(boundImDecision ? { boundImDecision } : {}),
+            startedAt,
+            finishedAt,
+        } : timedOut ? {
+            type: "execution:completed",
+            protocolVersion: 1,
+            executionId: spec.executionId,
+            outcome: "cancelled",
+            errorCode: "timeout",
+            errorMessage: "Execution exceeded its local timeout",
+            runtime: result.runtime,
+            ...(result.model === null ? {} : { model: result.model }),
+            resumed: result.resumed,
+            ...(boundImDecision ? { boundImDecision } : {}),
+            startedAt,
+            finishedAt,
+        } : {
             type: "execution:completed",
             protocolVersion: 1,
             executionId: spec.executionId,
@@ -779,6 +926,10 @@ export async function runExecution(config, input, dependencies) {
         });
     }
     catch (error) {
+        if (timeout !== undefined)
+            clearTimeout(timeout);
+        if (firstOutputTimer !== undefined)
+            clearTimeout(firstOutputTimer);
         const cancelled = error instanceof ExecutionCancelledError || error instanceof RuntimeCancelledError;
         if (cancelled) {
             await closeLaunchGate();
@@ -795,7 +946,7 @@ export async function runExecution(config, input, dependencies) {
         else {
             await proveActiveSupervisorStopped(error, spec.agent.projectSkillBindingGeneration !== undefined);
         }
-        completion = cancelled
+        const failureCompletion = cancelled
             ? ExecutionCompletedSchema.parse({
                 type: "execution:completed",
                 protocolVersion: 1,
@@ -810,9 +961,55 @@ export async function runExecution(config, input, dependencies) {
                 finishedAt: now().toISOString(),
             })
             : failedCompletion(spec, error, startedAt, now().toISOString());
+        if (error instanceof RuntimeRunningCallbackTimeoutError) {
+            abandonedRuntimeRunningCompletion = boundExecutionFrame(failureCompletion, config.executionLimits.maxEventBytes);
+            await telemetry.closeAndDrain();
+            throw error;
+        }
+        completion = failureCompletion;
     }
     completion = boundExecutionFrame(completion, config.executionLimits.maxEventBytes);
     await telemetry.closeAndDrain();
+    const retryReason = completion.errorCode === "queue_timeout"
+        ? "queue_timeout"
+        : completion.errorCode === "runtime_no_first_output"
+            ? "runtime_no_first_output"
+            : null;
+    if (retryReason !== null
+        && retryAttempt < MAX_LOCAL_EXECUTION_ATTEMPTS
+        && dependencies.onRetry !== undefined
+        && dependencies.journal.retry !== undefined
+        && !dependencies.cancellation?.isRequested()) {
+        try {
+            await dependencies.journal.retry(spec.executionId);
+            const nextSlot = await dependencies.onRetry({ attempt: retryAttempt, reason: retryReason });
+            if (nextSlot !== undefined) {
+                return runExecution(config, spec, {
+                    ...dependencies,
+                    slot: nextSlot,
+                    retryAttempt: retryAttempt + 1,
+                });
+            }
+        }
+        catch (error) {
+            dslog("execution.retry_requeue_failed", "execution 超时重排队失败,保留本次终态", {
+                level: "ERROR",
+                execution_id: spec.executionId,
+                attempt: retryAttempt,
+                reason: retryReason,
+                error_message: error instanceof Error ? error.message.slice(0, 500) : String(error).slice(0, 500),
+            });
+        }
+    }
+    if (retryReason !== null && retryAttempt >= MAX_LOCAL_EXECUTION_ATTEMPTS) {
+        dslog("execution.retry_exhausted", "execution 超时重试次数已耗尽", {
+            level: "WARN",
+            execution_id: spec.executionId,
+            attempt: retryAttempt,
+            max_attempts: MAX_LOCAL_EXECUTION_ATTEMPTS,
+            reason: retryReason,
+        });
+    }
     await dependencies.journal.complete(spec.executionId, completion);
     if (completion.outcome === "succeeded"
         && spec.agent.memoryEnabled === true

package/dist/runtimes/progress-watchdog.js

@@ -4,11 +4,23 @@ export const DEFAULT_FIRST_PROGRESS_TIMEOUT_MS = 120_000;
  * the runtime's configured total timeout remains authoritative; long-running tools are not killed
  * merely because they produce no output.
  */
-export function startFirstProgressWatchdog(_onTimeout, timeoutMs = DEFAULT_FIRST_PROGRESS_TIMEOUT_MS) {
+export function startFirstProgressWatchdog(onTimeout, timeoutMs = DEFAULT_FIRST_PROGRESS_TIMEOUT_MS) {
     if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
         throw new RangeError("First-progress timeout must be a positive finite number");
     }
-    // Progress is observational only. Runtime tasks must not be terminated because
-    // an upstream provider is temporarily silent.
-    return { observe: () => undefined, stop: () => undefined };
+    let active = true;
+    const timer = setTimeout(() => {
+        if (!active)
+            return;
+        active = false;
+        onTimeout();
+    }, timeoutMs);
+    timer.unref?.();
+    const stop = () => {
+        if (!active)
+            return;
+        active = false;
+        clearTimeout(timer);
+    };
+    return { observe: stop, stop };
 }

package/dist/serve.js

@@ -516,7 +516,7 @@ export function serve(config, opts = {}) {
                     const executionClass = pruneTraceId !== null
                         ? "memory_prune"
                         : spec.context.scheduledRunId === undefined ? "normal" : "scheduled";
-                    const reservation = sharedSlots.reserve(spec.agent.handle, "execution", executionClass);
+                    let reservation = sharedSlots.reserve(spec.agent.handle, "execution", executionClass);
                     if (!reservation.accepted) {
                         dslog("execution.machine_queue_rejected", "机器执行队列已满", {
                             level: "WARN",
@@ -533,7 +533,7 @@ export function serve(config, opts = {}) {
                         return;
                     }
                     knownExecutionHashes.set(spec.executionId, hash);
-                    const machineQueueEnteredAt = Date.now();
+                    let machineQueueEnteredAt = Date.now();
                     const executionConcurrencyIdentity = {
                         executionId: spec.executionId,
                         agentHandle: spec.agent.handle,
@@ -571,6 +571,70 @@ export function serve(config, opts = {}) {
                         });
                     };
                     const cancellation = cancellationFor(spec.executionId);
+                    const slotForReservation = (attemptReservation, queueEnteredAt) => ({
+                        state: attemptReservation.state ?? (attemptReservation.isQueued() ? "queued" : "ready"),
+                        ready: attemptReservation.ready.then(() => {
+                            markExecutionTaskKeyActive();
+                            attemptReservation.markPreparing();
+                            concurrencyTelemetry.transition(executionConcurrencyIdentity, "preparing", {
+                                queue_ms: Date.now() - queueEnteredAt,
+                            });
+                            const snapshot = sharedSlots.snapshot();
+                            dslog("execution.machine_slot_ready", "execution 获得机器执行名额", {
+                                execution_id: spec.executionId,
+                                agent_handle: spec.agent.handle,
+                                execution_class: executionClass,
+                                queue_ms: Date.now() - queueEnteredAt,
+                                active_total: snapshot.runningTotal,
+                                admitted_total: snapshot.admittedTotal,
+                                preparing_total: snapshot.preparingTotal,
+                                starting_total: snapshot.startingTotal,
+                                running_total: snapshot.runningTotal,
+                                queued_total: snapshot.queuedTotal,
+                                memory_prune_admitted_total: snapshot.memoryPruneAdmittedTotal,
+                                memory_prune_queued_total: snapshot.memoryPruneQueuedTotal,
+                                memory_prune_preparing_total: snapshot.memoryPrunePreparingTotal,
+                                memory_prune_starting_total: snapshot.memoryPruneStartingTotal,
+                                memory_prune_running_total: snapshot.memoryPruneRunningTotal,
+                            });
+                        }),
+                    });
+                    const onRetry = async ({ attempt, reason, }) => {
+                        if (stopped || cancellation.isRequested())
+                            return undefined;
+                        const previousReservation = reservation;
+                        previousReservation.release();
+                        concurrencyTelemetry.release(executionConcurrencyIdentity, `retry:${reason}`);
+                        const nextReservation = sharedSlots.reserve(spec.agent.handle, "execution", executionClass);
+                        if (!nextReservation.accepted) {
+                            executionReservations.delete(spec.executionId);
+                            dslog("execution.retry_queue_rejected", "execution 超时重排队被机器队列拒绝", {
+                                level: "WARN",
+                                execution_id: spec.executionId,
+                                agent_handle: spec.agent.handle,
+                                execution_class: executionClass,
+                                attempt,
+                                reason,
+                                ...nextReservation.facts,
+                            });
+                            return undefined;
+                        }
+                        reservation = nextReservation;
+                        machineQueueEnteredAt = Date.now();
+                        executionReservations.set(spec.executionId, nextReservation);
+                        if (nextReservation.isQueued()) {
+                            concurrencyTelemetry.transition(executionConcurrencyIdentity, "queued");
+                            dslog("execution.retry_queued", "execution 超时后重新进入机器队列", {
+                                execution_id: spec.executionId,
+                                agent_handle: spec.agent.handle,
+                                execution_class: executionClass,
+                                attempt: attempt + 1,
+                                reason,
+                                ...nextReservation.facts,
+                            });
+                        }
+                        return slotForReservation(nextReservation, machineQueueEnteredAt);
+                    };
                     const cleanupExecutionReservation = (reason = "execution_settled") => {
                         if (executionTaskKeyFinished)
                             return;
@@ -652,36 +716,8 @@ export function serve(config, opts = {}) {
                                 memory_prune_running_total: snapshot.memoryPruneRunningTotal,
                             });
                         },
-                        ...(reservation.state === undefined ? {} : {
-                            slot: {
-                                state: reservation.state,
-                                ready: reservation.ready.then(() => {
-                                    markExecutionTaskKeyActive();
-                                    reservation.markPreparing();
-                                    concurrencyTelemetry.transition(executionConcurrencyIdentity, "preparing", {
-                                        queue_ms: Date.now() - machineQueueEnteredAt,
-                                    });
-                                    const snapshot = sharedSlots.snapshot();
-                                    dslog("execution.machine_slot_ready", "execution 获得机器执行名额", {
-                                        execution_id: spec.executionId,
-                                        agent_handle: spec.agent.handle,
-                                        execution_class: executionClass,
-                                        queue_ms: Date.now() - machineQueueEnteredAt,
-                                        active_total: snapshot.runningTotal,
-                                        admitted_total: snapshot.admittedTotal,
-                                        preparing_total: snapshot.preparingTotal,
-                                        starting_total: snapshot.startingTotal,
-                                        running_total: snapshot.runningTotal,
-                                        queued_total: snapshot.queuedTotal,
-                                        memory_prune_admitted_total: snapshot.memoryPruneAdmittedTotal,
-                                        memory_prune_queued_total: snapshot.memoryPruneQueuedTotal,
-                                        memory_prune_preparing_total: snapshot.memoryPrunePreparingTotal,
-                                        memory_prune_starting_total: snapshot.memoryPruneStartingTotal,
-                                        memory_prune_running_total: snapshot.memoryPruneRunningTotal,
-                                    });
-                                }),
-                            },
-                        }),
+                        slot: slotForReservation(reservation, machineQueueEnteredAt),
+                        onRetry,
                         cancellation,
                     }).finally(() => {
                         cleanupExecutionReservation("execution_settled");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nowcrew/daemon",
-  "version": "0.6.45",
+  "version": "0.6.46",
   "type": "module",
   "description": "crew daemon — 运行在用户机器:拉起/管理 agent 进程,注入 crew CLI,归一化 runtime 事件",
   "license": "Apache-2.0",