@nowcrew/daemon 0.5.30 → 0.5.32

package/dist/runtime-startup-gate.js

@@ -0,0 +1,91 @@
+export function createRuntimeStartupGate(limits, now = Date.now) {
+    const activeByRuntime = new Map();
+    const queue = [];
+    let activeTotal = 0;
+    let lastLaunchAt = Number.NEGATIVE_INFINITY;
+    const promote = () => {
+        while (activeTotal < limits.maxStartingTotal) {
+            const index = queue.findIndex((entry) => !entry.released
+                && (activeByRuntime.get(entry.runtime) ?? 0) < limits.maxStartingPerRuntime);
+            if (index < 0)
+                return;
+            const [next] = queue.splice(index, 1);
+            if (!next || next.released)
+                continue;
+            next.promoted = true;
+            activeTotal += 1;
+            activeByRuntime.set(next.runtime, (activeByRuntime.get(next.runtime) ?? 0) + 1);
+            const grant = () => {
+                next.timer = null;
+                if (next.released)
+                    return;
+                next.launchGranted = true;
+                lastLaunchAt = now();
+                next.resolve();
+            };
+            const delay = Math.max(0, lastLaunchAt + limits.startupGapMs - now());
+            if (delay === 0)
+                grant();
+            else
+                next.timer = setTimeout(grant, delay);
+        }
+    };
+    return {
+        reserve: (runtime) => {
+            let resolveReady;
+            const ready = new Promise((resolve) => { resolveReady = resolve; });
+            const entry = {
+                runtime,
+                released: false,
+                promoted: false,
+                launchGranted: false,
+                timer: null,
+                resolve: resolveReady,
+            };
+            queue.push(entry);
+            promote();
+            return {
+                ready,
+                isQueued: () => !entry.launchGranted && !entry.released,
+                release: () => {
+                    if (entry.released)
+                        return;
+                    entry.released = true;
+                    if (entry.timer !== null)
+                        clearTimeout(entry.timer);
+                    if (entry.promoted) {
+                        activeTotal = Math.max(0, activeTotal - 1);
+                        const nextForRuntime = Math.max(0, (activeByRuntime.get(runtime) ?? 1) - 1);
+                        if (nextForRuntime === 0)
+                            activeByRuntime.delete(runtime);
+                        else
+                            activeByRuntime.set(runtime, nextForRuntime);
+                    }
+                    else {
+                        const index = queue.indexOf(entry);
+                        if (index >= 0)
+                            queue.splice(index, 1);
+                    }
+                    promote();
+                },
+            };
+        },
+        snapshot: () => ({
+            startingTotal: activeTotal,
+            queuedTotal: queue.length,
+            startingByRuntime: {
+                claude: activeByRuntime.get("claude") ?? 0,
+                codex: activeByRuntime.get("codex") ?? 0,
+                kimi: activeByRuntime.get("kimi") ?? 0,
+            },
+        }),
+    };
+}
+export function isRuntimeReadyEvent(runtime, event) {
+    if (typeof event !== "object" || event === null)
+        return false;
+    const value = event;
+    if (runtime === "claude")
+        return value.type === "system" && value.subtype === "init";
+    return value.type === "thread.started";
+}

package/dist/runtimes/codex-app-server-runner.js

@@ -24,7 +24,24 @@ const STDERR_TAIL_CAP = 1_200;
 const STDERR_LINE_CAPTURE_CAP = 1_200;
 const STDERR_LINE_OMITTED = "[stderr line omitted: exceeded capture limit]\n";
 const MAX_INITIALIZE_ATTEMPTS = 2;
+// 模型网关瞬态故障(过载/限流)导致 turn 失败时,整轮重试(15s→45s 递进退避):codex 自身
+// 的重试窗口只有 ~10-30s,网关过载往往持续数分钟,这里再兜一层,否则 agent 直接失败
+// 不回复(2026-08-10 事故,普通会话与定时任务都中招)。
+const MAX_TRANSIENT_TURN_RETRIES = 2;
+const TRANSIENT_TURN_RETRY_DELAY_MS = 15_000;
+const TRANSIENT_TURN_RETRY_BACKOFF_FACTOR = 3;
 const PROCESS_TREE_STOP_TIMEOUT_MS = 1_000;
+const TRANSIENT_TURN_ERROR_PATTERNS = [
+    /at capacity/i,
+    /overloaded/i,
+    /rate.?limit/i,
+    /too many requests/i,
+];
+/** 模型侧瞬态失败(网关过载/限流)→ 换个时间整轮重试大概率成功。
+ *  interrupted 不算:那是取消信号或 codex collab 连带中断,重跑语义不明确。 */
+export function isTransientTurnFailure(detail) {
+    return TRANSIENT_TURN_ERROR_PATTERNS.some((pattern) => pattern.test(detail));
+}
 class CodexRpcTimeoutError extends Error {
     method;
     timeoutMs;
@@ -469,6 +486,9 @@ async function runCodexAppServerAttempt(bin, input, attempt, initializeTimeoutMs
         return {
             code: completed.turn?.status === "interrupted" ? 130 : 1,
             initializeTimedOut: false,
+            transientTurnFailure: !cancelling
+                && completed.turn?.status === "failed"
+                && isTransientTurnFailure(detail),
         };
     }
     catch (error) {
@@ -501,13 +521,27 @@ async function runCodexAppServerAttempt(bin, input, attempt, initializeTimeoutMs
 export async function runCodexAppServer(bin, options = {}) {
     const input = await readRunnerInput();
     const initializeTimeoutMs = Math.min(options.initializeTimeoutMs ?? INITIALIZE_RPC_TIMEOUT_MS, INITIALIZE_RPC_TIMEOUT_MS);
-    for (let attempt = 1; attempt <= MAX_INITIALIZE_ATTEMPTS; attempt += 1) {
+    const turnRetryDelayMs = options.turnRetryDelayMs ?? TRANSIENT_TURN_RETRY_DELAY_MS;
+    let initializeTimeouts = 0;
+    let turnRetries = 0;
+    for (let attempt = 1;; attempt += 1) {
         const result = await runCodexAppServerAttempt(bin, input, attempt, initializeTimeoutMs);
-        if (!result.initializeTimedOut || attempt === MAX_INITIALIZE_ATTEMPTS)
-            return result.code;
-        process.stderr.write(`[codex-app-server] stage=retry status=start attempt=${attempt + 1} elapsed_ms=0 reason=initialize_timeout\n`);
+        if (result.initializeTimedOut) {
+            initializeTimeouts += 1;
+            if (initializeTimeouts >= MAX_INITIALIZE_ATTEMPTS)
+                return result.code;
+            process.stderr.write(`[codex-app-server] stage=retry status=start attempt=${attempt + 1} elapsed_ms=0 reason=initialize_timeout\n`);
+            continue;
+        }
+        if (result.transientTurnFailure && turnRetries < MAX_TRANSIENT_TURN_RETRIES) {
+            const delayMs = turnRetryDelayMs * TRANSIENT_TURN_RETRY_BACKOFF_FACTOR ** turnRetries;
+            turnRetries += 1;
+            process.stderr.write(`[codex-app-server] stage=retry status=start attempt=${attempt + 1} elapsed_ms=0 reason=transient_turn_failure\n`);
+            await new Promise((resolve) => setTimeout(resolve, delayMs));
+            continue;
+        }
+        return result.code;
     }
-    return 1;
 }
 function configFromArgv(argv) {
     const { values } = parseArgs({
@@ -515,24 +549,34 @@ function configFromArgv(argv) {
         options: {
             bin: { type: "string" },
             "initialize-timeout-ms": { type: "string" },
+            "turn-retry-delay-ms": { type: "string" },
         },
     });
     if (!values.bin)
         throw new Error("--bin is required");
-    const rawTimeout = values["initialize-timeout-ms"];
-    if (rawTimeout === undefined)
-        return { bin: values.bin };
-    const initializeTimeoutMs = Number(rawTimeout);
-    if (!Number.isInteger(initializeTimeoutMs) || initializeTimeoutMs <= 0) {
-        throw new Error("--initialize-timeout-ms must be a positive integer");
-    }
-    return { bin: values.bin, initializeTimeoutMs };
+    const positiveInteger = (raw, flag) => {
+        if (raw === undefined)
+            return undefined;
+        const value = Number(raw);
+        if (!Number.isInteger(value) || value <= 0) {
+            throw new Error(`${flag} must be a positive integer`);
+        }
+        return value;
+    };
+    const initializeTimeoutMs = positiveInteger(values["initialize-timeout-ms"], "--initialize-timeout-ms");
+    const turnRetryDelayMs = positiveInteger(values["turn-retry-delay-ms"], "--turn-retry-delay-ms");
+    return {
+        bin: values.bin,
+        ...(initializeTimeoutMs === undefined ? {} : { initializeTimeoutMs }),
+        ...(turnRetryDelayMs === undefined ? {} : { turnRetryDelayMs }),
+    };
 }
 if (process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href) {
     const config = configFromArgv(process.argv.slice(2));
-    runCodexAppServer(config.bin, config.initializeTimeoutMs === undefined
-        ? {}
-        : { initializeTimeoutMs: config.initializeTimeoutMs })
+    runCodexAppServer(config.bin, {
+        ...(config.initializeTimeoutMs === undefined ? {} : { initializeTimeoutMs: config.initializeTimeoutMs }),
+        ...(config.turnRetryDelayMs === undefined ? {} : { turnRetryDelayMs: config.turnRetryDelayMs }),
+    })
         .then((code) => { process.exitCode = code; })
         .catch((error) => {
         process.stderr.write(`Codex app-server runner failed: ${safeErrorMessage(error, [])}\n`);

package/dist/serve.js

@@ -28,6 +28,9 @@ import { closeWebSocketWithinDeadline } from "./websocket-shutdown.js";
 import { reconcileExecutionJournal } from "./execution-recovery.js";
 import { createSharedSlotManager } from "./shared-execution-slots.js";
 import { createCompletionRetransmitter } from "./completion-retransmitter.js";
+import { createAgentMemoryBridge } from "./agent-memory/bridge.js";
+import { createRuntimeStartupGate } from "./runtime-startup-gate.js";
+import { createHostExecutionCoordinator, hostCoordinatedSlotManager, hostCoordinatedStartupGate, } from "./host-execution-coordinator.js";
 // normalize.ts 的活动种类 → activity 枚举
 const ACTIVITY_MAP = {
     init: "working", text: "thinking", reading: "reading", sending: "sending",
@@ -56,11 +59,15 @@ export function serve(config, opts = {}) {
     let reconnectTimer = null;
     let stopPromise = null;
     const executionJournal = opts.execution?.journal ?? createExecutionJournal(config.agentsRoot);
+    const agentMemory = opts.execution?.dependencies?.agentMemory
+        ?? (config.agentMemory === null ? undefined : createAgentMemoryBridge(config.agentMemory));
     const executionTelemetry = createExecutionTelemetryJournal(config.agentsRoot);
     const executeProtocol = opts.execution?.runExecution ?? runExecution;
     let detectedExecutionRuntimes = [];
     let runtimeFacts = null;
-    const sharedSlots = createSharedSlotManager(config.executionLimits);
+    const hostCoordinator = opts.execution?.hostCoordinator ?? createHostExecutionCoordinator();
+    const sharedSlots = hostCoordinatedSlotManager(createSharedSlotManager(config.executionLimits), hostCoordinator);
+    const runtimeStartupGate = hostCoordinatedStartupGate(createRuntimeStartupGate(config.executionLimits), hostCoordinator);
     const knownExecutionHashes = new Map();
     const executionReservations = new Map();
     const executionRuns = new Map();
@@ -115,8 +122,11 @@ export function serve(config, opts = {}) {
         if ((entry.state === "completed" || entry.state === "interrupted") && entry.completion !== null) {
             return { executionId: entry.executionId, state: entry.state, completion: entry.completion, updatedAt: entry.updatedAt };
         }
+        if (entry.state === "running" && entry.runtimeReadyAt !== null) {
+            return { executionId: entry.executionId, state: "running", updatedAt: entry.runtimeReadyAt };
+        }
         if (entry.state === "accepted" || entry.state === "running") {
-            return { executionId: entry.executionId, state: entry.state, updatedAt: entry.updatedAt };
+            return { executionId: entry.executionId, state: "accepted", updatedAt: entry.acceptedAt };
         }
         return null;
     };
@@ -137,10 +147,10 @@ export function serve(config, opts = {}) {
                 state: acceptanceState, effectivePermission: entry.effectivePermission ?? "workspace_write",
                 at: entry.acceptedAt,
             });
-            if (entry.state === "running" && entry.processStartedAt !== null) {
+            if (entry.state === "running" && entry.runtimeReadyAt !== null) {
                 safeExecutionSend({
                     type: "execution:started", protocolVersion: 1,
-                    executionId: entry.executionId, at: entry.processStartedAt,
+                    executionId: entry.executionId, at: entry.runtimeReadyAt,
                 });
             }
         }
@@ -335,8 +345,30 @@ export function serve(config, opts = {}) {
                         }
                         return;
                     }
-                    knownExecutionHashes.set(spec.executionId, hash);
                     const reservation = sharedSlots.reserve(spec.agent.handle, "execution");
+                    if (!reservation.accepted) {
+                        dslog("execution.machine_queue_rejected", "机器执行队列已满", {
+                            level: "WARN",
+                            execution_id: spec.executionId,
+                            agent_handle: spec.agent.handle,
+                            ...reservation.facts,
+                        });
+                        safeExecutionSend(ExecutionRejectedSchema.parse({
+                            type: "execution:rejected", protocolVersion: 1, executionId: spec.executionId,
+                            reason: "resource_limit", message: "Local machine execution queue is full",
+                            at: new Date().toISOString(),
+                        }));
+                        return;
+                    }
+                    knownExecutionHashes.set(spec.executionId, hash);
+                    const machineQueueEnteredAt = Date.now();
+                    if (reservation.isQueued()) {
+                        dslog("execution.machine_queued", "execution 已进入机器队列", {
+                            execution_id: spec.executionId,
+                            agent_handle: spec.agent.handle,
+                            ...reservation.facts,
+                        });
+                    }
                     executionReservations.set(spec.executionId, reservation);
                     const cancellation = cancellationFor(spec.executionId);
                     const cleanupExecutionReservation = () => {
@@ -365,14 +397,29 @@ export function serve(config, opts = {}) {
                     }
                     const execution = executeProtocol(config, spec, {
                         ...opts.execution?.dependencies,
+                        ...(agentMemory === undefined ? {} : { agentMemory }),
                         journal: executionJournal,
                         facts: {
                             availableRuntimes,
                             ...reservation.facts,
                         },
                         report: reportExecutionFrame,
+                        startupGate: runtimeStartupGate,
+                        startupTimeoutMs: config.executionLimits.startupTimeoutMs,
                         ...(reservation.state === undefined ? {} : {
-                            slot: { state: reservation.state, ready: reservation.ready },
+                            slot: {
+                                state: reservation.state,
+                                ready: reservation.ready.then(() => {
+                                    const snapshot = sharedSlots.snapshot();
+                                    dslog("execution.machine_slot_ready", "execution 获得机器执行名额", {
+                                        execution_id: spec.executionId,
+                                        agent_handle: spec.agent.handle,
+                                        queue_ms: Date.now() - machineQueueEnteredAt,
+                                        active_total: snapshot.activeTotal,
+                                        queued_total: snapshot.queuedTotal,
+                                    });
+                                }),
+                            },
                         }),
                         cancellation,
                     }).finally(() => {
@@ -539,12 +586,26 @@ export function serve(config, opts = {}) {
                 }
                 running.add(key);
                 legacyReservation = sharedSlots.reserve(msg.agentHandle, "legacy");
+                if (legacyReservation.isQueued()) {
+                    const snapshot = sharedSlots.snapshot();
+                    dslog("run.machine_queued", "legacy run 已进入机器队列", {
+                        ...runKeys,
+                        active_total: snapshot.activeTotal,
+                        queued_total: snapshot.queuedTotal,
+                    });
+                }
                 await awaitWithCancellation(legacyReservation.ready, controller.cancellation);
                 const queueMs = Date.now() - queueWaitStart;
+                const machineSnapshot = sharedSlots.snapshot();
                 const threadLabel = threadId ?? null;
                 const from = msg.wake?.senderHandle ?? "?";
                 const incoming = msg.wake?.content ?? "";
-                dslog("run.start", `开始运行 ${msg.agentHandle}`, { ...runKeys, queue_ms: queueMs });
+                dslog("run.start", `开始运行 ${msg.agentHandle}`, {
+                    ...runKeys,
+                    queue_ms: queueMs,
+                    active_total: machineSnapshot.activeTotal,
+                    queued_total: machineSnapshot.queuedTotal,
+                });
                 log(`\n${"─".repeat(56)}`);
                 log(`🔔 唤醒 agent=${msg.agentHandle}  reason=${msg.reason ?? "?"}`);
                 log(`   channel = ${msg.channelId}`);
@@ -650,7 +711,11 @@ export function serve(config, opts = {}) {
                         ...(!scheduled && threadId ? { wakeMessageId: threadId } : {}),
                         ...(!scheduled && msg.wake?.seq !== undefined ? { wakeContextUpToSeq: msg.wake.seq } : {}),
                         ...(attemptWake ? { wake: attemptWake } : {}),
-                    }, reportActivity, reportConsole, { cancellation: controller.cancellation });
+                    }, reportActivity, reportConsole, {
+                        cancellation: controller.cancellation,
+                        startupGate: runtimeStartupGate,
+                        startupTimeoutMs: config.executionLimits.startupTimeoutMs,
+                    });
                 });
                 const result = mergeRunAgentResults(guarded.results);
                 // 本轮 token 用量上报:runner 已从 result 事件提取(含缓存读/写细分),
@@ -848,6 +913,7 @@ export function serve(config, opts = {}) {
                 for (const run of legacyRuns.values())
                     run.controller.request();
                 await deadline.waitFor(Promise.all(pending));
+                await deadline.waitFor(hostCoordinator.drain());
                 await executionJournal.close({ signal: deadline.signal });
             }
             finally {

package/dist/shared-execution-slots.js

@@ -1,46 +1,58 @@
 export function createSharedSlotManager(limits) {
     const activeByHandle = new Map();
-    const queuesByHandle = new Map();
-    const promoteNext = (handle) => {
-        const queue = queuesByHandle.get(handle) ?? [];
-        while ((activeByHandle.get(handle) ?? 0) < limits.maxParallelPerAgent && queue.length > 0) {
-            const next = queue.shift();
-            if (next.released)
+    const queue = [];
+    let activeTotal = 0;
+    const queuedFor = (handle) => queue.filter((entry) => !entry.released && entry.handle === handle).length;
+    const promote = () => {
+        while (activeTotal < limits.maxParallelTotal) {
+            const index = queue.findIndex((entry) => !entry.released
+                && (activeByHandle.get(entry.handle) ?? 0) < limits.maxParallelPerAgent);
+            if (index < 0)
+                return;
+            const [next] = queue.splice(index, 1);
+            if (!next || next.released)
                 continue;
             next.promoted = true;
-            activeByHandle.set(handle, (activeByHandle.get(handle) ?? 0) + 1);
+            activeTotal += 1;
+            activeByHandle.set(next.handle, (activeByHandle.get(next.handle) ?? 0) + 1);
             next.resolve();
         }
-        if (queue.length === 0)
-            queuesByHandle.delete(handle);
     };
     return {
         reserve: (handle, kind) => {
-            const active = activeByHandle.get(handle) ?? 0;
-            const queued = queuesByHandle.get(handle)?.length ?? 0;
-            const facts = { activeForAgent: active, queuedForAgent: queued };
-            if (kind === "execution" && active >= limits.maxParallelPerAgent
-                && queued >= limits.maxQueuedPerAgent) {
-                return { facts, ready: Promise.resolve(), isQueued: () => false, release: () => { } };
+            const activeForAgent = activeByHandle.get(handle) ?? 0;
+            const queuedForAgent = queuedFor(handle);
+            const facts = {
+                activeForAgent,
+                queuedForAgent,
+                activeTotal,
+                queuedTotal: queue.length,
+            };
+            const canStartImmediately = activeTotal < limits.maxParallelTotal
+                && activeForAgent < limits.maxParallelPerAgent
+                && queue.length === 0;
+            if (kind === "execution" && !canStartImmediately && (queuedForAgent >= limits.maxQueuedPerAgent
+                || queue.length >= limits.maxQueuedTotal)) {
+                return {
+                    accepted: false,
+                    facts,
+                    ready: Promise.resolve(),
+                    isQueued: () => false,
+                    release: () => { },
+                };
             }
             let resolveReady;
             const ready = new Promise((resolve) => { resolveReady = resolve; });
             const entry = {
-                kind,
+                handle,
                 released: false,
-                promoted: active < limits.maxParallelPerAgent,
+                promoted: false,
                 resolve: resolveReady,
             };
-            if (entry.promoted) {
-                activeByHandle.set(handle, active + 1);
-                resolveReady();
-            }
-            else {
-                const queue = queuesByHandle.get(handle) ?? [];
-                queue.push(entry);
-                queuesByHandle.set(handle, queue);
-            }
+            queue.push(entry);
+            promote();
             return {
+                accepted: true,
                 facts,
                 state: entry.promoted ? "ready" : "queued",
                 ready,
@@ -50,19 +62,22 @@ export function createSharedSlotManager(limits) {
                         return;
                     entry.released = true;
                     if (entry.promoted) {
-                        activeByHandle.set(handle, Math.max(0, (activeByHandle.get(handle) ?? 1) - 1));
+                        activeTotal = Math.max(0, activeTotal - 1);
+                        const nextForAgent = Math.max(0, (activeByHandle.get(handle) ?? 1) - 1);
+                        if (nextForAgent === 0)
+                            activeByHandle.delete(handle);
+                        else
+                            activeByHandle.set(handle, nextForAgent);
                     }
                     else {
-                        const queue = queuesByHandle.get(handle);
-                        const index = queue?.indexOf(entry) ?? -1;
-                        if (queue !== undefined && index >= 0)
+                        const index = queue.indexOf(entry);
+                        if (index >= 0)
                             queue.splice(index, 1);
-                        if (queue?.length === 0)
-                            queuesByHandle.delete(handle);
                     }
-                    promoteNext(handle);
+                    promote();
                 },
             };
         },
+        snapshot: () => ({ activeTotal, queuedTotal: queue.length }),
     };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nowcrew/daemon",
-  "version": "0.5.30",
+  "version": "0.5.32",
   "type": "module",
   "description": "crew daemon — 运行在用户机器:拉起/管理 agent 进程,注入 crew CLI,归一化 runtime 事件",
   "license": "Apache-2.0",