@nowcrew/daemon 0.6.32 → 0.6.34

package/dist/execution-concurrency-telemetry.js

@@ -0,0 +1,115 @@
+import { dslog } from "./slog.js";
+const RUNTIMES = [
+    "claude",
+    "codex",
+    "kimi",
+    "hermes",
+    "opencode",
+    "deepseek-harness",
+];
+function snapshotFields(snapshot, tracked) {
+    const runtimeFields = {};
+    for (const runtime of RUNTIMES) {
+        for (const phase of ["queued", "preparing", "starting", "running"]) {
+            runtimeFields[`${runtime.replaceAll("-", "_")}_${phase}_total`] = [...tracked.values()]
+                .filter((entry) => entry.runtime === runtime && entry.phase === phase).length;
+        }
+    }
+    return {
+        active_total: snapshot.activeTotal,
+        admitted_total: snapshot.admittedTotal,
+        queued_total: snapshot.queuedTotal,
+        preparing_total: snapshot.preparingTotal,
+        starting_total: snapshot.startingTotal,
+        running_total: snapshot.runningTotal,
+        memory_prune_admitted_total: snapshot.memoryPruneAdmittedTotal,
+        memory_prune_queued_total: snapshot.memoryPruneQueuedTotal,
+        memory_prune_preparing_total: snapshot.memoryPrunePreparingTotal,
+        memory_prune_starting_total: snapshot.memoryPruneStartingTotal,
+        memory_prune_running_total: snapshot.memoryPruneRunningTotal,
+        runtime_tracked_total: tracked.size,
+        ...runtimeFields,
+    };
+}
+function hasWork(snapshot, tracked) {
+    return tracked.size > 0
+        || snapshot.admittedTotal > 0
+        || snapshot.queuedTotal > 0
+        || snapshot.memoryPruneAdmittedTotal > 0
+        || snapshot.memoryPruneQueuedTotal > 0;
+}
+export function createExecutionConcurrencyTelemetry(slots, options = {}) {
+    const tracked = new Map();
+    const log = options.log ?? dslog;
+    const intervalMs = options.intervalMs ?? 30_000;
+    if (!Number.isFinite(intervalMs) || intervalMs <= 0) {
+        throw new RangeError("concurrency telemetry intervalMs must be positive");
+    }
+    let timer = null;
+    const writeSnapshot = (reason) => {
+        const current = slots.snapshot();
+        if (!hasWork(current, tracked))
+            return;
+        log("execution.concurrency_snapshot", "execution 并发快照", {
+            reason,
+            ...snapshotFields(current, tracked),
+        });
+    };
+    const ensureTimer = () => {
+        if (timer !== null)
+            return;
+        timer = setInterval(() => writeSnapshot("periodic"), intervalMs);
+        timer.unref?.();
+    };
+    const stopTimerIfIdle = () => {
+        if (timer === null || hasWork(slots.snapshot(), tracked))
+            return;
+        clearInterval(timer);
+        timer = null;
+    };
+    return {
+        transition: (identity, phase, fields = {}) => {
+            const previousPhase = tracked.get(identity.executionId)?.phase ?? null;
+            tracked.set(identity.executionId, { runtime: identity.runtime, phase });
+            ensureTimer();
+            log("execution.phase_changed", "execution 阶段变更", {
+                execution_id: identity.executionId,
+                agent_handle: identity.agentHandle,
+                runtime: identity.runtime,
+                execution_class: identity.executionClass,
+                protocol: identity.protocol,
+                previous_phase: previousPhase,
+                phase,
+                ...snapshotFields(slots.snapshot(), tracked),
+                ...fields,
+            });
+        },
+        release: (identity, reason, fields = {}) => {
+            const previousPhase = tracked.get(identity.executionId)?.phase ?? null;
+            tracked.delete(identity.executionId);
+            log("execution.phase_changed", "execution 已释放执行名额", {
+                execution_id: identity.executionId,
+                agent_handle: identity.agentHandle,
+                runtime: identity.runtime,
+                execution_class: identity.executionClass,
+                protocol: identity.protocol,
+                previous_phase: previousPhase,
+                phase: "released",
+                release_reason: reason,
+                ...snapshotFields(slots.snapshot(), tracked),
+                ...fields,
+            });
+            stopTimerIfIdle();
+        },
+        snapshot: (reason) => {
+            writeSnapshot(reason);
+            if (hasWork(slots.snapshot(), tracked))
+                ensureTimer();
+        },
+        stop: () => {
+            if (timer !== null)
+                clearInterval(timer);
+            timer = null;
+        },
+    };
+}

package/dist/execution-journal.js

@@ -129,10 +129,45 @@ const DEFAULT_TERMINATION_GRACE_MS = 30_000;
 const DEFAULT_KILL_VERIFICATION_DELAY_MS = 100;
 const DEFAULT_COMMAND_TIMEOUT_MS = 5_000;
 const directoryQueues = new Map();
-function runSerialized(directory, operation) {
-    const previous = directoryQueues.get(directory) ?? Promise.resolve();
-    const result = previous.then(operation, operation);
-    directoryQueues.set(directory, result.then(() => undefined, () => undefined));
+function directoryQueue(directory) {
+    const existing = directoryQueues.get(directory);
+    if (existing !== undefined)
+        return existing;
+    const created = {
+        exclusiveTail: Promise.resolve(),
+        keyedTails: new Map(),
+    };
+    directoryQueues.set(directory, created);
+    return created;
+}
+/**
+ * Serialize mutations for one execution while allowing unrelated execution
+ * records to make progress concurrently. Directory maintenance scheduled
+ * before this operation remains an ordering barrier.
+ */
+function runKeyedSerialized(directory, key, operation) {
+    const queue = directoryQueue(directory);
+    const previousKey = queue.keyedTails.get(key) ?? Promise.resolve();
+    const result = Promise.all([queue.exclusiveTail, previousKey]).then(operation);
+    const tail = result.then(() => undefined, () => undefined);
+    queue.keyedTails.set(key, tail);
+    void tail.then(() => {
+        if (queue.keyedTails.get(key) === tail)
+            queue.keyedTails.delete(key);
+    });
+    return result;
+}
+/**
+ * Directory-wide work is an exclusive barrier: it waits for every mutation
+ * already scheduled, and later keyed mutations wait for it. Recovery, prune,
+ * initialization, and close use this path because they inspect directory-wide
+ * state or journal lease ownership.
+ */
+function runDirectoryExclusive(directory, operation) {
+    const queue = directoryQueue(directory);
+    const blockers = [queue.exclusiveTail, ...queue.keyedTails.values()];
+    const result = Promise.all(blockers).then(operation);
+    queue.exclusiveTail = result.then(() => undefined, () => undefined);
     return result;
 }
 function runCommand(command, args, options = {}) {
@@ -312,6 +347,7 @@ export function createExecutionJournal(agentsRoot, options = {}) {
     if (!Number.isInteger(maxEntries))
         throw new RangeError("maxEntries must be an integer");
     let initialized = false;
+    let initializationPromise = null;
     const recordPath = (executionId) => join(directory, `${executionId}.json`);
     const runtimeReadyPath = (executionId) => join(directory, `${executionId}${RUNTIME_READY_SUFFIX}`);
     const parseRecord = (path, raw) => {
@@ -528,11 +564,57 @@ export function createExecutionJournal(agentsRoot, options = {}) {
         await pruneInternal();
         initialized = true;
     };
-    const serialized = (operation) => runSerialized(directory, async () => {
-        lease.assertUsable();
-        await initialize();
-        return operation();
-    });
+    const ensureInitialized = () => {
+        if (initialized)
+            return Promise.resolve();
+        if (initializationPromise !== null)
+            return initializationPromise;
+        const initializing = runDirectoryExclusive(directory, async () => {
+            lease.assertUsable();
+            await initialize();
+        });
+        initializationPromise = initializing;
+        void initializing.catch(() => {
+            if (initializationPromise === initializing)
+                initializationPromise = null;
+        });
+        return initializing;
+    };
+    const serializedExecution = async (executionId, operationName, operation) => {
+        await ensureInitialized();
+        const queuedAt = Date.now();
+        return runKeyedSerialized(directory, executionId, async () => {
+            lease.assertUsable();
+            const operationStartedAt = Date.now();
+            let outcome = "succeeded";
+            try {
+                return await operation();
+            }
+            catch (error) {
+                outcome = "failed";
+                throw error;
+            }
+            finally {
+                try {
+                    options.onOperationTiming?.({
+                        executionId,
+                        operation: operationName,
+                        queueWaitMs: operationStartedAt - queuedAt,
+                        operationMs: Date.now() - operationStartedAt,
+                        outcome,
+                    });
+                }
+                catch { /* diagnostics must never affect journal durability */ }
+            }
+        });
+    };
+    const serializedDirectory = async (operation) => {
+        await ensureInitialized();
+        return runDirectoryExclusive(directory, async () => {
+            lease.assertUsable();
+            return operation();
+        });
+    };
     const requireRecord = async (executionId) => {
         const entry = await readRecord(executionId);
         if (entry === null)
@@ -636,7 +718,7 @@ export function createExecutionJournal(agentsRoot, options = {}) {
             validateExecutionId(executionId);
             if (specHash.length === 0)
                 throw new TypeError("specHash must not be empty");
-            return serialized(async () => {
+            return serializedExecution(executionId, "accept", async () => {
                 const existing = await readRecord(executionId);
                 if (existing !== null) {
                     if (existing.specHash !== specHash) {
@@ -669,7 +751,7 @@ export function createExecutionJournal(agentsRoot, options = {}) {
         startGuarded: async (executionId, processStartedAt, startDormant, hooks) => {
             validateExecutionId(executionId);
             TimestampSchema.parse(processStartedAt);
-            return serialized(async () => {
+            return serializedExecution(executionId, "start_guarded", async () => {
                 const entry = await requireRecord(executionId);
                 if (entry.state !== "accepted") {
                     return { kind: "existing", entry: await confirmDurable(entry) };
@@ -729,7 +811,7 @@ export function createExecutionJournal(agentsRoot, options = {}) {
         markRuntimeReady: async (executionId, runtimeReadyAt) => {
             validateExecutionId(executionId);
             TimestampSchema.parse(runtimeReadyAt);
-            return serialized(async () => {
+            return serializedExecution(executionId, "mark_runtime_ready", async () => {
                 const entry = await requireRecord(executionId);
                 if (entry.state !== "running") {
                     throw new JournalTransitionError(`Cannot mark ${entry.state} execution runtime-ready`);
@@ -751,7 +833,7 @@ export function createExecutionJournal(agentsRoot, options = {}) {
             if (completion.executionId !== executionId) {
                 throw new JournalConflictError("Completion executionId does not match the journal record");
             }
-            return serialized(async () => {
+            return serializedExecution(executionId, "complete", async () => {
                 const entry = await requireRecord(executionId);
                 if (entry.state === "completed" || entry.state === "interrupted") {
                     if (sameValue(entry.completion, completion))
@@ -772,7 +854,7 @@ export function createExecutionJournal(agentsRoot, options = {}) {
         },
         acknowledgeCompletion: async (executionId) => {
             validateExecutionId(executionId);
-            return serialized(async () => {
+            const updated = await serializedExecution(executionId, "acknowledge_completion", async () => {
                 const entry = await requireRecord(executionId);
                 if (entry.state !== "completed" && entry.state !== "interrupted") {
                     throw new JournalTransitionError(`Cannot acknowledge ${entry.state} execution`);
@@ -785,20 +867,21 @@ export function createExecutionJournal(agentsRoot, options = {}) {
                     updatedAt: now().toISOString(),
                 });
                 await writeRecord(updated);
-                await pruneInternal();
                 return updated;
             });
+            await serializedDirectory(pruneInternal);
+            return updated;
         },
         get: async (executionId) => {
             validateExecutionId(executionId);
-            return serialized(() => readRecord(executionId));
+            return serializedExecution(executionId, "get", () => readRecord(executionId));
         },
-        replay: async () => serialized(async () => (await readAll())
+        replay: async () => serializedDirectory(async () => (await readAll())
             .filter((entry) => entry.state === "accepted"
             || entry.state === "running"
             || !entry.completionAcknowledged)
             .sort((left, right) => left.executionId.localeCompare(right.executionId))),
-        reconcileAfterRestart: async () => serialized(async () => {
+        reconcileAfterRestart: async () => serializedDirectory(async () => {
             const active = (await readAll()).filter((entry) => entry.state === "accepted" || entry.state === "running");
             const results = await Promise.allSettled(active.map(async (entry) => {
                 if (entry.state === "running")
@@ -812,11 +895,11 @@ export function createExecutionJournal(agentsRoot, options = {}) {
                 throw new AggregateError(failures, "One or more executions could not be reconciled");
             }
         }),
-        prune: async () => serialized(pruneInternal),
+        prune: async () => serializedDirectory(pruneInternal),
         close: async (closeOptions = {}) => {
             const signal = closeOptions.signal;
             let started = false;
-            const closing = runSerialized(directory, async () => {
+            const closing = runDirectoryExclusive(directory, async () => {
                 started = true;
                 if (signal?.aborted)
                     throw signal.reason ?? new Error("Journal close aborted");

package/dist/execution-runner.js

@@ -4,7 +4,7 @@ import { DaemonToServerExecutionFrameSchema, ExecutionCompletedSchema, Execution
 import { JournalConflictError } from "./execution-journal.js";
 import { boundExecutionFrame } from "./execution-event-limit.js";
 import { mintAgentToken } from "./token.js";
-import { executeLocal, withLocalExecutionFacts, } from "./local-executor.js";
+import { executeLocal, RuntimeRunningCallbackTimeoutError, withLocalExecutionFacts, } from "./local-executor.js";
 import { startDormantSupervisor, } from "./execution-supervisor.js";
 import { CLAUDE_EFFORT_LEVELS } from "./runtimes/claude.js";
 import { CODEX_EFFORT_LEVELS } from "./runtimes/codex.js";
@@ -21,6 +21,7 @@ import { createProjectRegistry } from "./project-skills/registry.js";
 import { ProjectContextUnavailableError, resolveProjectContext } from "./project-workspaces/resolver.js";
 import { PROJECT_WORKSPACES_CAPABILITY } from "./machine-info.js";
 import { PROJECT_SKILL_PROJECTION_V2_CAPABILITY } from "./project-skills/types.js";
+import { dslog } from "./slog.js";
 export { supervisorLaunch } from "./supervised-runtime.js";
 const ACTIVITY_KIND = {
     init: "working",
@@ -456,6 +457,7 @@ export async function runExecution(config, input, dependencies) {
     let timeout;
     let timedOut = false;
     let completion;
+    let abandonedRuntimeRunningCompletion = null;
     let memoryCaptureFinalText = null;
     let boundImDecision = spec.reporting.allowBoundImDecision
         ? "silent"
@@ -498,9 +500,11 @@ export async function runExecution(config, input, dependencies) {
         let externalOutputSequence = 0;
         const callbacks = {
             onRuntimeStarting: () => {
-                dependencies.onRuntimePhase?.("starting");
+                dependencies.onRuntimePhase?.("starting", spec.runtime.name);
             },
             onRuntimeRunning: async () => {
+                const callbackStartedAt = Date.now();
+                let callbackStage = "journal";
                 if (dependencies.cancellation?.isRequested())
                     return;
                 const cancel = runtimeCancel;
@@ -515,18 +519,50 @@ export async function runExecution(config, input, dependencies) {
                         }
                     }, effectiveTimeoutMs);
                 }
-                dependencies.onRuntimePhase?.("running");
-                const ready = await dependencies.journal.markRuntimeReady(spec.executionId, now().toISOString());
-                if (ready.runtimeReadyAt === null) {
-                    throw new Error(`Execution ${spec.executionId} runtime readiness was not persisted`);
+                dependencies.onRuntimePhase?.("running", spec.runtime.name);
+                const journalStartedAt = Date.now();
+                try {
+                    const ready = await dependencies.journal.markRuntimeReady(spec.executionId, now().toISOString());
+                    const journalMs = Date.now() - journalStartedAt;
+                    if (ready.runtimeReadyAt === null) {
+                        throw new Error(`Execution ${spec.executionId} runtime readiness was not persisted`);
+                    }
+                    if (abandonedRuntimeRunningCompletion !== null) {
+                        await dependencies.journal.complete(spec.executionId, abandonedRuntimeRunningCompletion);
+                        await dependencies.report(abandonedRuntimeRunningCompletion);
+                        return;
+                    }
+                    startedAt = ready.runtimeReadyAt;
+                    callbackStage = "started_report";
+                    const reportStartedAt = Date.now();
+                    const reportDelivered = await reportBestEffort(dependencies.report, boundExecutionFrame(DaemonToServerExecutionFrameSchema.parse({
+                        type: "execution:started",
+                        protocolVersion: 1,
+                        executionId: spec.executionId,
+                        at: startedAt,
+                    }), config.executionLimits.maxEventBytes), bestEffortTimeoutMs);
+                    dslog("execution.runtime_running_callback", "runtime-running callback 已完成", {
+                        execution_id: spec.executionId,
+                        runtime: spec.runtime.name,
+                        outcome: "succeeded",
+                        journal_ms: journalMs,
+                        started_report_ms: Date.now() - reportStartedAt,
+                        started_report_delivered: reportDelivered,
+                        callback_total_ms: Date.now() - callbackStartedAt,
+                    });
+                }
+                catch (error) {
+                    dslog("execution.runtime_running_callback", "runtime-running callback 失败", {
+                        level: "ERROR",
+                        execution_id: spec.executionId,
+                        runtime: spec.runtime.name,
+                        outcome: "failed",
+                        failed_stage: callbackStage,
+                        callback_total_ms: Date.now() - callbackStartedAt,
+                        error_message: error instanceof Error ? error.message.slice(0, 500) : String(error).slice(0, 500),
+                    });
+                    throw error;
                 }
-                startedAt = ready.runtimeReadyAt;
-                await reportBestEffort(dependencies.report, boundExecutionFrame(DaemonToServerExecutionFrameSchema.parse({
-                    type: "execution:started",
-                    protocolVersion: 1,
-                    executionId: spec.executionId,
-                    at: startedAt,
-                }), config.executionLimits.maxEventBytes), bestEffortTimeoutMs);
             },
             ...(spec.reporting.streamActivity ? {
                 onActivity: (activity) => {
@@ -812,7 +848,7 @@ export async function runExecution(config, input, dependencies) {
         else {
             await proveActiveSupervisorStopped(error, spec.agent.projectSkillBindingGeneration !== undefined);
         }
-        completion = cancelled
+        const failureCompletion = cancelled
             ? ExecutionCompletedSchema.parse({
                 type: "execution:completed",
                 protocolVersion: 1,
@@ -827,6 +863,12 @@ export async function runExecution(config, input, dependencies) {
                 finishedAt: now().toISOString(),
             })
             : failedCompletion(spec, error, startedAt, now().toISOString());
+        if (error instanceof RuntimeRunningCallbackTimeoutError) {
+            abandonedRuntimeRunningCompletion = boundExecutionFrame(failureCompletion, config.executionLimits.maxEventBytes);
+            await telemetry.closeAndDrain();
+            throw error;
+        }
+        completion = failureCompletion;
     }
     completion = boundExecutionFrame(completion, config.executionLimits.maxEventBytes);
     await telemetry.closeAndDrain();

package/dist/local-executor.js

@@ -60,8 +60,18 @@ function logMemoryPruneDiagnosticsFailure(input, traceId, phase, error) {
         error_code: errorCode,
     });
 }
+export class RuntimeRunningCallbackTimeoutError extends Error {
+    constructor(runtime, timeoutMs) {
+        super(`${runtime} runtime-running callback timed out after ${timeoutMs}ms`);
+        this.name = "RuntimeRunningCallbackTimeoutError";
+    }
+}
 const STDERR_TAIL_CAP = 2_000;
-const RUNTIME_RUNNING_CALLBACK_TIMEOUT_MS = 10_000;
+// This is a liveness fuse, not a normal durability latency budget. Journal
+// mutations are keyed per execution, so a healthy callback should finish in
+// milliseconds; 30s tolerates a transient filesystem stall without reviving
+// the old failure mode where a callback could hold execution slots forever.
+const RUNTIME_RUNNING_CALLBACK_TIMEOUT_MS = 30_000;
 const MEMORY_PRUNE_DIAGNOSTICS_TIMEOUT_MS = 2_000;
 const LOCAL_MEMORY_DIAGNOSTICS_TIMEOUT_MS = 250;
 const CLAUDE_INSTRUCTION_WARNING = "claude_additional_directory_instructions_unverified";
@@ -682,19 +692,29 @@ async function executeLocalUnlocked(input, callbacks, dependencies) {
             if (runtimeRunning)
                 return;
             runtimeRunning = true;
+            const callbackStartedAt = Date.now();
             let callbackTimer;
             runtimeRunningNotification = Promise.race([
                 Promise.resolve().then(() => callbacks.onRuntimeRunning?.(runtime.name)),
                 new Promise((_resolve, reject) => {
                     const callbackTimeoutMs = dependencies.runtimeRunningCallbackTimeoutMs
                         ?? RUNTIME_RUNNING_CALLBACK_TIMEOUT_MS;
-                    callbackTimer = setTimeout(() => reject(new Error(`${runtime.name} runtime-running callback timed out after ${callbackTimeoutMs}ms`)), callbackTimeoutMs);
+                    callbackTimer = setTimeout(() => reject(new RuntimeRunningCallbackTimeoutError(runtime.name, callbackTimeoutMs)), callbackTimeoutMs);
                 }),
             ]).finally(() => {
                 if (callbackTimer !== undefined)
                     clearTimeout(callbackTimer);
             });
             void runtimeRunningNotification.catch((error) => {
+                dslog("runtime.running_callback_failed", "runtime-running 回调失败", {
+                    level: "ERROR",
+                    execution_id: input.executionId,
+                    runtime: runtime.name,
+                    callback_ms: Date.now() - callbackStartedAt,
+                    error_message: error instanceof Error
+                        ? error.message.slice(0, 500)
+                        : String(error).slice(0, 500),
+                });
                 // Persisting the running phase is part of execution liveness. Propagate
                 // its failure immediately: cancellation is best-effort and must not be
                 // allowed to keep the execution or its slots alive indefinitely.

package/dist/project-skills/serve-startup.js

@@ -8,6 +8,21 @@ export async function initializeProjectSkillProtectedExecutionJournal(input) {
             input.protectedExecutionIds.add(executionId);
         const journal = input.injectedJournal ?? createExecutionJournal(input.config.agentsRoot, {
             protectedExecutionIds: () => input.protectedExecutionIds,
+            onOperationTiming: (event) => {
+                if (event.operation !== "mark_runtime_ready"
+                    && event.queueWaitMs < 250
+                    && event.operationMs < 250)
+                    return;
+                dslog("execution.journal_operation", "execution journal 操作完成", {
+                    level: event.outcome === "succeeded" ? "INFO" : "ERROR",
+                    execution_id: event.executionId,
+                    operation: event.operation,
+                    outcome: event.outcome,
+                    queue_wait_ms: event.queueWaitMs,
+                    operation_ms: event.operationMs,
+                    total_ms: event.queueWaitMs + event.operationMs,
+                });
+            },
         });
         await reconcileExecutionJournal(journal, {
             agentsRoot: input.config.agentsRoot,