@msm-core/jobs 0.4.0 → 0.6.0

package/dist/orchestrator.js

@@ -19,11 +19,31 @@ function isDuplicateKeyError(error) {
 export function createOrchestrator(deps) {
     const { jobStore, queue, ops, clock, registry, stepExecutor, resumeToken, config } = deps;
     const runConfig = resolveRunConfig(config);
-    function buildIdempotencyKey(input, version, currentStep) {
+    function buildIdempotencyKey(input, currentStep) {
         if (input.idempotencyKey)
             return input.idempotencyKey;
-        const reason = input.reason || "queue";
-        return `${input.jobId}:${version}:${currentStep}:${reason}`;
+        // STABLE across every re-delivery of the same logical step: a given job's
+        // step executes exactly once. The key MUST NOT include the version (the
+        // claim increments it on every delivery) or the trigger reason (varies by
+        // delivery) — either makes a post-crash re-delivery compute a different
+        // key, miss the already-recorded step, and re-run the side effect.
+        return `${input.jobId}:step:${currentStep}`;
+    }
+    /** Terminal step-failure path: force the job to `failed` and emit the event. */
+    async function failJob(input, claimed, error) {
+        await jobStore.markFailed({ jobId: input.jobId, tenantId: input.tenantId });
+        ops.record({
+            jobId: input.jobId,
+            tenantId: input.tenantId,
+            missionId: claimed.missionId,
+            source: "job_orchestrator",
+            eventType: "job.failed",
+            status: "failed",
+            queueName: QUEUE_NAME,
+            employeeId: claimed.employeeId,
+            correlationId: claimed.correlationId,
+            metadata: { error: error.message },
+        });
     }
     async function enqueueBurstContinuation(input) {
         const continuation = await jobStore.getJob(input.jobId, input.tenantId);
@@ -75,9 +95,65 @@ export function createOrchestrator(deps) {
             });
             if (!claimed)
                 return; // CAS claim lost — another worker owns this tick.
-            const idempotencyKey = buildIdempotencyKey(input, claimed.version, claimed.currentStep);
+            const idempotencyKey = buildIdempotencyKey(input, claimed.currentStep);
             const existingStep = await jobStore.findStepByIdempotencyKey(input.jobId, input.tenantId, idempotencyKey);
             if (existingStep) {
+                // The step already executed (its side effect ran exactly once). Two cases:
+                //
+                // (a) CRASH RECOVERY — the original finalize never landed (process died
+                //     between insertStep and finalizeStep), so the job is still at the
+                //     pre-step `currentStep`. Re-apply the transition the step recorded,
+                //     advancing the job WITHOUT re-running the side effect, then keep it
+                //     moving. This is what makes the engine crash-safe / exactly-once.
+                if (existingStep.finalize && claimed.currentStep < existingStep.stepNumber) {
+                    const recovered = await jobStore.finalizeStep({
+                        jobId: input.jobId,
+                        tenantId: input.tenantId,
+                        expectedVersion: claimed.version,
+                        set: existingStep.finalize.set,
+                        ...(existingStep.finalize.incCounters
+                            ? { incCounters: existingStep.finalize.incCounters }
+                            : {}),
+                    });
+                    ops.record({
+                        jobId: input.jobId,
+                        tenantId: input.tenantId,
+                        missionId: claimed.missionId,
+                        source: "job_orchestrator",
+                        eventType: "job.step.recovered",
+                        status: recovered?.status ?? claimed.status,
+                        queueName: QUEUE_NAME,
+                        employeeId: claimed.employeeId,
+                        correlationId: claimed.correlationId,
+                        metadata: { idempotencyKey, stepNumber: existingStep.stepNumber },
+                    });
+                    // Keep the recovered job alive: a waiting_time job needs its resume
+                    // re-enqueued; a runnable status needs another tick. Terminal /
+                    // waiting_event / awaiting_approval statuses are driven externally.
+                    if (recovered && config.resumeMode !== "scan_only") {
+                        if (recovered.status === "waiting_time" && recovered.resumeAt) {
+                            await queue.enqueue({
+                                jobId: input.jobId,
+                                tenantId: input.tenantId,
+                                reason: "crash_recovery_resume",
+                                delayMs: Math.max(1, recovered.resumeAt.getTime() - clock.now().getTime()),
+                                dedupeKey: `recover:${existingStep.stepNumber}`,
+                            });
+                        }
+                        else if (isRunnableJobStatus(recovered.status)) {
+                            await queue.enqueue({
+                                jobId: input.jobId,
+                                tenantId: input.tenantId,
+                                reason: "crash_recovery_continue",
+                                dedupeKey: `recover:${existingStep.stepNumber}`,
+                            });
+                        }
+                    }
+                    return;
+                }
+                // (b) CONCURRENT DUPLICATE — the finalize already landed (another worker
+                //     advanced the job past this step). Just restore our `running` claim
+                //     to the correct resting status and skip.
                 const restoredStatus = claimed.waitEvent
                     ? "waiting_event"
                     : claimed.resumeAt
@@ -121,8 +197,11 @@ export function createOrchestrator(deps) {
                 correlationId: claimed.correlationId,
                 metadata: { stepNumber, reason: input.reason || "queue", idempotencyKey },
             });
+            // ── Resolve the next transition (pure). A throw here is a workflow-definition
+            //    bug — fail the job. ──
+            let resolution;
             try {
-                const resolution = registry.resolveStep({
+                resolution = registry.resolveStep({
                     workflowType: claimed.workflowType,
                     workflowVersion: claimed.workflowVersion,
                     budget: claimed.budget,
@@ -131,276 +210,288 @@ export function createOrchestrator(deps) {
                     reason: input.reason || "queue",
                     now: stepStartedAt,
                 });
-                // COST CEILING (P2-8): before a COSTLY step, if the tenant has hit its day-ceiling, DEFER the
-                // job to the next UTC midnight (waiting_time) instead of running the step. No step is recorded.
-                if (resolution.stepType === "run_interaction_task" &&
-                    config.costCeilingUsd > 0 &&
-                    deps.costPort) {
-                    const now2 = clock.now();
-                    const spend = await deps.costPort.tenantDaySpendUsd(input.tenantId, startOfUtcDay(now2));
-                    if (spend >= config.costCeilingUsd) {
-                        const resumeAt = nextUtcMidnight(now2);
-                        const deferred = await jobStore.finalizeStep({
-                            jobId: input.jobId,
-                            tenantId: input.tenantId,
-                            expectedVersion: claimed.version,
-                            set: { status: "waiting_time", resumeAt },
-                        });
-                        ops.record({
+            }
+            catch (error) {
+                await failJob(input, claimed, error);
+                throw error;
+            }
+            // COST CEILING (P2-8): before a COSTLY step, if the tenant has hit its day-ceiling, DEFER the
+            // job to the next UTC midnight (waiting_time) instead of running the step. No step is recorded.
+            if (resolution.stepType === "run_interaction_task" &&
+                config.costCeilingUsd > 0 &&
+                deps.costPort) {
+                const now2 = clock.now();
+                const spend = await deps.costPort.tenantDaySpendUsd(input.tenantId, startOfUtcDay(now2));
+                if (spend >= config.costCeilingUsd) {
+                    const resumeAt = nextUtcMidnight(now2);
+                    const deferred = await jobStore.finalizeStep({
+                        jobId: input.jobId,
+                        tenantId: input.tenantId,
+                        expectedVersion: claimed.version,
+                        set: { status: "waiting_time", resumeAt },
+                    });
+                    ops.record({
+                        jobId: input.jobId,
+                        tenantId: input.tenantId,
+                        missionId: claimed.missionId,
+                        source: "job_orchestrator",
+                        eventType: "job.cost_ceiling_deferred",
+                        status: "waiting_time",
+                        queueName: QUEUE_NAME,
+                        employeeId: claimed.employeeId,
+                        correlationId: claimed.correlationId,
+                        metadata: { spendUsd: spend, ceilingUsd: config.costCeilingUsd, resumeAt: resumeAt.toISOString() },
+                    });
+                    if (deferred && config.resumeMode !== "scan_only") {
+                        await queue.enqueue({
                             jobId: input.jobId,
                             tenantId: input.tenantId,
-                            missionId: claimed.missionId,
-                            source: "job_orchestrator",
-                            eventType: "job.cost_ceiling_deferred",
-                            status: "waiting_time",
-                            queueName: QUEUE_NAME,
-                            employeeId: claimed.employeeId,
-                            correlationId: claimed.correlationId,
-                            metadata: { spendUsd: spend, ceilingUsd: config.costCeilingUsd, resumeAt: resumeAt.toISOString() },
+                            reason: "cost_ceiling_resume",
+                            delayMs: Math.max(1, resumeAt.getTime() - now2.getTime()),
+                            dedupeKey: `ceiling:${resumeAt.getTime()}`,
                         });
-                        if (deferred && config.resumeMode !== "scan_only") {
-                            await queue.enqueue({
-                                jobId: input.jobId,
-                                tenantId: input.tenantId,
-                                reason: "cost_ceiling_resume",
-                                delayMs: Math.max(1, resumeAt.getTime() - now2.getTime()),
-                                dedupeKey: `ceiling:${resumeAt.getTime()}`,
-                            });
-                        }
-                        return;
                     }
+                    return;
                 }
-                // Run the real side-effecting work (agent turn for run_interaction_task; {} for
-                // pure transitions). Throwing here fails the step via the catch below.
-                const execResult = await stepExecutor.execute({
+            }
+            // ── Run the real side-effecting work (agent turn for run_interaction_task; {}
+            //    for pure transitions). A throw here is a genuine STEP failure — fail the
+            //    job. (Infra failures further down do NOT fail the job; see below.) ──
+            let execResult;
+            try {
+                execResult = await stepExecutor.execute({
                     stepType: resolution.stepType,
                     triggeredBy: resolution.triggeredBy,
                     job: claimed,
                     resolution,
                     stepNumber,
                 });
-                const stepEndedAt = clock.now();
-                const durationMs = stepEndedAt.getTime() - stepStartedAt.getTime();
-                const mergedOutput = { ...(resolution.output ?? {}), ...(execResult.output ?? {}) };
-                const mergedStatePatch = {
-                    ...(resolution.statePatch ?? {}),
-                    ...(execResult.statePatch ?? {}),
-                };
-                const step = {
-                    stepId: deps.genStepId(),
+            }
+            catch (error) {
+                await failJob(input, claimed, error);
+                throw error;
+            }
+            const stepEndedAt = clock.now();
+            const durationMs = stepEndedAt.getTime() - stepStartedAt.getTime();
+            const mergedOutput = { ...(resolution.output ?? {}), ...(execResult.output ?? {}) };
+            const mergedStatePatch = {
+                ...(resolution.statePatch ?? {}),
+                ...(execResult.statePatch ?? {}),
+            };
+            // Compute the FULL transition BEFORE recording the step, so the step carries
+            // the durable decision (crash recovery re-applies it WITHOUT re-running).
+            const nextStatus = resolution.nextStatus;
+            const nextResumeAt = nextStatus === "waiting_time"
+                ? new Date(clock.now().getTime() + (resolution.resumeDelayMs ?? config.defaultWaitMs))
+                : null;
+            const waitEventName = resolution.waitEventName || "job.resume";
+            const issuedWaitEventToken = nextStatus === "waiting_event"
+                ? resumeToken.issue({
                     jobId: input.jobId,
                     tenantId: input.tenantId,
+                    eventName: waitEventName,
+                })
+                : null;
+            const nextWaitEvent = nextStatus === "waiting_event"
+                ? { eventName: waitEventName, token: issuedWaitEventToken?.token ?? null }
+                : null;
+            const nextState = { ...(claimed.state ?? {}), ...mergedStatePatch };
+            // Accrue this step's USD cost onto the job (powers the tenant/day ceiling, P2-8).
+            const stepCostUsd = typeof execResult.costUsd === "number" && execResult.costUsd > 0 ? execResult.costUsd : 0;
+            const finalizeArgs = {
+                set: {
+                    currentStep: stepNumber,
+                    status: nextStatus,
+                    resumeAt: nextResumeAt,
+                    waitEvent: nextWaitEvent,
+                    state: nextState,
+                    completedAt: isTerminalJobStatus(nextStatus) ? stepEndedAt : null,
+                },
+                incCounters: { totalStepsExecuted: 1, attempt: 1, ...(stepCostUsd ? { totalCostUsd: stepCostUsd } : {}) },
+            };
+            const step = {
+                stepId: deps.genStepId(),
+                jobId: input.jobId,
+                tenantId: input.tenantId,
+                stepNumber,
+                stepType: resolution.stepType,
+                status: "completed",
+                triggeredBy: resolution.triggeredBy,
+                startedAt: stepStartedAt,
+                endedAt: stepEndedAt,
+                durationMs,
+                toolCalls: [],
+                output: mergedOutput,
+                error: null,
+                nextStep: null,
+                transitionReason: resolution.transitionReason,
+                idempotencyKey,
+                finalize: finalizeArgs,
+            };
+            // ── Record the step (the idempotency marker + durable transition). A concurrent
+            //    insert of the same key → skip. ANY OTHER throw here is an infra/transport
+            //    failure (Mongo blip): it must PROPAGATE so the queue retries the tick — it
+            //    must NOT fail an otherwise-healthy job. The stable idempotency key makes the
+            //    retry safe (it finds this step and recovers). ──
+            try {
+                await jobStore.insertStep(step);
+            }
+            catch (error) {
+                if (isDuplicateKeyError(error)) {
+                    ops.record({
+                        jobId: input.jobId,
+                        tenantId: input.tenantId,
+                        missionId: claimed.missionId,
+                        source: "job_orchestrator",
+                        eventType: "job.step.duplicate_skipped",
+                        status: claimed.status,
+                        queueName: QUEUE_NAME,
+                        employeeId: claimed.employeeId,
+                        correlationId: claimed.correlationId,
+                        metadata: { idempotencyKey },
+                    });
+                    return;
+                }
+                throw error; // retryable infra error — do NOT markFailed (H4)
+            }
+            const finalized = await jobStore.finalizeStep({
+                jobId: input.jobId,
+                tenantId: input.tenantId,
+                expectedVersion: claimed.version,
+                ...finalizeArgs,
+            });
+            if (!finalized)
+                return; // CAS finalize lost (or crash); a re-delivery recovers it.
+            stepsExecuted += 1;
+            ops.record({
+                jobId: input.jobId,
+                tenantId: input.tenantId,
+                missionId: finalized.missionId,
+                source: "job_orchestrator",
+                eventType: "job.step.completed",
+                status: finalized.status,
+                queueName: QUEUE_NAME,
+                employeeId: finalized.employeeId,
+                correlationId: finalized.correlationId,
+                metadata: {
                     stepNumber,
                     stepType: resolution.stepType,
-                    status: "completed",
-                    triggeredBy: resolution.triggeredBy,
-                    startedAt: stepStartedAt,
-                    endedAt: stepEndedAt,
-                    durationMs,
-                    toolCalls: [],
-                    output: mergedOutput,
-                    error: null,
-                    nextStep: null,
                     transitionReason: resolution.transitionReason,
+                    durationMs,
                     idempotencyKey,
-                };
-                await jobStore.insertStep(step);
-                const nextStatus = resolution.nextStatus;
-                const nextResumeAt = nextStatus === "waiting_time"
-                    ? new Date(clock.now().getTime() + (resolution.resumeDelayMs ?? config.defaultWaitMs))
-                    : null;
-                const waitEventName = resolution.waitEventName || "job.resume";
-                const issuedWaitEventToken = nextStatus === "waiting_event"
-                    ? resumeToken.issue({
-                        jobId: input.jobId,
-                        tenantId: input.tenantId,
-                        eventName: waitEventName,
-                    })
-                    : null;
-                const nextWaitEvent = nextStatus === "waiting_event"
-                    ? { eventName: waitEventName, token: issuedWaitEventToken?.token ?? null }
-                    : null;
-                const nextState = { ...(claimed.state ?? {}), ...mergedStatePatch };
-                // Accrue this step's USD cost onto the job (powers the tenant/day ceiling, P2-8).
-                const stepCostUsd = typeof execResult.costUsd === "number" && execResult.costUsd > 0 ? execResult.costUsd : 0;
-                const finalized = await jobStore.finalizeStep({
+                    stepsExecutedThisTick: stepsExecuted,
+                    maxStepsPerTick: runConfig.maxStepsPerTick,
+                },
+            });
+            if (nextStatus === "completed") {
+                ops.record({
                     jobId: input.jobId,
                     tenantId: input.tenantId,
-                    expectedVersion: claimed.version,
-                    set: {
-                        currentStep: stepNumber,
-                        status: nextStatus,
-                        resumeAt: nextResumeAt,
-                        waitEvent: nextWaitEvent,
-                        state: nextState,
-                        completedAt: isTerminalJobStatus(nextStatus) ? clock.now() : null,
-                    },
-                    incCounters: { totalStepsExecuted: 1, attempt: 1, ...(stepCostUsd ? { totalCostUsd: stepCostUsd } : {}) },
+                    missionId: finalized.missionId,
+                    source: "job_orchestrator",
+                    eventType: "job.completed",
+                    status: finalized.status,
+                    queueName: QUEUE_NAME,
+                    employeeId: finalized.employeeId,
+                    correlationId: finalized.correlationId,
                 });
-                if (!finalized)
-                    return; // CAS finalize lost.
-                stepsExecuted += 1;
+                return;
+            }
+            if (nextStatus === "failed") {
                 ops.record({
                     jobId: input.jobId,
                     tenantId: input.tenantId,
                     missionId: finalized.missionId,
                     source: "job_orchestrator",
-                    eventType: "job.step.completed",
+                    eventType: "job.failed",
                     status: finalized.status,
                     queueName: QUEUE_NAME,
                     employeeId: finalized.employeeId,
                     correlationId: finalized.correlationId,
-                    metadata: {
-                        stepNumber,
-                        stepType: resolution.stepType,
-                        transitionReason: resolution.transitionReason,
-                        durationMs,
-                        idempotencyKey,
-                        stepsExecutedThisTick: stepsExecuted,
-                        maxStepsPerTick: runConfig.maxStepsPerTick,
-                    },
                 });
-                if (nextStatus === "completed") {
-                    ops.record({
-                        jobId: input.jobId,
-                        tenantId: input.tenantId,
-                        missionId: finalized.missionId,
-                        source: "job_orchestrator",
-                        eventType: "job.completed",
-                        status: finalized.status,
-                        queueName: QUEUE_NAME,
-                        employeeId: finalized.employeeId,
-                        correlationId: finalized.correlationId,
-                    });
-                    return;
-                }
-                if (nextStatus === "failed") {
-                    ops.record({
-                        jobId: input.jobId,
-                        tenantId: input.tenantId,
-                        missionId: finalized.missionId,
-                        source: "job_orchestrator",
-                        eventType: "job.failed",
-                        status: finalized.status,
-                        queueName: QUEUE_NAME,
-                        employeeId: finalized.employeeId,
-                        correlationId: finalized.correlationId,
-                    });
-                    return;
-                }
-                if (nextStatus === "waiting_event") {
-                    ops.record({
-                        jobId: input.jobId,
-                        tenantId: input.tenantId,
-                        missionId: finalized.missionId,
-                        source: "job_orchestrator",
-                        eventType: "job.waiting_event",
-                        status: finalized.status,
-                        queueName: QUEUE_NAME,
-                        employeeId: finalized.employeeId,
-                        correlationId: finalized.correlationId,
-                        metadata: {
-                            stepNumber,
-                            waitEventName: nextWaitEvent?.eventName ?? null,
-                            resumeTokenExpiresAt: issuedWaitEventToken?.expiresAt ?? null,
-                        },
-                    });
-                    return;
-                }
-                if (nextStatus === "awaiting_approval") {
-                    await handleAwaitingApproval(finalized, resolution, stepNumber, input);
-                    return;
-                }
-                if (nextStatus !== "waiting_time")
-                    return;
+                return;
+            }
+            if (nextStatus === "waiting_event") {
                 ops.record({
                     jobId: input.jobId,
                     tenantId: input.tenantId,
                     missionId: finalized.missionId,
                     source: "job_orchestrator",
-                    eventType: "job.waiting_time",
+                    eventType: "job.waiting_event",
                     status: finalized.status,
                     queueName: QUEUE_NAME,
                     employeeId: finalized.employeeId,
                     correlationId: finalized.correlationId,
                     metadata: {
                         stepNumber,
-                        transitionReason: resolution.transitionReason,
-                        resumeAt: nextResumeAt?.toISOString(),
-                        resumeMode: config.resumeMode,
+                        waitEventName: nextWaitEvent?.eventName ?? null,
+                        resumeTokenExpiresAt: issuedWaitEventToken?.expiresAt ?? null,
                     },
                 });
-                const resumeDelayMs = nextResumeAt
-                    ? Math.max(1, nextResumeAt.getTime() - clock.now().getTime())
-                    : config.defaultWaitMs;
-                const canInlineContinue = stepsExecuted < runConfig.maxStepsPerTick &&
-                    resumeDelayMs <= runConfig.inlineResumeMaxDelayMs;
-                if (canInlineContinue) {
-                    const inlined = await jobStore.claim({
-                        jobId: input.jobId,
-                        tenantId: input.tenantId,
-                        expectedVersion: finalized.version,
-                        statusIn: ["waiting_time"],
-                        set: { status: "queued", resumeAt: null },
-                    });
-                    if (!inlined)
-                        return;
-                    ops.record({
-                        jobId: input.jobId,
-                        tenantId: input.tenantId,
-                        missionId: inlined.missionId,
-                        source: "job_orchestrator",
-                        eventType: "job.waiting_time.inline_continued",
-                        status: inlined.status,
-                        queueName: QUEUE_NAME,
-                        employeeId: inlined.employeeId,
-                        correlationId: inlined.correlationId,
-                        metadata: { stepNumber, resumeDelayMs, inlineResumeMaxDelayMs: runConfig.inlineResumeMaxDelayMs },
-                    });
-                    continue;
-                }
-                if (config.resumeMode === "scan_only")
-                    return;
-                await queue.enqueue({
+                return;
+            }
+            if (nextStatus === "awaiting_approval") {
+                await handleAwaitingApproval(finalized, resolution, stepNumber, input);
+                return;
+            }
+            if (nextStatus !== "waiting_time")
+                return;
+            ops.record({
+                jobId: input.jobId,
+                tenantId: input.tenantId,
+                missionId: finalized.missionId,
+                source: "job_orchestrator",
+                eventType: "job.waiting_time",
+                status: finalized.status,
+                queueName: QUEUE_NAME,
+                employeeId: finalized.employeeId,
+                correlationId: finalized.correlationId,
+                metadata: {
+                    stepNumber,
+                    transitionReason: resolution.transitionReason,
+                    resumeAt: nextResumeAt?.toISOString(),
+                    resumeMode: config.resumeMode,
+                },
+            });
+            const resumeDelayMs = nextResumeAt
+                ? Math.max(1, nextResumeAt.getTime() - clock.now().getTime())
+                : config.defaultWaitMs;
+            const canInlineContinue = stepsExecuted < runConfig.maxStepsPerTick &&
+                resumeDelayMs <= runConfig.inlineResumeMaxDelayMs;
+            if (canInlineContinue) {
+                const inlined = await jobStore.claim({
                     jobId: input.jobId,
                     tenantId: input.tenantId,
-                    reason: "waiting_time_resume",
-                    delayMs: resumeDelayMs,
-                    dedupeKey: `${stepNumber}:${idempotencyKey}`,
+                    expectedVersion: finalized.version,
+                    statusIn: ["waiting_time"],
+                    set: { status: "queued", resumeAt: null },
                 });
-                return;
-            }
-            catch (error) {
-                if (isDuplicateKeyError(error)) {
-                    ops.record({
-                        jobId: input.jobId,
-                        tenantId: input.tenantId,
-                        missionId: claimed.missionId,
-                        source: "job_orchestrator",
-                        eventType: "job.step.duplicate_skipped",
-                        status: claimed.status,
-                        queueName: QUEUE_NAME,
-                        employeeId: claimed.employeeId,
-                        correlationId: claimed.correlationId,
-                        metadata: { idempotencyKey },
-                    });
+                if (!inlined)
                     return;
-                }
-                await jobStore.markFailed({ jobId: input.jobId, tenantId: input.tenantId });
                 ops.record({
                     jobId: input.jobId,
                     tenantId: input.tenantId,
-                    missionId: claimed.missionId,
+                    missionId: inlined.missionId,
                     source: "job_orchestrator",
-                    eventType: "job.failed",
-                    status: "failed",
+                    eventType: "job.waiting_time.inline_continued",
+                    status: inlined.status,
                     queueName: QUEUE_NAME,
-                    employeeId: claimed.employeeId,
-                    correlationId: claimed.correlationId,
-                    metadata: { error: error.message },
+                    employeeId: inlined.employeeId,
+                    correlationId: inlined.correlationId,
+                    metadata: { stepNumber, resumeDelayMs, inlineResumeMaxDelayMs: runConfig.inlineResumeMaxDelayMs },
                 });
-                throw error;
+                continue;
             }
+            if (config.resumeMode === "scan_only")
+                return;
+            await queue.enqueue({
+                jobId: input.jobId,
+                tenantId: input.tenantId,
+                reason: "waiting_time_resume",
+                delayMs: resumeDelayMs,
+                dedupeKey: `${stepNumber}:${idempotencyKey}`,
+            });
+            return;
         }
         await enqueueBurstContinuation(input);
     }
@@ -452,4 +543,3 @@ export function createOrchestrator(deps) {
     }
     return { orchestrateJobStep };
 }
-//# sourceMappingURL=orchestrator.js.map

package/dist/port.d.ts

@@ -178,4 +178,3 @@ export interface ApprovalCreatorPort {
 export interface IdGeneratorPort {
     jobStepId(): string;
 }
-//# sourceMappingURL=port.d.ts.map

package/dist/port.js

@@ -14,4 +14,3 @@ export class DuplicateStepError extends Error {
         this.name = "DuplicateStepError";
     }
 }
-//# sourceMappingURL=port.js.map

package/dist/reconciler.d.ts

@@ -22,4 +22,3 @@ export interface Reconciler {
     reconcileWaitingTimeJobs(asOf?: Date): Promise<WaitingTimeReconcileResult>;
 }
 export declare function createReconciler(deps: ReconcilerDeps): Reconciler;
-//# sourceMappingURL=reconciler.d.ts.map

package/dist/reconciler.js

@@ -72,4 +72,3 @@ export function createReconciler(deps) {
     }
     return { reconcileWaitingTimeJobs };
 }
-//# sourceMappingURL=reconciler.js.map

package/dist/resume-token.d.ts

@@ -37,4 +37,3 @@ export interface ResumeTokenService {
     ttlSeconds: number;
 }
 export declare function createResumeTokenService(deps: ResumeTokenDeps): ResumeTokenService;
-//# sourceMappingURL=resume-token.d.ts.map