@xopcai/xopc 0.0.178 → 0.0.180

package/dist/src/automations/service/action-executor.js

@@ -2,35 +2,71 @@ import { createLogger } from "../../utils/logger/index.js";
 import { init_logger } from "../../utils/logger.js";
 import { init_agent_scope, normalizeAgentId } from "../../agent/agent-scope.js";
 import { buildSessionKey, init_session_key } from "../../routing/session-key.js";
-import "../domain/defaults.js";
+import { resolveAutomationTimeoutSeconds } from "../domain/defaults.js";
 import crypto from "node:crypto";
 //#region src/automations/service/action-executor.ts
 init_agent_scope();
 init_session_key();
 init_logger();
 const log = createLogger("Automation:ActionExecutor");
-function withTimeout(promise, timeoutMs, signal) {
-	return new Promise((resolve, reject) => {
-		if (signal.aborted) {
-			reject(/* @__PURE__ */ new Error("Automation run was cancelled"));
-			return;
+const CANCELLATION_GRACE_MS = 1e4;
+var AutomationExecutionStoppedError = class extends Error {
+	constructor(status, cancellationConfirmed, deadlineAtMs, timeoutMs) {
+		super(status === "timeout" ? `Automation timed out after ${timeoutMs}ms` : "Automation run was cancelled");
+		this.status = status;
+		this.cancellationConfirmed = cancellationConfirmed;
+		this.deadlineAtMs = deadlineAtMs;
+		this.name = "AutomationExecutionStoppedError";
+	}
+};
+async function waitForCancellation(promise) {
+	let timer;
+	try {
+		return await Promise.race([promise.then(() => true, () => true), new Promise((resolve) => {
+			timer = setTimeout(() => resolve(false), CANCELLATION_GRACE_MS);
+		})]);
+	} finally {
+		if (timer) clearTimeout(timer);
+	}
+}
+async function executeWithDeadline(operation, timeoutMs, parentSignal, deadlineAtMs) {
+	const controller = new AbortController();
+	let stoppedAs;
+	const stop = (status, reason) => {
+		if (controller.signal.aborted) return;
+		stoppedAs = status;
+		controller.abort(reason);
+	};
+	const onParentAbort = () => stop("cancelled", parentSignal.reason);
+	parentSignal.addEventListener("abort", onParentAbort, { once: true });
+	if (parentSignal.aborted) onParentAbort();
+	const timer = setTimeout(() => stop("timeout", /* @__PURE__ */ new Error(`Automation timed out after ${timeoutMs}ms`)), Math.max(0, deadlineAtMs - Date.now()));
+	const operationPromise = operation(controller.signal);
+	const rejectForAbort = () => {
+		throw controller.signal.reason ?? /* @__PURE__ */ new Error("Automation run was cancelled");
+	};
+	let removeDeadlineAbortListener;
+	const abortPromise = new Promise((_, reject) => {
+		const rejectPromise = () => reject(controller.signal.reason ?? /* @__PURE__ */ new Error("Automation run was cancelled"));
+		if (controller.signal.aborted) rejectPromise();
+		else {
+			controller.signal.addEventListener("abort", rejectPromise, { once: true });
+			removeDeadlineAbortListener = () => controller.signal.removeEventListener("abort", rejectPromise);
 		}
-		const timer = setTimeout(() => reject(/* @__PURE__ */ new Error(`Automation timed out after ${timeoutMs}ms`)), timeoutMs);
-		const onAbort = () => {
-			clearTimeout(timer);
-			reject(/* @__PURE__ */ new Error("Automation run was cancelled"));
-		};
-		signal.addEventListener("abort", onAbort, { once: true });
-		promise.then((value) => {
-			clearTimeout(timer);
-			signal.removeEventListener("abort", onAbort);
-			resolve(value);
-		}, (err) => {
-			clearTimeout(timer);
-			signal.removeEventListener("abort", onAbort);
-			reject(err);
-		});
 	});
+	try {
+		const result = await Promise.race([operationPromise, abortPromise]);
+		if (stoppedAs) rejectForAbort();
+		return result;
+	} catch (err) {
+		if (!stoppedAs) throw err;
+		const cancellationConfirmed = await waitForCancellation(operationPromise);
+		throw new AutomationExecutionStoppedError(stoppedAs, cancellationConfirmed, deadlineAtMs, timeoutMs);
+	} finally {
+		clearTimeout(timer);
+		parentSignal.removeEventListener("abort", onParentAbort);
+		removeDeadlineAbortListener?.();
+	}
 }
 var AutomationActionExecutor = class {
 	deps = {};
@@ -40,26 +76,41 @@ var AutomationActionExecutor = class {
 			...deps
 		};
 	}
-	async execute(automation, run, signal) {
-		const timeoutMs = (automation.action.timeoutSeconds ?? automation.reliability?.timeoutSeconds ?? 0) > 0 ? (automation.action.timeoutSeconds ?? automation.reliability?.timeoutSeconds) * 1e3 : 300 * 1e3;
-		return withTimeout(this.executeWithoutTimeout(automation, run, signal), timeoutMs, signal).catch((err) => {
+	async execute(automation, run, signal, hooks = {}) {
+		const configuredTimeoutMs = resolveAutomationTimeoutSeconds(automation.action, automation.reliability) * 1e3;
+		const deadlineAtMs = run.deadlineAtMs ?? Date.now() + configuredTimeoutMs;
+		const timeoutMs = Math.max(1, deadlineAtMs - Date.now());
+		await hooks.onRunPatch?.({ deadlineAtMs });
+		return executeWithDeadline((deadlineSignal) => this.executeWithoutTimeout(automation, run, deadlineSignal, hooks, deadlineAtMs), timeoutMs, signal, deadlineAtMs).catch((err) => {
 			const message = err instanceof Error ? err.message : String(err);
+			if (err instanceof AutomationExecutionStoppedError) return {
+				status: err.status,
+				error: message,
+				deadlineAtMs: err.deadlineAtMs,
+				termination: {
+					reason: err.status === "timeout" ? "deadline_exceeded" : "user_cancelled",
+					component: "automation",
+					cancellationConfirmed: err.cancellationConfirmed
+				}
+			};
 			return {
 				status: message.toLowerCase().includes("cancelled") ? "cancelled" : message.toLowerCase().includes("timed out") ? "timeout" : "failed",
-				error: message
+				error: message,
+				deadlineAtMs
 			};
 		});
 	}
-	async executeWithoutTimeout(automation, run, signal) {
+	async executeWithoutTimeout(automation, run, signal, hooks, deadlineAtMs) {
 		if (signal.aborted) return {
 			status: "cancelled",
 			error: "Automation run was cancelled"
 		};
-		if (automation.action.kind === "workflow") return this.executeWorkflow(automation, automation.action, run);
-		if (automation.action.kind === "browser_recipe") return this.executeBrowserRecipe(automation, automation.action, signal);
-		return this.executeAgent(automation, automation.action);
+		if (automation.action.kind === "workflow") return this.executeWorkflow(automation, automation.action, run, signal, hooks);
+		if (automation.action.kind === "browser_recipe") return this.executeBrowserRecipe(automation, automation.action, signal, hooks);
+		return this.executeAgent(automation, automation.action, run, signal, hooks, deadlineAtMs);
 	}
-	async executeBrowserRecipe(automation, action, signal) {
+	async executeBrowserRecipe(automation, action, signal, hooks) {
+		await hooks.onRunPatch?.({ currentPhase: "action" });
 		const safetyMode = automation.safety?.mode ?? "auto_apply";
 		if (safetyMode !== "auto_apply") return {
 			status: "succeeded",
@@ -80,7 +131,7 @@ var AutomationActionExecutor = class {
 			error: run.error ?? `Browser automation ${action.recipeId} ${run.status}`
 		};
 	}
-	async executeAgent(automation, action) {
+	async executeAgent(automation, action, run, signal, hooks, deadlineAtMs) {
 		const agentService = this.deps.agentService;
 		if (!agentService?.turnDispatcher?.processDirect) return {
 			status: "failed",
@@ -94,6 +145,10 @@ var AutomationActionExecutor = class {
 			peerId: `${automation.id}-${crypto.randomUUID().slice(0, 8)}`
 		});
 		const effectiveWorkingDirectory = action.workingDirectory ?? (automation.projectId ? this.deps.getProjectWorkspaceRoot?.(automation.projectId) : void 0);
+		await hooks.onRunPatch?.({
+			sessionKey,
+			currentPhase: "action"
+		});
 		await agentService.sessionConfig?.applyAutomationWorkingDirectory?.(sessionKey, effectiveWorkingDirectory);
 		if (agentService.sessionConfig?.applyAutomationModelOverride) {
 			if (!await agentService.sessionConfig.applyAutomationModelOverride(sessionKey, action.model) && action.model) log.warn({
@@ -102,7 +157,11 @@ var AutomationActionExecutor = class {
 				model: action.model
 			}, "Automation model override invalid; using agent default");
 		}
-		const response = await agentService.turnDispatcher.processDirect(buildSafetyInstruction(automation, action.instruction), sessionKey);
+		const response = await agentService.turnDispatcher.processDirect(buildSafetyInstruction(automation, action.instruction), sessionKey, void 0, void 0, {
+			signal,
+			runId: run.id,
+			deadlineAtMs
+		});
 		const model = agentService.getModelForSession?.(sessionKey);
 		return {
 			status: "succeeded",
@@ -111,7 +170,7 @@ var AutomationActionExecutor = class {
 			model
 		};
 	}
-	async executeWorkflow(automation, action, run) {
+	async executeWorkflow(automation, action, run, signal, hooks) {
 		const safetyMode = automation.safety?.mode ?? "auto_apply";
 		if (safetyMode === "suggest_only") return {
 			status: "succeeded",
@@ -148,14 +207,65 @@ var AutomationActionExecutor = class {
 			status: "failed",
 			error: result.message
 		};
-		return {
+		await hooks.onRunPatch?.({
+			sessionKey: result.sessionKey,
+			workflowRunId: result.runId,
+			currentPhase: "action"
+		});
+		if (!workflowRunService.readWorkflowRunView) return {
 			status: "succeeded",
 			summary: `Started workflow run ${result.runId}`,
 			sessionKey: result.sessionKey,
 			workflowRunId: result.runId
 		};
+		while (!signal.aborted) {
+			const view = await workflowRunService.readWorkflowRunView(agentId, result.runId);
+			if (!view) return {
+				status: "failed",
+				error: `Workflow run ${result.runId} disappeared`
+			};
+			const workflowStatus = view.run.status;
+			if (workflowStatus === "succeeded") return {
+				status: "succeeded",
+				summary: `Workflow run ${result.runId} completed`,
+				sessionKey: result.sessionKey,
+				workflowRunId: result.runId
+			};
+			if (workflowStatus === "failed" || workflowStatus === "cancelled" || workflowStatus === "timeout") return {
+				status: workflowStatus,
+				error: view.run.error?.message ?? `Workflow run ${result.runId} ${workflowStatus}`,
+				sessionKey: result.sessionKey,
+				workflowRunId: result.runId
+			};
+			await waitForSignalOrDelay(signal, 500);
+		}
+		await workflowRunService.cancelWorkflowRun?.({
+			agentId,
+			runId: result.runId,
+			reason: "Parent automation was cancelled or reached its deadline"
+		});
+		return {
+			status: "cancelled",
+			error: "Parent automation stopped while the workflow was running",
+			sessionKey: result.sessionKey,
+			workflowRunId: result.runId
+		};
 	}
 };
+function waitForSignalOrDelay(signal, delayMs) {
+	return new Promise((resolve) => {
+		const timer = setTimeout(() => {
+			signal.removeEventListener("abort", onAbort);
+			resolve();
+		}, delayMs);
+		const onAbort = () => {
+			clearTimeout(timer);
+			resolve();
+		};
+		if (signal.aborted) onAbort();
+		else signal.addEventListener("abort", onAbort, { once: true });
+	});
+}
 function buildSafetyInstruction(automation, instruction) {
 	const mode = automation.safety?.mode ?? "auto_apply";
 	if (mode === "suggest_only") return [

package/dist/src/automations/service/automation-service.d.ts

@@ -15,6 +15,8 @@ export declare class AutomationService {
     private deps;
     private maxConcurrentRuns;
     private readonly activeRuns;
+    private readonly heartbeatTimers;
+    private readonly leaseOwner;
     constructor(signals?: ProactiveSignalPublisher);
     setDeps(deps: AutomationDeps): void;
     initialize(options?: {
@@ -54,6 +56,8 @@ export declare class AutomationService {
     private onTimer;
     private startRun;
     private executeRun;
+    private recoverInterruptedRuns;
+    private waitForRetry;
     private appendRunEvent;
     private finishAutomationRun;
     private postAfterRunWebhook;

package/dist/src/automations/service/automation-service.js

@@ -5,7 +5,7 @@ import { computeNextAutomationRunAtMs, timerDelayUntil } from "../domain/schedul
 import { AutomationSchema, CreateAutomationSchema, UpdateAutomationSchema } from "../domain/validation.js";
 import { AutomationActionExecutor } from "./action-executor.js";
 import { deleteAutomation, getAutomation, listAutomations, saveAutomation, saveAutomations } from "../storage/automation-repository.js";
-import { appendAutomationRunEvent, getAutomationRun, listAutomationRunEvents, listAutomationRuns, listAutomationRunsForProductEvent, saveAutomationRun } from "../storage/automation-run-repository.js";
+import { appendAutomationRunEvent, getAutomationRun, listAutomationRunEvents, listAutomationRuns, listAutomationRunsForProductEvent, saveAutomationRun, touchAutomationRunLease } from "../storage/automation-run-repository.js";
 import "../storage/index.js";
 import { randomUUID } from "node:crypto";
 //#region src/automations/service/automation-service.ts
@@ -13,6 +13,9 @@ init_logger();
 const log = createLogger("AutomationService");
 const DEFAULT_MAX_CONCURRENT_RUNS = 5;
 const MISSED_RUN_STAGGER_MS = 5e3;
+const RUN_HEARTBEAT_INTERVAL_MS = 1e4;
+const RUN_LEASE_DURATION_MS = 3e4;
+const RETRY_BASE_DELAY_MS = 1e3;
 var AutomationAlreadyRunningError = class extends Error {
 	constructor(automationId, runningRunId) {
 		super(`Automation is already running: ${automationId}`);
@@ -29,6 +32,8 @@ var AutomationService = class {
 	deps = {};
 	maxConcurrentRuns = DEFAULT_MAX_CONCURRENT_RUNS;
 	activeRuns = /* @__PURE__ */ new Map();
+	heartbeatTimers = /* @__PURE__ */ new Map();
+	leaseOwner = `automation-service:${process.pid}:${randomUUID()}`;
 	constructor(signals) {
 		this.signals = signals;
 	}
@@ -42,6 +47,7 @@ var AutomationService = class {
 	async initialize(options) {
 		this.maxConcurrentRuns = Math.max(1, Math.floor(options?.maxConcurrentRuns ?? DEFAULT_MAX_CONCURRENT_RUNS));
 		this.stopped = false;
+		await this.recoverInterruptedRuns();
 		await this.recomputeNextRuns({ catchUp: true });
 		this.armTimer();
 		log.info("Automation service initialized");
@@ -53,6 +59,8 @@ var AutomationService = class {
 			this.timer = null;
 		}
 		for (const controller of this.activeRuns.values()) controller.abort();
+		for (const timer of this.heartbeatTimers.values()) clearInterval(timer);
+		this.heartbeatTimers.clear();
 		log.info("Automation service stopped");
 	}
 	async create(input) {
@@ -160,7 +168,19 @@ var AutomationService = class {
 	async cancelRun(runId) {
 		const controller = this.activeRuns.get(runId);
 		if (!controller) return false;
-		controller.abort();
+		const now = Date.now();
+		const current = getAutomationRun(runId);
+		if (current && current.status === "running") {
+			const cancelling = {
+				...current,
+				status: "cancelling",
+				currentPhase: "cancelling",
+				cancelRequestedAtMs: now
+			};
+			saveAutomationRun(cancelling);
+			this.appendRunEvent(cancelling, "run.cancel_requested", "Automation cancellation requested", { reason: "user_cancelled" });
+		}
+		controller.abort(/* @__PURE__ */ new Error("Automation run was cancelled by the user"));
 		return true;
 	}
 	async getMetrics() {
@@ -187,7 +207,7 @@ var AutomationService = class {
 				...automation,
 				state: { ...automation.state }
 			};
-			if (next.state.runningRunId) {
+			if (next.state.runningRunId && !this.activeRuns.has(next.state.runningRunId)) {
 				next.state.lastRunStatus = "failed";
 				next.state.lastError = "Interrupted by gateway restart";
 				next.state.consecutiveFailures = (next.state.consecutiveFailures ?? 0) + 1;
@@ -237,8 +257,10 @@ var AutomationService = class {
 			triggerSnapshot: automation.trigger,
 			actionSnapshot: automation.action,
 			manual: opts.manual,
-			createdAtMs: now
+			createdAtMs: now,
+			attemptNumber: 1
 		};
+		run.rootRunId = run.id;
 		saveAutomationRun(run);
 		this.appendRunEvent(run, "run.queued", opts.event ? `Event ${opts.event.type} queued automation` : opts.manual ? "Manual run queued" : "Scheduled run queued", {
 			trigger: automation.trigger,
@@ -270,21 +292,69 @@ var AutomationService = class {
 		let run = {
 			...initialRun,
 			status: "running",
-			startedAtMs
+			startedAtMs,
+			currentPhase: "action",
+			heartbeatAtMs: startedAtMs,
+			leaseOwner: this.leaseOwner,
+			leaseExpiresAtMs: startedAtMs + RUN_LEASE_DURATION_MS
 		};
 		saveAutomationRun(run);
 		this.appendRunEvent(run, "run.started", "Automation run started");
+		const heartbeat = setInterval(() => {
+			const now = Date.now();
+			touchAutomationRunLease(run.id, this.leaseOwner, now, now + RUN_LEASE_DURATION_MS);
+		}, RUN_HEARTBEAT_INTERVAL_MS);
+		heartbeat.unref?.();
+		this.heartbeatTimers.set(run.id, heartbeat);
 		let status = "failed";
 		let error;
 		let activePhase = "action";
 		try {
-			this.appendRunEvent(run, "action.started", `Running ${automation.action.kind} action`, {
-				actionKind: automation.action.kind,
-				safety: automation.safety ?? { mode: "auto_apply" }
-			});
-			const outcome = await this.executor.execute(automation, run, controller.signal);
+			const maxAttempts = Math.max(1, (automation.reliability?.retryCount ?? 0) + 1);
+			let outcome;
+			for (let attempt = 1;; attempt += 1) {
+				run = {
+					...run,
+					attemptNumber: attempt
+				};
+				saveAutomationRun(run);
+				this.appendRunEvent(run, "action.started", `Running ${automation.action.kind} action`, {
+					actionKind: automation.action.kind,
+					safety: automation.safety ?? { mode: "auto_apply" },
+					attempt,
+					maxAttempts
+				});
+				outcome = await this.executor.execute(automation, run, controller.signal, { onRunPatch: (patch) => {
+					const previousDeadline = run.deadlineAtMs;
+					run = {
+						...run,
+						...patch
+					};
+					saveAutomationRun(run);
+					if (patch.deadlineAtMs !== void 0 && patch.deadlineAtMs !== previousDeadline) this.appendRunEvent(run, "run.deadline_resolved", "Automation deadline resolved", {
+						deadlineAtMs: patch.deadlineAtMs,
+						timeoutMs: patch.deadlineAtMs - startedAtMs
+					});
+				} });
+				if (!(outcome.status === "failed") || attempt >= maxAttempts || controller.signal.aborted) break;
+				const delayMs = Math.min(RETRY_BASE_DELAY_MS * 2 ** (attempt - 1), 3e4);
+				this.appendRunEvent(run, "action.retry_scheduled", "Automation action retry scheduled", {
+					attempt,
+					nextAttempt: attempt + 1,
+					maxAttempts,
+					delayMs,
+					status: outcome.status,
+					error: outcome.error
+				});
+				await this.waitForRetry(delayMs, controller.signal);
+			}
 			status = outcome.status;
 			error = outcome.error;
+			const persistedRun = getAutomationRun(run.id);
+			if (persistedRun) run = {
+				...run,
+				...persistedRun
+			};
 			this.appendRunEvent(run, status === "succeeded" ? "action.completed" : "action.failed", status === "succeeded" ? `${automation.action.kind} action completed` : `${automation.action.kind} action ${status}`, {
 				summary: outcome.summary,
 				error: outcome.error,
@@ -299,23 +369,52 @@ var AutomationService = class {
 				error,
 				sessionKey: outcome.sessionKey,
 				workflowRunId: outcome.workflowRunId,
-				model: outcome.model
+				model: outcome.model,
+				deadlineAtMs: outcome.deadlineAtMs ?? run.deadlineAtMs,
+				termination: outcome.termination ?? (status === "succeeded" ? {
+					reason: "completed",
+					cancellationConfirmed: true
+				} : status === "failed" ? {
+					reason: "failed",
+					cancellationConfirmed: true
+				} : run.termination)
 			};
-			if ((automation.safety?.mode ?? "auto_apply") !== "auto_apply" && automation.afterRun?.kind === "webhook") this.appendRunEvent(run, "after_run.completed", "After-run webhook skipped by safety mode", { safety: automation.safety ?? { mode: "auto_apply" } });
+			if (outcome.termination) {
+				const now = Date.now();
+				run = {
+					...run,
+					currentPhase: "cancelling",
+					cancelRequestedAtMs: run.cancelRequestedAtMs ?? now,
+					cancelConfirmedAtMs: outcome.termination.cancellationConfirmed ? now : void 0
+				};
+				this.appendRunEvent(run, outcome.termination.cancellationConfirmed ? "run.cancel_confirmed" : "run.cancellation_unconfirmed", outcome.termination.cancellationConfirmed ? "Automation cancellation confirmed" : "Automation cancellation was not confirmed within the cleanup grace period", { termination: outcome.termination });
+			}
+			if (status === "timeout" || status === "cancelled") {} else if ((automation.safety?.mode ?? "auto_apply") !== "auto_apply" && automation.afterRun?.kind === "webhook") this.appendRunEvent(run, "after_run.completed", "After-run webhook skipped by safety mode", { safety: automation.safety ?? { mode: "auto_apply" } });
 			else if (automation.afterRun?.kind === "webhook") {
 				activePhase = "after_run";
+				run = {
+					...run,
+					currentPhase: "after_run"
+				};
+				saveAutomationRun(run);
 				this.appendRunEvent(run, "after_run.started", "Calling after-run webhook", { url: automation.afterRun.url });
-				await this.postAfterRunWebhook(automation.afterRun.url, run);
+				await this.postAfterRunWebhook(automation.afterRun.url, run, controller.signal);
 				this.appendRunEvent(run, "after_run.completed", "After-run webhook completed");
 			}
 		} catch (err) {
 			error = err instanceof Error ? err.message : String(err);
-			status = controller.signal.aborted ? "cancelled" : "failed";
+			const deadlineExceeded = run.deadlineAtMs !== void 0 && Date.now() >= run.deadlineAtMs;
+			status = controller.signal.aborted ? "cancelled" : deadlineExceeded ? "timeout" : "failed";
 			this.appendRunEvent(run, activePhase === "after_run" ? "after_run.failed" : "action.failed", `Automation run ${status}`, { error });
 			run = {
 				...run,
 				status,
-				error
+				error,
+				termination: {
+					reason: status === "cancelled" ? "user_cancelled" : status === "timeout" ? "deadline_exceeded" : "failed",
+					component: activePhase === "after_run" ? "automation" : run.termination?.component,
+					cancellationConfirmed: status !== "cancelled" || controller.signal.aborted
+				}
 			};
 		} finally {
 			const endedAtMs = Date.now();
@@ -323,9 +422,19 @@ var AutomationService = class {
 				...run,
 				status,
 				endedAtMs,
-				durationMs: endedAtMs - startedAtMs
+				durationMs: endedAtMs - startedAtMs,
+				currentPhase: "completed"
 			};
 			this.activeRuns.delete(initialRun.id);
+			const heartbeatTimer = this.heartbeatTimers.get(initialRun.id);
+			if (heartbeatTimer) clearInterval(heartbeatTimer);
+			this.heartbeatTimers.delete(initialRun.id);
+			run = {
+				...run,
+				heartbeatAtMs: endedAtMs,
+				leaseOwner: void 0,
+				leaseExpiresAtMs: void 0
+			};
 			runSqliteWriteTransaction(() => {
 				saveAutomationRun(run);
 				this.appendRunEvent(run, "run.completed", `Automation run ${status}`, {
@@ -370,6 +479,73 @@ var AutomationService = class {
 			}
 		}
 	}
+	async recoverInterruptedRuns() {
+		const recoverable = listAutomationRuns({ limit: 500 }).filter((run) => run.status === "queued" || run.status === "running" || run.status === "cancelling");
+		for (const run of recoverable) {
+			const automation = getAutomation(run.automationId);
+			if (run.status === "queued" && automation) {
+				const claimed = {
+					...automation,
+					state: {
+						...automation.state,
+						runningRunId: run.id
+					}
+				};
+				saveAutomation(claimed);
+				this.appendRunEvent(run, "run.recovered", "Queued automation run recovered after restart", { previousLeaseOwner: run.leaseOwner });
+				this.executeRun(claimed, run).catch((err) => {
+					log.error({
+						err,
+						runId: run.id
+					}, "Recovered automation run failed");
+				});
+				continue;
+			}
+			const now = Date.now();
+			const status = run.status === "cancelling" ? "cancelled" : "failed";
+			const error = run.status === "cancelling" ? "Cancellation interrupted by gateway restart" : "Automation execution interrupted by gateway restart";
+			const recovered = {
+				...run,
+				status,
+				error,
+				endedAtMs: now,
+				durationMs: run.startedAtMs ? now - run.startedAtMs : void 0,
+				currentPhase: "completed",
+				heartbeatAtMs: now,
+				leaseOwner: void 0,
+				leaseExpiresAtMs: void 0,
+				termination: {
+					reason: status === "cancelled" ? "user_cancelled" : "failed",
+					component: "automation",
+					cancellationConfirmed: false
+				}
+			};
+			saveAutomationRun(recovered);
+			this.appendRunEvent(recovered, "run.recovered", error, {
+				previousLeaseOwner: run.leaseOwner,
+				previousLeaseExpiresAtMs: run.leaseExpiresAtMs
+			});
+			this.appendRunEvent(recovered, "run.completed", `Automation run ${status}`, {
+				status,
+				error
+			});
+			if (automation) this.finishAutomationRun(automation.id, status, error, now);
+		}
+	}
+	waitForRetry(delayMs, signal) {
+		return new Promise((resolve, reject) => {
+			const timer = setTimeout(() => {
+				signal.removeEventListener("abort", onAbort);
+				resolve();
+			}, delayMs);
+			const onAbort = () => {
+				clearTimeout(timer);
+				reject(signal.reason ?? /* @__PURE__ */ new Error("Automation run was cancelled"));
+			};
+			if (signal.aborted) onAbort();
+			else signal.addEventListener("abort", onAbort, { once: true });
+		});
+	}
 	appendRunEvent(run, type, message, data) {
 		appendAutomationRunEvent({
 			id: randomUUID(),
@@ -404,11 +580,14 @@ var AutomationService = class {
 		saveAutomation(next);
 		this.armTimer();
 	}
-	async postAfterRunWebhook(url, run) {
+	async postAfterRunWebhook(url, run, signal) {
+		const remainingMs = run.deadlineAtMs === void 0 ? 3e4 : Math.max(1, run.deadlineAtMs - Date.now());
+		const requestSignal = AbortSignal.any([signal, AbortSignal.timeout(remainingMs)]);
 		const response = await fetch(url, {
 			method: "POST",
 			headers: { "content-type": "application/json" },
-			body: JSON.stringify({ run })
+			body: JSON.stringify({ run }),
+			signal: requestSignal
 		});
 		if (!response.ok) {
 			const text = await response.text().catch(() => "");

package/dist/src/automations/storage/automation-run-repository.d.ts

@@ -1,6 +1,7 @@
 import type { AutomationProductEventRun, AutomationRun, AutomationRunEvent } from '../domain/types.js';
 export declare function appendAutomationRunEvent(event: AutomationRunEvent): void;
 export declare function saveAutomationRun(run: AutomationRun): void;
+export declare function touchAutomationRunLease(runId: string, leaseOwner: string, heartbeatAtMs: number, leaseExpiresAtMs: number): void;
 export declare function listAutomationRunEvents(runId: string): AutomationRunEvent[];
 export declare function getAutomationRun(runId: string): AutomationRun | null;
 export declare function listAutomationRuns(options?: {