@rebasepro/server 0.21.2-canary.g1ea48be → 0.23.0

package/dist/{cron-scheduler-COPQxlEq.js → cron-scheduler-B0pLfAix.js}

@@ -3,8 +3,10 @@ import __rebaseProcess from "process";
 globalThis.process ??= __rebaseProcess;
 __rebaseCreateRequire(import.meta.url);
 import { n as __exportAll } from "./rolldown-runtime-dW7B1o5h.js";
-import "./src-Br6ARbs6.js";
+import "./src-vkcwKXbT.js";
 import { a as redactSensitiveText, r as logger } from "./logger-DO2PZc4i.js";
+import { randomUUID } from "node:crypto";
+import { hostname } from "node:os";
 //#region ../types/src/types/env_boolean.ts
 var YES = /* @__PURE__ */ new Set([
 	"true",
@@ -148,6 +150,7 @@ function buildScaleToZeroWarning(jobs, env = process.env) {
 var cron_scheduler_exports = /* @__PURE__ */ __exportAll({
 	CronScheduler: () => CronScheduler,
 	findMostRecentSlot: () => findMostRecentSlot,
+	isAlreadyExecutingSkip: () => isAlreadyExecutingSkip,
 	isValidTimeZone: () => isValidTimeZone,
 	parseCronExpression: () => parseCronExpression,
 	validateCronExpression: () => validateCronExpression
@@ -326,7 +329,6 @@ function matchesCronFields(candidate, fields, zone) {
 	};
 	return fields.months.includes(wall.month) && fields.doms.includes(wall.dom) && fields.dows.includes(wall.dow) && fields.hours.includes(wall.hour) && fields.minutes.includes(wall.minute);
 }
-/** ~1 year in minutes — the walk bound for both search directions. */
 /**
 * How far forward to look for the next matching slot.
 *
@@ -335,6 +337,7 @@ function matchesCronFields(candidate, fields, zone) {
 * search never found it.
 */
 var MAX_SLOT_SEARCH_MINUTES = 2105280;
+var MINUTE_MS = 6e4;
 /**
 * Calculate the next Date after `after` that matches the cron expression.
 * Throws on invalid expressions.
@@ -342,11 +345,11 @@ var MAX_SLOT_SEARCH_MINUTES = 2105280;
 function parseCronExpression(expression, after, timezone) {
 	const fields = parseCronFields(expression);
 	const candidate = new Date(after);
-	candidate.setSeconds(0, 0);
-	candidate.setMinutes(candidate.getMinutes() + 1);
+	candidate.setUTCSeconds(0, 0);
+	candidate.setTime(candidate.getTime() + MINUTE_MS);
 	for (let i = 0; i < MAX_SLOT_SEARCH_MINUTES; i++) {
 		if (matchesCronFields(candidate, fields, timezone)) return candidate;
-		candidate.setMinutes(candidate.getMinutes() + 1);
+		candidate.setTime(candidate.getTime() + MINUTE_MS);
 	}
 	throw new Error(`Cron expression "${expression}" has no matching time within ${Math.round(MAX_SLOT_SEARCH_MINUTES / 525960)} years of ${after.toISOString()}. Check the day-of-month and month fields — a date such as 31 February never occurs.`);
 }
@@ -369,10 +372,10 @@ function parseCronExpression(expression, after, timezone) {
 function findMostRecentSlot(expression, from, to, timezone) {
 	const fields = parseCronFields(expression);
 	const candidate = new Date(to);
-	candidate.setSeconds(0, 0);
+	candidate.setUTCSeconds(0, 0);
 	for (let i = 0; i < MAX_SLOT_SEARCH_MINUTES && candidate.getTime() >= from.getTime(); i++) {
 		if (matchesCronFields(candidate, fields, timezone)) return candidate;
-		candidate.setMinutes(candidate.getMinutes() - 1);
+		candidate.setTime(candidate.getTime() - MINUTE_MS);
 	}
 }
 var MAX_LOGS_PER_JOB = 50;
@@ -387,6 +390,66 @@ var MIN_SCHEDULE_INTERVAL_MS = 5e3;
 * further out than this must be reached in hops rather than one timer.
 */
 var MAX_TIMER_DELAY_MS = 2147483647;
+/**
+* `setTimeout` for a delay that may exceed {@link MAX_TIMER_DELAY_MS}: armed in
+* hops no longer than the ceiling, re-measured against the deadline on each
+* wake. Returns the function that cancels it.
+*/
+function setLongTimeout(callback, delayMs) {
+	const deadline = Date.now() + delayMs;
+	let handle;
+	const arm = () => {
+		handle = setTimeout(() => {
+			if (Date.now() < deadline) arm();
+			else callback();
+		}, Math.min(Math.max(deadline - Date.now(), 0), MAX_TIMER_DELAY_MS));
+	};
+	arm();
+	return () => clearTimeout(handle);
+}
+/** A run's timeout when its definition names none. */
+var DEFAULT_TIMEOUT_SECONDS = 300;
+/**
+* How long a run lease outlasts the run's own timeout.
+*
+* The lease is timed on the database's clock from the moment it is taken, the
+* timeout on this process's from a moment later; without a margin a lease
+* could lapse a beat before its run is aborted and released, and another
+* process start the job alongside it.
+*/
+var LEASE_GRACE_SECONDS = 30;
+/**
+* The lease a run with no timeout (`timeoutSeconds: Infinity`) holds.
+*
+* A lease is released when its run ends, so this only matters in two cases.
+* The holder crashed: the job is blocked — every trigger answered "already
+* executing", every slot skipped — until the lease lapses, so it must lapse
+* within an hour rather than never. The run is still going after an hour:
+* from then on another process may start the job beside it. A job that
+* legitimately runs for hours should say so with a finite `timeoutSeconds`,
+* which its lease then follows.
+*/
+var MAX_UNBOUNDED_LEASE_SECONDS = 3600;
+/**
+* Whether a log entry records a run that did not happen because one was
+* already executing — on this process, or on another that holds the job's run
+* lease. The shape is the documented marker of a skip, the one the Studio
+* panel reads.
+*/
+function isAlreadyExecutingSkip(entry) {
+	const result = entry.result;
+	return typeof result === "object" && result !== null && "skipped" in result && result.skipped === true && "reason" in result && result.reason === "already_executing";
+}
+/**
+* Why `timeoutSeconds` cannot be used, or `undefined` when it can. Infinity is
+* a timeout that never fires; zero, a negative number or NaN would fail every
+* run the moment it started.
+*/
+function invalidTimeoutReason(timeoutSeconds) {
+	if (timeoutSeconds === void 0 || timeoutSeconds === Infinity) return void 0;
+	if (typeof timeoutSeconds === "number" && Number.isFinite(timeoutSeconds) && timeoutSeconds > 0) return void 0;
+	return `timeoutSeconds must be a positive number of seconds, or Infinity for none — got ${String(timeoutSeconds)}`;
+}
 var CronScheduler = class {
 	jobs = /* @__PURE__ */ new Map();
 	rejected = /* @__PURE__ */ new Map();
@@ -394,6 +457,19 @@ var CronScheduler = class {
 	store;
 	client;
 	/**
+	* Work `stop()` has to wait for: every run executing right now, and every
+	* run's log write still on its way to the store.
+	*/
+	inFlight = /* @__PURE__ */ new Set();
+	/** The controller that ends each executing run — what `stop()` aborts. */
+	runControllers = /* @__PURE__ */ new Set();
+	/**
+	* This process, as a run lease names it: what "already executing on …"
+	* tells an operator. Each run adds its own suffix, so a release can only
+	* ever clear the lease that run took.
+	*/
+	instanceLabel = `${hostname()}:${process.pid}`;
+	/**
 	* Set the server singleton to make it available to cron job handlers.
 	*
 	* `RebaseServerClient`, not `RebaseClient`: the object `init.ts` passes is
@@ -445,13 +521,24 @@ var CronScheduler = class {
 				});
 				continue;
 			}
+			const timeoutProblem = invalidTimeoutReason(loaded.definition.timeoutSeconds);
+			if (timeoutProblem) {
+				logger.error(`[cron] Rejecting job "${loaded.id}": ${timeoutProblem}.`);
+				this.rejected.set(loaded.id, {
+					id: loaded.id,
+					name: loaded.definition.name ?? loaded.id,
+					schedule: loaded.definition.schedule,
+					reason: timeoutProblem
+				});
+				continue;
+			}
 			this.rejected.delete(loaded.id);
 			if (this.jobs.get(loaded.id)) {
 				logger.warn(`[cron] Duplicate cron job id: "${loaded.id}". Overwriting.`);
 				this.stopJob(loaded.id);
 			}
 			const enabled = loaded.definition.enabled !== false;
-			this.jobs.set(loaded.id, {
+			const job = {
 				id: loaded.id,
 				definition: loaded.definition,
 				enabled,
@@ -460,8 +547,9 @@ var CronScheduler = class {
 				totalFailures: 0,
 				logs: [],
 				executing: false
-			});
-			if (this.started && enabled) this.scheduleNext(loaded.id);
+			};
+			this.jobs.set(loaded.id, job);
+			if (this.started && this.shouldArm(job)) this.scheduleNext(loaded.id);
 		}
 	}
 	/**
@@ -482,21 +570,42 @@ var CronScheduler = class {
 		}).catch((err) => {
 			logger.warn("[cron] Failed to seed job stats from database", { error: err });
 		});
-		for (const [id, job] of this.jobs) if (job.enabled) this.scheduleNext(id);
+		for (const [id, job] of this.jobs) if (this.shouldArm(job)) this.scheduleNext(id);
 		if (!this.store) logger.warn("[cron] No cron store attached — runs are uncoordinated; with multiple app instances every instance will execute every job");
 		this.warnIfScaleToZero();
 		this.catchUpMissedSlots();
 		logger.info(`⏰ Cron scheduler started with ${this.jobs.size} job(s)`);
 	}
 	/**
-	* Stop the scheduler and clear all timers.
+	* Stop the scheduler: clear every timer, then wait for the runs executing
+	* right now — for at most `timeoutMs` when given.
+	*
+	* A run still going when the budget runs out has its `ctx.signal` aborted
+	* and ends there, recorded as a failure that says why. Its slot stays
+	* claimed, so no other instance re-runs it; a handler that must not be cut
+	* short should watch its signal and leave its work resumable.
 	*
-	* Currently-executing handlers run to completion (they are async),
-	* but no further scheduling occurs after stop.
+	* Nothing is scheduled after `stop()`, whether or not the returned promise
+	* is awaited: the timers are cleared before it first yields.
 	*/
-	stop() {
+	async stop(timeoutMs) {
 		this.started = false;
 		for (const [id] of this.jobs) this.stopJob(id);
+		if (this.inFlight.size === 0) return;
+		const settled = Promise.allSettled([...this.inFlight]).then(() => true);
+		if (timeoutMs === void 0) {
+			await settled;
+			return;
+		}
+		let budget;
+		const finished = await Promise.race([settled, new Promise((resolve) => {
+			budget = setTimeout(() => resolve(false), timeoutMs);
+			if (budget && typeof budget === "object" && "unref" in budget) budget.unref();
+		})]);
+		clearTimeout(budget);
+		if (finished || this.runControllers.size === 0) return;
+		logger.warn(`[cron] ${this.runControllers.size} run(s) still executing after ${timeoutMs}ms — aborting them`);
+		for (const controller of this.runControllers) controller.abort(/* @__PURE__ */ new Error("The server is shutting down; the run was stopped before it finished"));
 	}
 	/**
 	* List all registered jobs with their current status.
@@ -522,6 +631,31 @@ var CronScheduler = class {
 		return job ? this.toStatus(job) : void 0;
 	}
 	/**
+	* {@link listJobs}, as the fleet sees it rather than this process alone.
+	*
+	* Reads the store: every job's enabled state — a pause another replica
+	* made — and whether another process holds its run lease. On a process
+	* whose scheduler is not started (the `api` role) it also reads each job's
+	* run count and last run from `cron_logs`, since that process runs nothing
+	* and its own counters would say 0 runs for a job the worker has run a
+	* thousand times.
+	*
+	* A store that cannot answer leaves this process's own view, with a
+	* warning: a listing is not worth failing.
+	*/
+	async fetchJobs() {
+		const jobs = [...this.jobs.values()];
+		const views = await this.readFleet(jobs);
+		return jobs.map((job) => this.toStatus(job, views.get(job.id)));
+	}
+	/** {@link getJob}, as {@link fetchJobs} reads it. */
+	async fetchJob(id) {
+		const job = this.jobs.get(id);
+		if (!job) return void 0;
+		const views = await this.readFleet([job]);
+		return this.toStatus(job, views.get(id));
+	}
+	/**
 	* Get log entries for a job.
 	*/
 	getJobLogs(id, limit) {
@@ -542,27 +676,49 @@ var CronScheduler = class {
 		return this.getJobLogs(id, limit);
 	}
 	/**
-	* Enable or disable a job at runtime.
+	* Enable or disable a job in this process only. `null` returns it to what
+	* its code declares.
+	*
+	* {@link persistJobEnabled} is what the admin API calls: this alone is
+	* forgotten on restart and reaches no other replica.
 	*/
 	setJobEnabled(id, enabled) {
 		const job = this.jobs.get(id);
 		if (!job) return void 0;
-		job.enabled = enabled;
-		if (enabled && this.started) {
-			job.state = "idle";
-			this.scheduleNext(id);
-		} else if (!enabled) {
-			this.stopJob(id);
-			job.state = "disabled";
-		}
+		const effective = enabled ?? this.codeEnabled(job);
+		this.applyEnabled(job, effective);
+		if (effective && this.started) this.scheduleNext(id);
+		else if (!this.shouldArm(job)) this.stopJob(id);
 		return this.toStatus(job);
 	}
 	/**
+	* Enable or disable a job for every process that runs it, and keep it that
+	* way across restarts and redeploys: `true` or `false` overrides the code's
+	* `enabled`, `null` hands the decision back to it.
+	*
+	* The override is written to the store first, and only then applied here,
+	* so a write that fails throws and changes nothing — a pause that only the
+	* replica serving the request would honour is the failure this exists to
+	* end. Every scheduler reads the override when a slot comes due, before it
+	* claims the slot.
+	*
+	* Without a store that keeps the state (a driver with no SQL, persistence
+	* switched off, a custom store) this is {@link setJobEnabled}: local, and
+	* gone on restart.
+	*/
+	async persistJobEnabled(id, enabled, updatedBy) {
+		if (!this.jobs.has(id)) return void 0;
+		if (this.store?.saveEnabledOverride) await this.store.saveEnabledOverride(id, enabled, updatedBy);
+		return this.setJobEnabled(id, enabled);
+	}
+	/**
 	* Manually trigger a job execution immediately.
 	*
 	* Returns `undefined` if the job doesn't exist.
-	* If the job is currently executing, returns the log entry with
-	* a `skipped: true` result rather than running concurrently.
+	* If the job is currently executing — here, or on another process that
+	* holds its run lease — returns the log entry with a `skipped: true`
+	* result rather than running concurrently; {@link isAlreadyExecutingSkip}
+	* recognises it.
 	*/
 	async triggerJob(id) {
 		const job = this.jobs.get(id);
@@ -587,8 +743,9 @@ var CronScheduler = class {
 	* `success: true` is deliberate: nothing failed. The `result.skipped` flag
 	* and the reason are what distinguishes it, and the Studio panel reads them.
 	*/
-	recordSkip(job, reason, manual) {
+	recordSkip(job, reason, manual, runningOn) {
 		const now = (/* @__PURE__ */ new Date()).toISOString();
+		const why = reason === "already_executing" ? "the previous run has not finished" : reason;
 		const logEntry = {
 			jobId: job.id,
 			startedAt: now,
@@ -599,16 +756,143 @@ var CronScheduler = class {
 				skipped: true,
 				reason
 			},
-			logs: [`Skipped: ${reason === "already_executing" ? "the previous run has not finished" : reason}`],
+			logs: [`Skipped: ${why}${runningOn ? ` — it is running on ${runningOn}` : ""}`],
 			manual
 		};
 		job.logs.push(logEntry);
 		if (job.logs.length > MAX_LOGS_PER_JOB) job.logs.shift();
-		this.store?.insertLog(logEntry).catch((persistErr) => {
+		if (this.store) this.track(this.store.insertLog(logEntry).catch((persistErr) => {
 			logger.error(`[cron] Failed to persist skip for "${job.id}"`, { error: persistErr });
-		});
+		}));
 		return logEntry;
 	}
+	/** What the job's own definition says, before any override. */
+	codeEnabled(job) {
+		return job.definition.enabled !== false;
+	}
+	/**
+	* Whether the enabled state lives in the store, where every process reads
+	* it, rather than in this process alone.
+	*/
+	sharesEnabledState() {
+		return Boolean(this.store?.fetchJobStates && this.store.saveEnabledOverride);
+	}
+	/**
+	* Whether a job gets a timer here. A job paused in this process's memory
+	* needs none — nothing but this process can resume it. A job whose state
+	* the store keeps does, paused or not: another replica may resume it, and
+	* reading the state when its slot comes due is how this one finds out.
+	*/
+	shouldArm(job) {
+		return job.enabled || this.sharesEnabledState();
+	}
+	/** Set a job's enabled flag, and its state with it unless a run is in progress. */
+	applyEnabled(job, enabled) {
+		job.enabled = enabled;
+		if (job.executing) return;
+		if (!enabled) job.state = "disabled";
+		else if (job.state === "disabled") job.state = "idle";
+	}
+	/**
+	* Whether a job may run now: the store's override when it keeps one, else
+	* the code's `enabled`.
+	*
+	* One query per fire, before the slot is claimed — a paused job does not
+	* spend its slot. A store that cannot answer falls back to the code's
+	* `enabled`, with a warning: a scheduled run fails open, as it does when
+	* the claims table cannot answer, because a broken state table must not
+	* silently stop every job.
+	*/
+	async isEnabledNow(job) {
+		if (!this.store?.fetchJobStates || !this.sharesEnabledState()) return job.enabled;
+		try {
+			const enabled = (await this.store.fetchJobStates([job.id])).get(job.id)?.enabled ?? this.codeEnabled(job);
+			this.applyEnabled(job, enabled);
+			return enabled;
+		} catch (err) {
+			const enabled = this.codeEnabled(job);
+			logger.warn(`[cron] Could not read the enabled state of "${job.id}" — falling back to the code's enabled (${enabled}); a pause made from the admin API is not being honoured`, { error: err });
+			return enabled;
+		}
+	}
+	/**
+	* How long a run's lease lasts: the run's timeout and a grace, so that a
+	* crashed holder frees the job once its run would have been aborted
+	* anyway. No timeout gets {@link MAX_UNBOUNDED_LEASE_SECONDS}.
+	*/
+	leaseSeconds(job) {
+		const timeout = job.definition.timeoutSeconds ?? DEFAULT_TIMEOUT_SECONDS;
+		return Number.isFinite(timeout) ? timeout + LEASE_GRACE_SECONDS : MAX_UNBOUNDED_LEASE_SECONDS;
+	}
+	/**
+	* Take the job's run lease, the cross-process half of "already
+	* executing". A store without leases, or one that cannot answer, lets the
+	* run go ahead unleased — with a warning in the second case — the same way
+	* a claim that cannot answer does.
+	*/
+	async acquireLease(job) {
+		if (!this.store?.tryAcquireRunLease) return { acquired: true };
+		const token = `${this.instanceLabel}#${randomUUID().slice(0, 8)}`;
+		try {
+			const lease = await this.store.tryAcquireRunLease(job.id, token, this.leaseSeconds(job));
+			return lease.acquired ? {
+				acquired: true,
+				token
+			} : lease;
+		} catch (err) {
+			logger.warn(`[cron] Could not take the run lease for "${job.id}" — running without it, so another process may run it at the same time`, { error: err });
+			return { acquired: true };
+		}
+	}
+	/** Release a lease this run took. A release that fails leaves it to lapse. */
+	async releaseLease(job, token) {
+		try {
+			await this.store?.releaseRunLease?.(job.id, token);
+		} catch (err) {
+			logger.warn(`[cron] Could not release the run lease for "${job.id}" — it lapses on its own within ${this.leaseSeconds(job)}s`, { error: err });
+		}
+	}
+	/**
+	* What the store says about `jobs` beyond this process — see
+	* {@link fetchJobs}. Applies the enabled state it reads, and returns the
+	* rest as a view for {@link toStatus}. Never throws.
+	*/
+	async readFleet(jobs) {
+		const views = /* @__PURE__ */ new Map();
+		const viewOf = (id) => {
+			let view = views.get(id);
+			if (!view) {
+				view = {};
+				views.set(id, view);
+			}
+			return view;
+		};
+		const store = this.store;
+		const ids = jobs.map((job) => job.id);
+		if (store?.fetchJobStates && this.sharesEnabledState() && ids.length > 0) try {
+			const states = await store.fetchJobStates(ids);
+			for (const job of jobs) {
+				const state = states.get(job.id);
+				this.applyEnabled(job, state?.enabled ?? this.codeEnabled(job));
+				if (state?.runningBy && !job.executing) viewOf(job.id).runningElsewhere = true;
+			}
+		} catch (err) {
+			logger.warn("[cron] Could not read the jobs' shared state — listing what this process knows", { error: err });
+		}
+		if (!this.started) {
+			const now = /* @__PURE__ */ new Date();
+			for (const job of jobs) try {
+				viewOf(job.id).nextRunAt = parseCronExpression(job.definition.schedule, now, job.definition.timezone);
+			} catch {}
+			if (store?.fetchRunSummaries && ids.length > 0) try {
+				const summaries = await store.fetchRunSummaries(ids);
+				for (const [id, summary] of summaries) viewOf(id).summary = summary;
+			} catch (err) {
+				logger.warn("[cron] Could not read the jobs' run history — listing what this process knows", { error: err });
+			}
+		}
+		return views;
+	}
 	/**
 	* Warn once at start when the process looks like it is running on a
 	* platform that freezes or evicts instances between requests, where the
@@ -638,7 +922,7 @@ var CronScheduler = class {
 	*/
 	scheduleNext(id) {
 		const job = this.jobs.get(id);
-		if (!job || !job.enabled || !this.started) return;
+		if (!job || !this.started || !this.shouldArm(job)) return;
 		this.stopJob(id);
 		try {
 			const now = /* @__PURE__ */ new Date();
@@ -648,21 +932,26 @@ var CronScheduler = class {
 			const delay = Math.max(rawDelay, MIN_SCHEDULE_INTERVAL_MS);
 			if (delay > MAX_TIMER_DELAY_MS) {
 				const hop = setTimeout(() => {
-					if (this.started && job.enabled) this.scheduleNext(id);
+					if (this.started && this.shouldArm(job)) this.scheduleNext(id);
 				}, MAX_TIMER_DELAY_MS);
 				if (hop && typeof hop === "object" && "unref" in hop) hop.unref();
 				job.timerId = hop;
 				return;
 			}
 			const timer = setTimeout(async () => {
-				if (!job.enabled || !this.started) return;
-				if (job.executing) {
-					logger.warn(`[cron] Skipping scheduled run of "${id}" — still executing from previous run`);
-					this.recordSkip(job, "already_executing", false);
+				if (!this.started) return;
+				if (Date.now() < nextRun.getTime()) {
 					this.scheduleNext(id);
 					return;
 				}
-				if (Date.now() < nextRun.getTime()) {
+				if (!await this.isEnabledNow(job)) {
+					if (this.started && this.shouldArm(job)) this.scheduleNext(id);
+					return;
+				}
+				if (!this.started) return;
+				if (job.executing) {
+					logger.warn(`[cron] Skipping scheduled run of "${id}" — still executing from previous run`);
+					this.recordSkip(job, "already_executing", false);
 					this.scheduleNext(id);
 					return;
 				}
@@ -675,12 +964,12 @@ var CronScheduler = class {
 					}
 					if (!claimed) {
 						logger.info(`[cron] Slot ${nextRun.toISOString()} for "${id}" claimed by another instance — skipping`);
-						if (this.started && job.enabled) this.scheduleNext(id);
+						if (this.started && this.shouldArm(job)) this.scheduleNext(id);
 						return;
 					}
 				}
 				await this.executeJob(job, false);
-				if (this.started && job.enabled) this.scheduleNext(id);
+				if (this.started && this.shouldArm(job)) this.scheduleNext(id);
 			}, delay);
 			if (timer && typeof timer === "object" && "unref" in timer) timer.unref();
 			job.timerId = timer;
@@ -707,15 +996,17 @@ var CronScheduler = class {
 	* otherwise ticking correctly.
 	*/
 	async catchUpMissedSlots() {
-		const candidates = [...this.jobs.values()].filter((job) => job.enabled && (job.definition.catchUpWindowSeconds ?? 0) > 0);
+		const candidates = [...this.jobs.values()].filter((job) => this.shouldArm(job) && (job.definition.catchUpWindowSeconds ?? 0) > 0);
 		if (candidates.length === 0) return;
 		if (!this.store?.tryClaimRun) {
 			logger.warn(`[cron] Catch-up is configured on ${candidates.length} job(s) but no claims-capable store is attached — skipping. Without claims a restart cannot tell an unrun slot from one the previous instance already ran.`);
 			return;
 		}
-		const now = /* @__PURE__ */ new Date();
 		for (const job of candidates) try {
-			if (!this.started || !job.enabled || job.executing) continue;
+			if (!this.started || job.executing) continue;
+			if (!await this.isEnabledNow(job)) continue;
+			if (!this.started || job.executing) continue;
+			const now = /* @__PURE__ */ new Date();
 			const windowSeconds = job.definition.catchUpWindowSeconds;
 			const from = /* @__PURE__ */ new Date(now.getTime() - windowSeconds * 1e3);
 			const slot = findMostRecentSlot(job.definition.schedule, from, now, job.definition.timezone);
@@ -756,11 +1047,34 @@ var CronScheduler = class {
 	* - Persists to store (non-blocking) if available
 	* - Always restores state even on catastrophic errors
 	*/
-	async executeJob(job, manual, seedLog) {
+	executeJob(job, manual, seedLog) {
+		const abort = new AbortController();
+		this.runControllers.add(abort);
+		const run = this.runJob(job, manual, abort, seedLog);
+		this.track(run);
+		return run.finally(() => {
+			this.runControllers.delete(abort);
+		});
+	}
+	/** Hold `stop()` until `work` settles. */
+	track(work) {
+		this.inFlight.add(work);
+		const release = () => {
+			this.inFlight.delete(work);
+		};
+		work.then(release, release);
+	}
+	async runJob(job, manual, abort, seedLog) {
+		if (job.executing) return this.recordSkip(job, "already_executing", manual);
+		job.executing = true;
+		const lease = await this.acquireLease(job);
+		if (!lease.acquired) {
+			job.executing = false;
+			logger.warn(`[cron] Skipping ${manual ? "manual trigger" : "scheduled run"} of "${job.id}" — it is running on ${lease.holder ?? "another process"}`);
+			return this.recordSkip(job, "already_executing", manual, lease.holder ?? "another process");
+		}
 		const startedAt = /* @__PURE__ */ new Date();
 		const capturedLogs = seedLog ? [seedLog] : [];
-		job.executing = true;
-		const abort = new AbortController();
 		const ctx = {
 			jobId: job.id,
 			scheduledAt: startedAt,
@@ -777,26 +1091,26 @@ var CronScheduler = class {
 		let success = true;
 		let error;
 		let result;
+		const aborted = new Promise((_, reject) => {
+			const end = () => reject(abort.signal.reason);
+			if (abort.signal.aborted) end();
+			else abort.signal.addEventListener("abort", end, { once: true });
+		});
+		aborted.catch(() => void 0);
 		try {
-			const timeout = (job.definition.timeoutSeconds ?? 300) * 1e3;
-			const handlerPromise = Promise.resolve(job.definition.handler(ctx));
-			let timeoutHandle;
-			const timeoutPromise = new Promise((_, reject) => {
-				timeoutHandle = setTimeout(() => {
-					abort.abort(/* @__PURE__ */ new Error(`Cron job "${job.id}" timed out after ${timeout}ms`));
-					reject(/* @__PURE__ */ new Error(`Cron job "${job.id}" timed out after ${timeout}ms`));
-				}, timeout);
-			});
+			const timeout = (job.definition.timeoutSeconds ?? DEFAULT_TIMEOUT_SECONDS) * 1e3;
+			const cancelTimeout = Number.isFinite(timeout) ? setLongTimeout(() => abort.abort(/* @__PURE__ */ new Error(`Cron job "${job.id}" timed out after ${timeout}ms`)), timeout) : void 0;
 			try {
-				result = await Promise.race([handlerPromise, timeoutPromise]);
+				result = await Promise.race([Promise.resolve(job.definition.handler(ctx)), aborted]);
 			} finally {
-				clearTimeout(timeoutHandle);
+				cancelTimeout?.();
 			}
 		} catch (err) {
 			success = false;
 			error = redactSensitiveText(err instanceof Error ? err.message : String(err));
 			job.totalFailures++;
 		} finally {
+			if (lease.token) await this.releaseLease(job, lease.token);
 			job.executing = false;
 		}
 		const finishedAt = /* @__PURE__ */ new Date();
@@ -817,31 +1131,43 @@ var CronScheduler = class {
 		};
 		job.logs.push(logEntry);
 		if (job.logs.length > MAX_LOGS_PER_JOB) job.logs.shift();
-		if (this.store) this.store.insertLog(logEntry).catch((persistErr) => {
+		if (this.store) this.track(this.store.insertLog(logEntry).catch((persistErr) => {
 			logger.error(`[cron] Failed to persist log for "${job.id}"`, { error: persistErr });
-		});
+		}));
 		if (success) logger.info(`✅ [cron] "${job.id}" completed in ${durationMs}ms`);
 		else logger.error(`❌ [cron] "${job.id}" failed in ${durationMs}ms: ${error}`);
 		return logEntry;
 	}
-	toStatus(job) {
+	/**
+	* A job's status: this process's own record, overlaid with what `view`
+	* read from the store. A paused job shows no next run, even where its
+	* timer stays armed to read the state again at that slot.
+	*/
+	toStatus(job, view = {}) {
+		const summary = view.summary;
+		const nextRunAt = job.enabled ? job.nextRunAt ?? view.nextRunAt : void 0;
+		let state = job.state;
+		if (!job.executing) {
+			if (view.runningElsewhere) state = "running";
+			else if (summary && job.enabled) state = summary.lastSuccess === false ? "error" : "idle";
+		}
 		return {
 			id: job.id,
 			name: job.definition.name ?? job.id,
 			description: job.definition.description,
 			schedule: job.definition.schedule,
 			enabled: job.enabled,
-			state: job.state,
-			lastRunAt: job.lastRunAt?.toISOString(),
-			nextRunAt: job.nextRunAt?.toISOString(),
-			lastDurationMs: job.lastDurationMs,
-			lastError: job.lastError,
-			totalRuns: job.totalRuns,
-			totalFailures: job.totalFailures
+			state,
+			lastRunAt: summary ? summary.lastRunAt : job.lastRunAt?.toISOString(),
+			nextRunAt: nextRunAt?.toISOString(),
+			lastDurationMs: summary ? summary.lastDurationMs : job.lastDurationMs,
+			lastError: summary ? summary.lastSuccess === false ? summary.lastError : void 0 : job.lastError,
+			totalRuns: summary ? summary.totalRuns : job.totalRuns,
+			totalFailures: summary ? summary.totalFailures : job.totalFailures
 		};
 	}
 };
 //#endregion
-export { parseEnvBoolean as i, cron_scheduler_exports as n, validateCronExpression as r, CronScheduler as t };
+export { parseEnvBoolean as a, validateCronExpression as i, cron_scheduler_exports as n, isAlreadyExecutingSkip as r, CronScheduler as t };
 
-//# sourceMappingURL=cron-scheduler-COPQxlEq.js.map
+//# sourceMappingURL=cron-scheduler-B0pLfAix.js.map