@happyvertical/smrt-jobs 0.37.2 → 0.37.4

package/dist/chunks/runner-DLsWfeY-.js

@@ -0,0 +1,1536 @@
+import { a as resolveEngine, c as unregisterLiveWorker, i as registerLiveWorker, n as isWorkerAlive, o as resolveUrl, r as offLoopEligible, s as tuneSqliteForConcurrency, t as createWorkerKey } from "./worker-liveness-C1Gjnhax.js";
+import { ObjectRegistry, SmrtCollection, SmrtObject, detectEngine, ensureJobEventsSystemTableCompatibility, ensureJobsSystemTableCompatibility, field, foreignKey, getClassConfigResolvers, resolveLazyConfig, smrt } from "@happyvertical/smrt-core";
+import { fromConfig } from "@happyvertical/jobs";
+import { TenantContext, TenantScoped, getTenantId, tenantId } from "@happyvertical/smrt-tenancy";
+import { EventEmitter } from "node:events";
+import { Worker } from "node:worker_threads";
+import { createLogger } from "@happyvertical/logger";
+import { createId } from "@happyvertical/utils";
+//#region src/__smrt-register__.ts
+ObjectRegistry.registerPackageManifest(new URL("./manifest.json", "" + import.meta.url));
+//#endregion
+//#region src/background-policy.ts
+var MAX_JOB_RETRIES = 25;
+var DEFAULT_TENANT_JOB_CAP = 1e4;
+function clampRetries(requested) {
+	if (Number.isNaN(requested) || requested < 0) return 0;
+	if (requested === Number.POSITIVE_INFINITY) return 25;
+	return Math.min(Math.floor(requested), 25);
+}
+var TenantJobCapExceededError = class extends Error {
+	constructor(tenantId, cap, current) {
+		super(`Tenant "${tenantId}" has reached its background-job cap (${current}/${cap} in-flight). Refusing to enqueue another job.`);
+		this.tenantId = tenantId;
+		this.cap = cap;
+		this.current = current;
+		this.name = "TenantJobCapExceededError";
+	}
+	tenantId;
+	cap;
+	current;
+};
+function assertWithinTenantCreationCap(tenantId, current, cap) {
+	if (!tenantId || cap <= 0) return;
+	if (current >= cap) throw new TenantJobCapExceededError(tenantId, cap, current);
+}
+function markBackgroundEligible(ctor, ...methods) {
+	const target = ctor;
+	const existing = target.backgroundEligibleMethods;
+	const set = existing instanceof Set ? new Set(existing) : new Set(existing ?? []);
+	for (const method of methods) set.add(method);
+	target.backgroundEligibleMethods = set;
+}
+function backgroundEligible() {
+	return (target, propertyKey, descriptor) => {
+		const ctor = target.constructor;
+		markBackgroundEligible(ctor, String(propertyKey));
+		return descriptor;
+	};
+}
+function getBackgroundEligibleMethods(ctor) {
+	const declared = ctor?.backgroundEligibleMethods;
+	if (declared == null) return null;
+	return declared instanceof Set ? declared : new Set(declared);
+}
+function isBackgroundEligibleMethod(ctor, method) {
+	const allow = getBackgroundEligibleMethods(ctor);
+	if (allow == null) return true;
+	return allow.has(method);
+}
+//#endregion
+//#region src/error-redaction.ts
+var REDACTED = "***REDACTED***";
+var CREDENTIAL_URL_RE = /([a-z][a-z0-9+.-]*:\/\/)[^/?#@\s]+@/gi;
+var BEARER_TOKEN_RE = /\b(bearer|token)\s+[A-Za-z0-9._\-+/=]{8,}/gi;
+var AUTHORIZATION_HEADER_RE = /\bauthorization\s*[:=]\s*(?:[A-Za-z]+\s+)?[^\s,;)]+/gi;
+var SECRET_KEY_VALUE_RE = /\b([A-Za-z0-9_-]*(?:password|passwd|pwd|secret|api[_-]?key|access[_-]?key|secret[_-]?key|private[_-]?key|client[_-]?secret|token|credential|auth)[A-Za-z0-9_-]*)\s*([:=])\s*("[^"]*"|'[^']*'|[^\s,;)]+)/gi;
+var JSON_SECRET_KEY_VALUE_RE = /("(?:[A-Za-z0-9_-]*(?:password|passwd|pwd|secret|api[_-]?key|access[_-]?key|secret[_-]?key|private[_-]?key|client[_-]?secret|token|credential|auth)[A-Za-z0-9_-]*)")\s*:\s*"[^"]*"/gi;
+var STANDALONE_SECRET_RE = /\b(sk-[A-Za-z0-9_-]{14,}[A-Za-z0-9]|sk_(?:live|test)_[A-Za-z0-9_-]{14,}[A-Za-z0-9]|AKIA[0-9A-Z]{12,}|gh[pousr]_[A-Za-z0-9]{20,}|AIza[A-Za-z0-9_-]{20,}|xox[bp]-[A-Za-z0-9-]{10,})\b/g;
+function redactErrorMessage(message) {
+	if (typeof message !== "string" || message.length === 0) return message;
+	return message.replace(CREDENTIAL_URL_RE, `$1${REDACTED}@`).replace(AUTHORIZATION_HEADER_RE, `authorization=${REDACTED}`).replace(BEARER_TOKEN_RE, `$1 ${REDACTED}`).replace(JSON_SECRET_KEY_VALUE_RE, `$1:"${REDACTED}"`).replace(SECRET_KEY_VALUE_RE, `$1$2${REDACTED}`).replace(STANDALONE_SECRET_RE, REDACTED);
+}
+function redactErrorForPersistence(error) {
+	if (error instanceof Error) return redactErrorMessage(error.message);
+	return redactErrorMessage(String(error));
+}
+//#endregion
+//#region src/logger-extension.ts
+var JobContextLogger = class {
+	constructor(baseLogger, jobContext) {
+		this.baseLogger = baseLogger;
+		this.jobContext = jobContext;
+	}
+	baseLogger;
+	jobContext;
+	addContext(data) {
+		return {
+			...data,
+			_job: {
+				id: this.jobContext.jobId,
+				attempt: this.jobContext.attempt,
+				queue: this.jobContext.queue,
+				objectType: this.jobContext.objectType,
+				method: this.jobContext.method
+			}
+		};
+	}
+	debug(message, data) {
+		this.baseLogger.debug(message, this.addContext(data));
+	}
+	info(message, data) {
+		this.baseLogger.info(message, this.addContext(data));
+	}
+	warn(message, data) {
+		this.baseLogger.warn(message, this.addContext(data));
+	}
+	error(message, data) {
+		this.baseLogger.error(message, this.addContext(data));
+	}
+};
+//#endregion
+//#region src/smrt-job.ts
+var __defProp$2 = Object.defineProperty;
+var __getOwnPropDesc$2 = Object.getOwnPropertyDescriptor;
+var __decorateClass$2 = (decorators, target, key, kind) => {
+	var result = kind > 1 ? void 0 : kind ? __getOwnPropDesc$2(target, key) : target;
+	for (var i = decorators.length - 1, decorator; i >= 0; i--) if (decorator = decorators[i]) result = (kind ? decorator(target, key, result) : decorator(result)) || result;
+	if (kind && result) __defProp$2(target, key, result);
+	return result;
+};
+var SmrtJob = class extends SmrtObject {
+	tenantId = void 0;
+	queue = "default";
+	objectType = "";
+	objectId = null;
+	method = "";
+	args = {};
+	runAt = /* @__PURE__ */ new Date();
+	priority = 50;
+	status = "pending";
+	attempts = 0;
+	maxAttempts = 3;
+	timeout = 3e5;
+	timeoutBehavior = "fail";
+	startedAt = null;
+	completedAt = null;
+	lastError = null;
+	resultPointer = null;
+	retryStrategy = {
+		type: "exponential",
+		config: {
+			initialDelay: 1e3,
+			multiplier: 2,
+			maxDelay: 3e5
+		}
+	};
+	workerId = null;
+	workerHeartbeat = null;
+	/**
+	* Capture ambient tenant context when a job is saved inside withTenant().
+	*
+	* Scheduled jobs can also set this explicitly from their owning schedule.
+	*/
+	async save() {
+		if (this.tenantId === void 0) {
+			const contextTenantId = getTenantId();
+			if (contextTenantId) this.tenantId = contextTenantId;
+		}
+		return super.save();
+	}
+	/**
+	* Mark the job for retry
+	*/
+	async retry() {
+		if (this.status === "completed") throw new Error("Cannot retry a completed job");
+		this.status = "pending";
+		this.attempts = 0;
+		this.lastError = null;
+		this.startedAt = null;
+		this.completedAt = null;
+		this.workerId = null;
+		this.workerHeartbeat = null;
+		await this.save();
+	}
+	/**
+	* Cancel the job
+	*/
+	async cancel() {
+		if (this.status === "completed" || this.status === "cancelled") throw new Error(`Cannot cancel job with status: ${this.status}`);
+		this.status = "cancelled";
+		this.completedAt = /* @__PURE__ */ new Date();
+		await this.save();
+	}
+	/**
+	* Get a human-readable description of the job
+	*/
+	getDescription() {
+		return `${this.objectId ? `${this.objectType}#${this.objectId}` : this.objectType}.${this.method}()`;
+	}
+};
+__decorateClass$2([tenantId({ nullable: true })], SmrtJob.prototype, "tenantId", 2);
+__decorateClass$2([field({
+	type: "text",
+	required: true,
+	default: "default"
+})], SmrtJob.prototype, "queue", 2);
+__decorateClass$2([field({
+	type: "text",
+	required: true
+})], SmrtJob.prototype, "objectType", 2);
+__decorateClass$2([field({
+	type: "text",
+	nullable: true
+})], SmrtJob.prototype, "objectId", 2);
+__decorateClass$2([field({
+	type: "text",
+	required: true
+})], SmrtJob.prototype, "method", 2);
+__decorateClass$2([field({ type: "json" })], SmrtJob.prototype, "args", 2);
+__decorateClass$2([field({
+	type: "datetime",
+	required: true
+})], SmrtJob.prototype, "runAt", 2);
+__decorateClass$2([field({
+	type: "integer",
+	required: true,
+	default: 50
+})], SmrtJob.prototype, "priority", 2);
+__decorateClass$2([field({
+	type: "text",
+	required: true,
+	default: "pending"
+})], SmrtJob.prototype, "status", 2);
+__decorateClass$2([field({
+	type: "integer",
+	required: true,
+	default: 0
+})], SmrtJob.prototype, "attempts", 2);
+__decorateClass$2([field({
+	type: "integer",
+	required: true,
+	default: 3
+})], SmrtJob.prototype, "maxAttempts", 2);
+__decorateClass$2([field({
+	type: "integer",
+	required: true,
+	default: 3e5
+})], SmrtJob.prototype, "timeout", 2);
+__decorateClass$2([field({
+	type: "text",
+	required: true,
+	default: "fail"
+})], SmrtJob.prototype, "timeoutBehavior", 2);
+__decorateClass$2([field({
+	type: "datetime",
+	nullable: true
+})], SmrtJob.prototype, "startedAt", 2);
+__decorateClass$2([field({
+	type: "datetime",
+	nullable: true
+})], SmrtJob.prototype, "completedAt", 2);
+__decorateClass$2([field({
+	type: "text",
+	nullable: true
+})], SmrtJob.prototype, "lastError", 2);
+__decorateClass$2([field({
+	type: "text",
+	nullable: true
+})], SmrtJob.prototype, "resultPointer", 2);
+__decorateClass$2([field({ type: "json" })], SmrtJob.prototype, "retryStrategy", 2);
+__decorateClass$2([field({
+	type: "text",
+	nullable: true
+})], SmrtJob.prototype, "workerId", 2);
+__decorateClass$2([field({
+	type: "datetime",
+	nullable: true
+})], SmrtJob.prototype, "workerHeartbeat", 2);
+SmrtJob = __decorateClass$2([smrt({
+	tableName: "_smrt_jobs",
+	api: false,
+	cli: {
+		include: [
+			"list",
+			"get",
+			"retry",
+			"cancel"
+		],
+		skipApiCheck: true,
+		http: false
+	},
+	mcp: false
+}), TenantScoped({ mode: "optional" })], SmrtJob);
+var SmrtJobCollection = class extends SmrtCollection {
+	static _itemClass = SmrtJob;
+	async initialize() {
+		await super.initialize();
+		await ensureJobsSystemTableCompatibility(this.db);
+		return this;
+	}
+	/**
+	* List jobs by status
+	*/
+	async listByStatus(status, options = {}) {
+		const where = { status: Array.isArray(status) ? status : [status] };
+		if (options.queue) where.queue = options.queue;
+		return this.list({
+			where,
+			orderBy: ["priority DESC", "run_at ASC"],
+			limit: options.limit
+		});
+	}
+	/**
+	* List pending jobs ready to run
+	*/
+	async listReady(options = {}) {
+		const now = (/* @__PURE__ */ new Date()).toISOString();
+		const whereConditions = ["status = 'pending'", "run_at <= ?"];
+		const params = [now];
+		if (options.queues?.length) {
+			const placeholders = options.queues.map(() => "?").join(", ");
+			whereConditions.push(`queue IN (${placeholders})`);
+			params.push(...options.queues);
+		}
+		params.push(options.limit || 100);
+		return this.query(`SELECT * FROM _smrt_jobs WHERE ${whereConditions.join(" AND ")} ORDER BY priority DESC, run_at ASC LIMIT ?`, params, { allowRawOnTenantScoped: true });
+	}
+	/**
+	* Atomically claim pending jobs ready to run for a worker.
+	*
+	* The claim is performed as one conditional UPDATE so concurrent workers
+	* cannot receive the same pending row. PostgreSQL additionally skips rows
+	* locked by other workers instead of waiting behind them.
+	*/
+	async claimReady(options) {
+		const limit = options.limit ?? 100;
+		if (limit <= 0) return [];
+		const nowIso = (options.now ?? /* @__PURE__ */ new Date()).toISOString();
+		const whereConditions = ["status = 'pending'", "run_at <= ?"];
+		const whereParams = [nowIso];
+		if (options.queues?.length) {
+			const placeholders = options.queues.map(() => "?").join(", ");
+			whereConditions.push(`queue IN (${placeholders})`);
+			whereParams.push(...options.queues);
+		}
+		const lockClause = getDatabaseEngine(this.db) === "postgres" ? " FOR UPDATE SKIP LOCKED" : "";
+		const candidateSelect = `
+      SELECT id
+        FROM _smrt_jobs
+       WHERE ${whereConditions.join(" AND ")}
+       ORDER BY priority DESC, run_at ASC, created_at ASC, id ASC
+       LIMIT ?${lockClause}
+    `;
+		return (await this.query(`UPDATE _smrt_jobs
+          SET status = 'running',
+              worker_id = ?,
+              worker_heartbeat = ?,
+              started_at = ?,
+              attempts = attempts + 1,
+              updated_at = ?
+        WHERE id IN (${candidateSelect})
+          AND status = 'pending'
+        RETURNING *`, [
+			options.workerId,
+			nowIso,
+			nowIso,
+			nowIso,
+			...whereParams,
+			limit
+		], { allowRawOnTenantScoped: true })).toSorted(compareClaimOrder);
+	}
+	/**
+	* Count non-terminal (pending/running) jobs owned by a tenant.
+	*
+	* Used to enforce the per-tenant creation cap so one tenant cannot exhaust
+	* the shared worker pool (S5 audit #1402). Reads `_smrt_jobs` directly so it
+	* works regardless of ambient tenant context.
+	*
+	* @param tenantId - Tenant to count for. `null` counts global (NULL-tenant)
+	*   jobs.
+	*/
+	async countInFlightForTenant(tenantId2) {
+		const predicate = tenantId2 === null ? "tenant_id IS NULL" : "tenant_id = ?";
+		const params = tenantId2 === null ? [] : [tenantId2];
+		const row = (await this.db.query(`SELECT COUNT(*) AS count
+         FROM _smrt_jobs
+        WHERE status IN ('pending', 'running')
+          AND ${predicate}`, ...params)).rows[0];
+		return Number(row?.count ?? 0);
+	}
+	/**
+	* The single creation path for queued jobs.
+	*
+	* Centralizes the two creation-time security guards from the S5 audit (#1402)
+	* so every enqueue — the fluent {@link "./job-builder".JobBuilder} *and* the
+	* ScheduleRunner's cron-triggered jobs — goes through one place:
+	*
+	* 1. `maxAttempts` is clamped to {@link MAX_JOB_RETRIES} so a misconfigured
+	*    caller cannot pin a worker on a poison job indefinitely.
+	* 2. A per-tenant in-flight cap bounds how many non-terminal jobs one tenant
+	*    may hold, so one tenant cannot exhaust the shared worker pool
+	*    (cross-tenant denial of service). The cap applies to the row's effective
+	*    tenant (explicit `data.tenantId` or, when absent, the ambient context);
+	*    global (null-tenant) jobs are exempt.
+	*
+	* Atomicity note (best-effort soft cap, by design): the cap is a
+	* count-then-insert, NOT a hard transactional invariant. It is intentionally
+	* left non-atomic. A plain transaction would not help — under the adapters'
+	* default isolation two concurrent same-tenant enqueues would each read the
+	* same COUNT and both insert, so serializing them would require either a
+	* per-tenant lock row (`SELECT ... FOR UPDATE`) or SERIALIZABLE-isolation
+	* retry loops. That cross-process locking is fragile (lock-row contention,
+	* adapter-specific isolation behavior, the `transaction` adapter method being
+	* optional) and out of proportion to the threat: this cap is defense in depth
+	* against runaway/accidental creation exhausting the shared worker pool, not a
+	* billing/quota boundary. So under truly simultaneous enqueues a tenant may
+	* momentarily overshoot by the number of in-flight creators; the bound still
+	* prevents unbounded growth and closes the prior ScheduleRunner bypass. If a
+	* hard guarantee is ever needed, enforce it with a DB CHECK/trigger or a
+	* dedicated counter row, not an application-level lock.
+	*/
+	async enqueueJob(data, options = {}) {
+		const cap = options.tenantJobCap ?? 1e4;
+		const explicitTenant = typeof data.tenantId === "string" && data.tenantId.length > 0 ? data.tenantId : data.tenantId === null ? null : void 0;
+		const effectiveTenant = explicitTenant !== void 0 ? explicitTenant : getTenantId() ?? null;
+		if (effectiveTenant && cap > 0) assertWithinTenantCreationCap(effectiveTenant, await this.countInFlightForTenant(effectiveTenant), cap);
+		const job = await this.create({
+			...data,
+			maxAttempts: clampRetries(data.maxAttempts ?? 3)
+		});
+		await job.save();
+		return job;
+	}
+	/**
+	* Get job statistics
+	*/
+	async stats(queue) {
+		const query = queue ? "SELECT status, COUNT(*) as count FROM _smrt_jobs WHERE queue = ? GROUP BY status" : "SELECT status, COUNT(*) as count FROM _smrt_jobs GROUP BY status";
+		const params = queue ? [queue] : [];
+		const result = await this._db.query(query, ...params);
+		const counts = {};
+		for (const row of result.rows) counts[row.status] = row.count;
+		return {
+			pending: counts.pending ?? 0,
+			running: counts.running ?? 0,
+			completed: counts.completed ?? 0,
+			failed: counts.failed ?? 0,
+			cancelled: counts.cancelled ?? 0
+		};
+	}
+	/**
+	* Cleanup old completed/failed jobs
+	*/
+	async cleanup(options) {
+		const conditions = [];
+		const params = [];
+		if (options.completedBefore) {
+			conditions.push("(status = 'completed' AND completed_at < ?)");
+			params.push(options.completedBefore.toISOString());
+		}
+		if (options.failedBefore) {
+			conditions.push("(status = 'failed' AND completed_at < ?)");
+			params.push(options.failedBefore.toISOString());
+		}
+		if (options.cancelledBefore) {
+			conditions.push("(status = 'cancelled' AND completed_at < ?)");
+			params.push(options.cancelledBefore.toISOString());
+		}
+		if (conditions.length === 0) return 0;
+		let query = `DELETE FROM _smrt_jobs WHERE (${conditions.join(" OR ")})`;
+		if (options.limit) {
+			query = `
+        DELETE FROM _smrt_jobs
+        WHERE id IN (
+          SELECT id FROM _smrt_jobs
+          WHERE (${conditions.join(" OR ")})
+          LIMIT ?
+        )
+      `;
+			params.push(options.limit);
+		}
+		return (await this._db.query(query, ...params)).rowCount ?? 0;
+	}
+};
+function getDatabaseEngine(db) {
+	const dbWithConfig = db;
+	return detectEngine(db.url || dbWithConfig.config?.url || "", dbWithConfig.type || dbWithConfig.config?.type);
+}
+function compareClaimOrder(left, right) {
+	const priority = right.priority - left.priority;
+	if (priority !== 0) return priority;
+	const runAt = left.runAt.getTime() - right.runAt.getTime();
+	if (runAt !== 0) return runAt;
+	const createdAt = timestamp(left.created_at) - timestamp(right.created_at);
+	if (createdAt !== 0) return createdAt;
+	return (left.id ?? "").localeCompare(right.id ?? "");
+}
+function timestamp(value) {
+	return value?.getTime() ?? 0;
+}
+//#endregion
+//#region src/smrt-job-event.ts
+var __defProp$1 = Object.defineProperty;
+var __getOwnPropDesc$1 = Object.getOwnPropertyDescriptor;
+var __decorateClass$1 = (decorators, target, key, kind) => {
+	var result = kind > 1 ? void 0 : kind ? __getOwnPropDesc$1(target, key) : target;
+	for (var i = decorators.length - 1, decorator; i >= 0; i--) if (decorator = decorators[i]) result = (kind ? decorator(target, key, result) : decorator(result)) || result;
+	if (kind && result) __defProp$1(target, key, result);
+	return result;
+};
+var JOB_EVENT_STORAGE_COLUMNS = [
+	"id",
+	"slug",
+	"context",
+	"created_at",
+	"updated_at",
+	"tenant_id",
+	"job_id",
+	"type",
+	"level",
+	"stage",
+	"progress",
+	"message",
+	"data"
+].join(", ");
+var SmrtJobEvent = class extends SmrtObject {
+	tenantId = void 0;
+	jobId = "";
+	type = "log";
+	level = "info";
+	stage = null;
+	progress = null;
+	message = "";
+	data = {};
+	createdAt = /* @__PURE__ */ new Date();
+	toCursor() {
+		return `${this.createdAt instanceof Date ? this.createdAt.toISOString() : String(this.createdAt)}|${this.id ?? ""}`;
+	}
+};
+__decorateClass$1([tenantId({ nullable: true })], SmrtJobEvent.prototype, "tenantId", 2);
+__decorateClass$1([foreignKey("SmrtJob", { required: true })], SmrtJobEvent.prototype, "jobId", 2);
+__decorateClass$1([field({
+	type: "text",
+	required: true,
+	default: "log"
+})], SmrtJobEvent.prototype, "type", 2);
+__decorateClass$1([field({
+	type: "text",
+	required: true,
+	default: "info"
+})], SmrtJobEvent.prototype, "level", 2);
+__decorateClass$1([field({
+	type: "text",
+	nullable: true
+})], SmrtJobEvent.prototype, "stage", 2);
+__decorateClass$1([field({
+	type: "integer",
+	nullable: true
+})], SmrtJobEvent.prototype, "progress", 2);
+__decorateClass$1([field({
+	type: "text",
+	required: true,
+	default: ""
+})], SmrtJobEvent.prototype, "message", 2);
+__decorateClass$1([field({ type: "json" })], SmrtJobEvent.prototype, "data", 2);
+__decorateClass$1([field({
+	type: "datetime",
+	required: true
+})], SmrtJobEvent.prototype, "createdAt", 2);
+SmrtJobEvent = __decorateClass$1([smrt({
+	tableName: "_smrt_job_events",
+	api: false,
+	cli: {
+		include: ["list", "get"],
+		http: false,
+		skipApiCheck: true
+	},
+	mcp: false
+}), TenantScoped({ mode: "optional" })], SmrtJobEvent);
+function normalizeLimit(limit) {
+	return Math.max(1, Math.min(1e3, Math.floor(typeof limit === "number" && Number.isFinite(limit) ? limit : 250)));
+}
+function normalizeProgress(progress) {
+	if (typeof progress !== "number" || !Number.isFinite(progress)) return null;
+	return Math.max(0, Math.min(100, Math.round(progress)));
+}
+function parseCursor(cursor) {
+	if (typeof cursor !== "string") return cursor;
+	const separator = cursor.lastIndexOf("|");
+	if (separator === -1) return {
+		createdAt: cursor,
+		id: ""
+	};
+	return {
+		createdAt: cursor.slice(0, separator),
+		id: cursor.slice(separator + 1)
+	};
+}
+function normalizeCursorDate(value) {
+	if (value instanceof Date) return value.toISOString();
+	const parsed = new Date(value);
+	if (!Number.isNaN(parsed.getTime())) return parsed.toISOString();
+	return value;
+}
+function usesSqliteDateFunctions(dbUrl) {
+	const normalized = dbUrl.toLowerCase();
+	return !(normalized.startsWith("postgres:") || normalized.startsWith("postgresql:"));
+}
+function getQueryRows(result) {
+	if (Array.isArray(result)) return result;
+	return result.rows ?? [];
+}
+var SmrtJobEventCollection = class extends SmrtCollection {
+	static _itemClass = SmrtJobEvent;
+	async initialize() {
+		await super.initialize();
+		await ensureJobEventsSystemTableCompatibility(this.db);
+		return this;
+	}
+	async append(input) {
+		return this.create({
+			tenantId: input.tenantId,
+			jobId: input.jobId,
+			type: input.type ?? "log",
+			level: input.level ?? "info",
+			stage: input.stage ?? null,
+			progress: normalizeProgress(input.progress),
+			message: input.message ?? "",
+			data: input.data ?? {},
+			createdAt: input.createdAt ?? /* @__PURE__ */ new Date()
+		});
+	}
+	async listByJob(jobId, options = {}) {
+		return this.listSinceCursor({
+			...options,
+			jobId
+		});
+	}
+	async listSinceCursor(options = {}) {
+		const where = [];
+		const params = [];
+		if (options.jobId) {
+			where.push("job_id = ?");
+			params.push(options.jobId);
+		}
+		this.addTenantPredicate(where, params, options);
+		if (options.cursor) {
+			const cursor = parseCursor(options.cursor);
+			const createdAt = await this.resolveCursorCreatedAt(cursor, options);
+			const createdAtExpression = this.createdAtComparableExpression();
+			where.push(`(${createdAtExpression} > ? OR (${createdAtExpression} = ? AND id > ?))`);
+			params.push(createdAt, createdAt, cursor.id);
+		} else if (options.since) {
+			where.push(`${this.createdAtComparableExpression()} > ?`);
+			params.push(normalizeCursorDate(options.since));
+		}
+		if (options.afterId) {
+			where.push("id > ?");
+			params.push(options.afterId);
+		}
+		params.push(normalizeLimit(options.limit));
+		const whereSql = where.length ? `WHERE ${where.join(" AND ")}` : "";
+		return this.query(`SELECT ${JOB_EVENT_STORAGE_COLUMNS}
+         FROM _smrt_job_events
+        ${whereSql}
+        ORDER BY ${this.createdAtComparableExpression()} ASC, id ASC
+        LIMIT ?`, params, { allowRawOnTenantScoped: true });
+	}
+	async latestProgressByJobIds(jobIds, options = {}) {
+		const uniqueJobIds = [...new Set(jobIds.filter(Boolean))];
+		const latestByJobId = /* @__PURE__ */ new Map();
+		if (uniqueJobIds.length === 0) return latestByJobId;
+		const where = [`job_id IN (${uniqueJobIds.map(() => "?").join(", ")})`, "type = 'progress'"];
+		const params = [...uniqueJobIds];
+		this.addTenantPredicate(where, params, options);
+		const createdAtExpression = this.createdAtComparableExpression();
+		const events = await this.query(`SELECT ${JOB_EVENT_STORAGE_COLUMNS}
+         FROM (
+           SELECT ${JOB_EVENT_STORAGE_COLUMNS},
+                  ${createdAtExpression} AS smrt_created_at_sort,
+                  ROW_NUMBER() OVER (
+                    PARTITION BY job_id
+                    ORDER BY ${createdAtExpression} DESC, id DESC
+                  ) AS smrt_rank
+             FROM _smrt_job_events
+            WHERE ${where.join(" AND ")}
+         ) ranked
+        WHERE smrt_rank = 1
+        ORDER BY smrt_created_at_sort DESC, id DESC`, params, { allowRawOnTenantScoped: true });
+		for (const event of events) latestByJobId.set(event.jobId, event);
+		return latestByJobId;
+	}
+	addTenantPredicate(where, params, options) {
+		if (options.tenantId === null) {
+			where.push("tenant_id IS NULL");
+			return;
+		}
+		const tenantId2 = typeof options.tenantId === "string" ? options.tenantId : getTenantId();
+		if (tenantId2) {
+			where.push("tenant_id = ?");
+			params.push(tenantId2);
+			return;
+		}
+		throw new Error("Tenant-scoped job event queries require tenantId, tenantId: null, or an ambient tenant context.");
+	}
+	createdAtComparableExpression() {
+		if (usesSqliteDateFunctions(this.db.url)) return "strftime('%Y-%m-%dT%H:%M:%fZ', created_at)";
+		return "created_at";
+	}
+	async resolveCursorCreatedAt(cursor, options) {
+		if (!cursor.id) return normalizeCursorDate(cursor.createdAt);
+		const where = ["id = ?"];
+		const params = [cursor.id];
+		if (options.jobId) {
+			where.push("job_id = ?");
+			params.push(options.jobId);
+		}
+		this.addTenantPredicate(where, params, options);
+		const cursorCreatedAt = getQueryRows(await this.db.query(`SELECT ${this.createdAtComparableExpression()} AS cursor_created_at
+         FROM _smrt_job_events
+        WHERE ${where.join(" AND ")}
+        LIMIT 1`, ...params))[0]?.cursor_created_at;
+		return typeof cursorCreatedAt === "string" && cursorCreatedAt.trim() ? cursorCreatedAt : normalizeCursorDate(cursor.createdAt);
+	}
+};
+//#endregion
+//#region src/smrt-worker.ts
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __decorateClass = (decorators, target, key, kind) => {
+	var result = kind > 1 ? void 0 : kind ? __getOwnPropDesc(target, key) : target;
+	for (var i = decorators.length - 1, decorator; i >= 0; i--) if (decorator = decorators[i]) result = (kind ? decorator(target, key, result) : decorator(result)) || result;
+	if (kind && result) __defProp(target, key, result);
+	return result;
+};
+var SmrtWorker = class extends SmrtObject {
+	workerId = "";
+	pid = null;
+	hostname = null;
+	startedAt = null;
+	heartbeatAt = null;
+	leaseExpiresAt = null;
+	status = "running";
+};
+__decorateClass([field({
+	type: "text",
+	required: true
+})], SmrtWorker.prototype, "workerId", 2);
+__decorateClass([field({
+	type: "integer",
+	nullable: true
+})], SmrtWorker.prototype, "pid", 2);
+__decorateClass([field({
+	type: "text",
+	nullable: true
+})], SmrtWorker.prototype, "hostname", 2);
+__decorateClass([field({
+	type: "datetime",
+	nullable: true
+})], SmrtWorker.prototype, "startedAt", 2);
+__decorateClass([field({
+	type: "datetime",
+	nullable: true
+})], SmrtWorker.prototype, "heartbeatAt", 2);
+__decorateClass([field({
+	type: "datetime",
+	nullable: true
+})], SmrtWorker.prototype, "leaseExpiresAt", 2);
+__decorateClass([field({
+	type: "text",
+	required: true,
+	default: "running"
+})], SmrtWorker.prototype, "status", 2);
+SmrtWorker = __decorateClass([smrt({
+	tableName: "_smrt_workers",
+	conflictColumns: ["worker_id"],
+	api: false,
+	cli: false,
+	mcp: false
+})], SmrtWorker);
+var SmrtWorkerCollection = class extends SmrtCollection {
+	static _itemClass = SmrtWorker;
+	/**
+	* Fail fast if the `_smrt_workers` table has not been migrated.
+	*
+	* The framework never creates application/system tables at runtime; the
+	* table is created by `smrt db:migrate` (or `getTestDatabase`). A consumer
+	* that upgrades smrt-jobs without migrating must get a clear, actionable
+	* error at `start()` rather than a confusing recovery failure later.
+	*/
+	async assertReady() {
+		try {
+			await this.db.query("SELECT 1 FROM _smrt_workers LIMIT 1");
+		} catch (error) {
+			throw new Error(`The _smrt_workers table is missing. Run \`smrt db:migrate\` to create job-system tables before starting a TaskRunner/ScheduleRunner. (underlying error: ${error.message})`);
+		}
+	}
+	/** Whether the `_smrt_workers` table exists (recovery skips lease checks if not). */
+	async tableReady() {
+		try {
+			await this.db.query("SELECT 1 FROM _smrt_workers LIMIT 1");
+			return true;
+		} catch {
+			return false;
+		}
+	}
+	/** Register a worker incarnation with its lease seeded to `now + ttl`. */
+	async registerWorker(input) {
+		const now = /* @__PURE__ */ new Date();
+		await this.create({
+			workerId: input.workerKey,
+			pid: input.pid ?? null,
+			hostname: input.hostname ?? null,
+			startedAt: now,
+			heartbeatAt: now,
+			leaseExpiresAt: new Date(now.getTime() + input.leaseTtlMs),
+			status: "running"
+		});
+	}
+	/** Renew a worker's lease to `now + ttl`. */
+	async renewLease(workerKey, leaseTtlMs) {
+		const now = /* @__PURE__ */ new Date();
+		await this.db.query(`UPDATE _smrt_workers
+          SET lease_expires_at = ?,
+              heartbeat_at = ?
+        WHERE worker_id = ?`, new Date(now.getTime() + leaseTtlMs).toISOString(), now.toISOString(), workerKey);
+	}
+	/** Remove a worker incarnation (graceful shutdown). */
+	async expireWorker(workerKey) {
+		await this.db.query("DELETE FROM _smrt_workers WHERE worker_id = ?", workerKey);
+	}
+	/** Worker keys whose database lease is still fresh (alive cross-process). */
+	async freshLeaseWorkerKeys() {
+		const result = await this.db.query(`SELECT worker_id
+         FROM _smrt_workers
+        WHERE lease_expires_at IS NOT NULL
+          AND lease_expires_at >= ?`, (/* @__PURE__ */ new Date()).toISOString());
+		const keys = /* @__PURE__ */ new Set();
+		for (const row of result.rows) if (typeof row.worker_id === "string") keys.add(row.worker_id);
+		return keys;
+	}
+	/** Delete worker rows whose lease expired more than `graceMs` ago. */
+	async pruneExpired(graceMs) {
+		const cutoff = new Date(Date.now() - Math.max(0, graceMs)).toISOString();
+		await this.db.query(`DELETE FROM _smrt_workers
+        WHERE lease_expires_at IS NOT NULL
+          AND lease_expires_at < ?`, cutoff);
+	}
+};
+//#endregion
+//#region src/stale-recovery.ts
+var DEFAULT_TASK_HEARTBEAT_INTERVAL_MS = 3e4;
+var DEFAULT_LEASE_TICK_MS = 1e4;
+var DEFAULT_LEASE_TTL_MS = 3e4;
+var LEASE_TTL_GRACE_MULTIPLIER = 3;
+function getEffectiveLeaseTtlMs(leaseTtlMs, leaseTickMs) {
+	return Math.max(leaseTtlMs, leaseTickMs * LEASE_TTL_GRACE_MULTIPLIER);
+}
+//#endregion
+//#region src/runner.ts
+var LIVENESS_THREAD_START_TIMEOUT_MS = 1e4;
+var JobTimeoutError = class extends Error {
+	constructor(message) {
+		super(message);
+		this.name = "JobTimeoutError";
+	}
+};
+var DEFAULT_CONFIG = {
+	id: "",
+	concurrency: 5,
+	queues: ["default"],
+	pollInterval: 1e3,
+	heartbeatInterval: DEFAULT_TASK_HEARTBEAT_INTERVAL_MS,
+	shutdownTimeout: 3e4,
+	staleJobThresholdMs: 9e4,
+	leaseTtlMs: DEFAULT_LEASE_TTL_MS,
+	leaseTickMs: DEFAULT_LEASE_TICK_MS
+};
+var TaskRunner = class extends EventEmitter {
+	id;
+	/**
+	* Per-incarnation-unique worker key. Stored as the `worker_id` on claimed
+	* jobs and in `_smrt_workers`, so a restart of a runner sharing the same
+	* configured `id` does not look like it still owns the previous
+	* incarnation's orphaned jobs. The human-facing {@link id} stays stable for
+	* events/logs.
+	*/
+	workerKey;
+	config;
+	effectiveLeaseTtlMs;
+	collection = null;
+	eventCollection = null;
+	workerCollection = null;
+	workersTableVerified = false;
+	lastRecoverySweepAt = 0;
+	running = false;
+	activeJobs = /* @__PURE__ */ new Map();
+	pollTimer = null;
+	heartbeatTimer = null;
+	leaseTimer = null;
+	livenessWorker = null;
+	shutdownPromise = null;
+	db = null;
+	logger = createLogger(true);
+	constructor(config = {}) {
+		super();
+		this.config = {
+			...DEFAULT_CONFIG,
+			...config,
+			id: config.id || `runner_${createId().slice(0, 8)}`
+		};
+		this.id = this.config.id;
+		this.workerKey = createWorkerKey(this.id);
+		this.effectiveLeaseTtlMs = getEffectiveLeaseTtlMs(this.config.leaseTtlMs, this.config.leaseTickMs);
+	}
+	/**
+	* Initialize the runner with database connection
+	*/
+	async initialize(db) {
+		this.db = db;
+		this.collection = await SmrtJobCollection.create({ db });
+		this.eventCollection = await SmrtJobEventCollection.create({ db });
+		this.workerCollection = await SmrtWorkerCollection.create({ db });
+	}
+	/**
+	* Start processing jobs
+	*/
+	async start() {
+		if (this.running) return;
+		if (!this.collection || !this.workerCollection) throw new Error("TaskRunner not initialized. Call initialize() first.");
+		await this.workerCollection.assertReady();
+		if (this.db && resolveEngine(this.db) === "sqlite") await tuneSqliteForConcurrency(this.db);
+		await this.workerCollection.registerWorker({
+			workerKey: this.workerKey,
+			pid: typeof process !== "undefined" ? process.pid : null,
+			hostname: typeof process !== "undefined" ? process.env.HOSTNAME ?? null : null,
+			leaseTtlMs: this.effectiveLeaseTtlMs
+		});
+		registerLiveWorker(this.workerKey);
+		this.running = true;
+		if (offLoopEligible(this.db)) {
+			if (!await this.startLivenessThread()) this.startLeaseRenewal();
+		} else this.startLeaseRenewal();
+		this.startPolling();
+		this.startHeartbeat();
+		this.emit("runner:started");
+	}
+	/**
+	* Stop processing jobs (graceful shutdown)
+	*/
+	async stop() {
+		if (!this.running) return;
+		if (this.shutdownPromise) return this.shutdownPromise;
+		this.running = false;
+		if (this.pollTimer) {
+			clearTimeout(this.pollTimer);
+			this.pollTimer = null;
+		}
+		if (this.heartbeatTimer) {
+			clearInterval(this.heartbeatTimer);
+			this.heartbeatTimer = null;
+		}
+		this.shutdownPromise = this.waitForActiveJobs();
+		try {
+			await this.shutdownPromise;
+		} finally {
+			this.shutdownPromise = null;
+			await this.stopLivenessThread();
+			if (this.leaseTimer) {
+				clearInterval(this.leaseTimer);
+				this.leaseTimer = null;
+			}
+			unregisterLiveWorker(this.workerKey);
+			if (this.activeJobs.size === 0) try {
+				await this.workerCollection?.expireWorker(this.workerKey);
+			} catch {}
+			this.emit("runner:stopped");
+		}
+	}
+	/**
+	* Check if runner is running
+	*/
+	isRunning() {
+		return this.running;
+	}
+	/**
+	* Get count of active jobs
+	*/
+	activeJobCount() {
+		return this.activeJobs.size;
+	}
+	/**
+	* Start the polling loop
+	*/
+	startPolling() {
+		const poll = async () => {
+			if (!this.running) return;
+			try {
+				await this.poll();
+			} catch (error) {
+				this.emit("runner:error", error);
+			}
+			if (this.running) this.pollTimer = setTimeout(poll, this.config.pollInterval);
+		};
+		poll();
+	}
+	/**
+	* Poll for and process jobs
+	*/
+	async poll() {
+		if (!this.collection || !this.db) return;
+		await this.recoverStaleJobs();
+		const available = this.config.concurrency - this.activeJobs.size;
+		if (available <= 0) return;
+		const jobs = await this.collection.claimReady({
+			workerId: this.workerKey,
+			queues: this.config.queues,
+			limit: available
+		});
+		for (const job of jobs) {
+			if (!job.id) continue;
+			this.processJob(job).catch((error) => {
+				this.emit("runner:error", error);
+			});
+		}
+	}
+	/**
+	* Process a single job.
+	*
+	* AT-LEAST-ONCE EXECUTION CONTRACT: a `timeout` only races the handler's
+	* promise — JavaScript cannot preempt an already-running handler, so on a
+	* `'fail'` (or `'kill'`) timeout the original handler keeps executing in the
+	* background while the job row is failed. Timeouts are NOT auto-retried (see
+	* handleJobError) precisely so a still-running handler is not duplicated by a
+	* retry; but the orphaned handler's own side effects still happen, and an
+	* ordinary (non-timeout) failure IS retried and re-claimable by any worker.
+	* Handlers invoked from a job MUST be idempotent (e.g. keyed by
+	* `context.job.jobId` or a caller-supplied idempotency key); do not rely on a
+	* job body running exactly once. See AGENTS.md "Timeouts & at-least-once".
+	*/
+	async processJob(job) {
+		const jobId = job.id;
+		if (!jobId) {
+			this.emit("runner:error", /* @__PURE__ */ new Error("Job has no ID"));
+			return;
+		}
+		this.activeJobs.set(jobId, job);
+		this.emit("job:started", job);
+		await this.appendJobEvent(job, {
+			type: "status",
+			level: "info",
+			stage: "started",
+			progress: 0,
+			message: `Started job: ${job.getDescription()}`
+		});
+		let timeoutHandle = null;
+		try {
+			const result = await this.runWithTimeout(job, (handle) => {
+				timeoutHandle = handle;
+			});
+			const completedAt = /* @__PURE__ */ new Date();
+			if (await this.writeOwnedJob(jobId, {
+				status: "completed",
+				completed_at: completedAt.toISOString(),
+				result_pointer: result?.resultPointer ?? null,
+				updated_at: completedAt.toISOString()
+			})) {
+				job.status = "completed";
+				job.completedAt = completedAt;
+				job.resultPointer = result?.resultPointer ?? null;
+				await this.appendJobEvent(job, {
+					type: "progress",
+					level: "info",
+					stage: "completed",
+					progress: 100,
+					message: `Completed job: ${job.getDescription()}`
+				});
+				this.emit("job:completed", job, result);
+			}
+		} catch (error) {
+			try {
+				await this.handleJobError(job, error);
+			} catch (handlerError) {
+				this.emit("runner:error", handlerError);
+			}
+		} finally {
+			if (timeoutHandle) clearTimeout(timeoutHandle);
+			this.activeJobs.delete(jobId);
+		}
+	}
+	/**
+	* Execute a job honoring its {@link SmrtJob.timeoutBehavior}.
+	*
+	* - `'fail'` (default) and `'kill'`: race the handler against a timeout. On
+	*   timeout the race rejects and the caller fails/retries the job.
+	*   `'kill'` cannot actually preempt the handler in-process (JavaScript has
+	*   no thread interruption), so it is treated identically to `'fail'` — the
+	*   handler keeps running in the background; only the job row is failed. This
+	*   is documented in AGENTS.md so `'kill'` is honest about what it does
+	*   rather than silently behaving like a no-op.
+	* - `'warn'`: do NOT fail on timeout. Arm a one-shot warning (logged + emitted
+	*   as a job event) at the deadline, but await the handler to completion so a
+	*   slow-but-successful handler still completes. This makes `'warn'` honest:
+	*   previously every timeout was treated as `'fail'` regardless of the
+	*   persisted/UI-shown behavior.
+	*/
+	async runWithTimeout(job, captureHandle) {
+		if (job.timeoutBehavior === "warn") {
+			captureHandle(setTimeout(() => {
+				this.logger.warn(`Job exceeded timeout (${job.timeout}ms) but timeoutBehavior='warn'; letting it finish: ${job.getDescription()}`);
+				this.appendJobEvent(job, {
+					type: "status",
+					level: "warn",
+					stage: "timeout-warning",
+					message: `Job exceeded timeout of ${job.timeout}ms (timeoutBehavior='warn')`,
+					data: { timeout: job.timeout }
+				});
+			}, job.timeout));
+			return this.executeJob(job);
+		}
+		const timeoutPromise = new Promise((_, reject) => {
+			captureHandle(setTimeout(() => {
+				reject(new JobTimeoutError(`Job timeout after ${job.timeout}ms`));
+			}, job.timeout));
+		});
+		return Promise.race([this.executeJob(job), timeoutPromise]);
+	}
+	/**
+	* Apply a terminal/retry state transition to a job only if this worker still
+	* owns it and it is still `running`. Returns whether the write applied.
+	*
+	* This closes the completion-vs-recovery race: if recovery already failed a
+	* job out from under a finishing handler (a genuine zombie), the handler's
+	* outcome is dropped rather than resurrecting the row.
+	*/
+	async writeOwnedJob(jobId, assignments) {
+		if (!this.db) return false;
+		const columns = Object.keys(assignments);
+		const setSql = columns.map((column) => `${column} = ?`).join(", ");
+		const values = columns.map((column) => assignments[column]);
+		return ((await this.db.query(`UPDATE _smrt_jobs
+          SET ${setSql}
+        WHERE id = ? AND worker_id = ? AND status = 'running'
+        RETURNING id`, ...values, jobId, this.workerKey)).rows?.length ?? 0) > 0;
+	}
+	/**
+	* Execute a job by invoking the method on the SmrtObject
+	*/
+	async executeJob(job) {
+		const runJob = async () => {
+			const registeredClass = ObjectRegistry.getClass(job.objectType);
+			if (!registeredClass) throw new Error(`Unknown object type: ${job.objectType}`);
+			const ObjectClass = registeredClass.constructor;
+			const rawArgs = job.args ?? {};
+			const persistedAgentConfig = rawArgs._agentConfig ?? {};
+			const { _agentConfig: _, _scheduleId: __, ...methodArgs } = rawArgs;
+			const agentConfig = await resolveLazyConfig(persistedAgentConfig, {
+				classResolvers: getClassConfigResolvers(ObjectClass),
+				onError: "throw"
+			});
+			let instance;
+			if (job.objectId) {
+				instance = new ObjectClass({
+					db: this.db,
+					...agentConfig
+				});
+				await instance.initialize();
+				await instance.loadFromId(job.objectId);
+			} else {
+				instance = new ObjectClass({
+					db: this.db,
+					...agentConfig
+				});
+				await instance.initialize();
+			}
+			const jobId = job.id;
+			if (!jobId) throw new Error("Job has no ID");
+			const contextLogger = new JobContextLogger(createLogger(true), {
+				jobId,
+				attempt: job.attempts,
+				queue: job.queue,
+				objectType: job.objectType,
+				method: job.method
+			});
+			contextLogger.info(`Starting job: ${job.getDescription()}`);
+			const executionContext = this.createExecutionContext(job, contextLogger);
+			const method = instance[job.method];
+			if (typeof method !== "function") throw new Error(`Method not found: ${job.objectType}.${job.method}`);
+			if (!isBackgroundEligibleMethod(ObjectClass, job.method)) throw new Error(`Method not background-eligible: ${job.objectType}.${job.method}`);
+			return { result: await method.call(instance, methodArgs, executionContext) };
+		};
+		if (job.tenantId) return TenantContext.runWithJobContext({ tenantId: job.tenantId }, runJob);
+		return runJob();
+	}
+	/**
+	* Handle job execution error
+	*/
+	async handleJobError(job, error) {
+		const decision = fromConfig(job.retryStrategy).shouldRetry(job.attempts, error);
+		const jobId = job.id;
+		if (!jobId) return;
+		const isTimeout = error instanceof JobTimeoutError;
+		const safeMessage = redactErrorForPersistence(error);
+		if (!isTimeout && decision.shouldRetry && job.attempts < job.maxAttempts) {
+			const nextRunAt = new Date(Date.now() + decision.delay);
+			if (!await this.writeOwnedJob(jobId, {
+				status: "pending",
+				last_error: safeMessage,
+				run_at: nextRunAt.toISOString(),
+				worker_id: null,
+				worker_heartbeat: null,
+				updated_at: (/* @__PURE__ */ new Date()).toISOString()
+			})) return;
+			job.status = "pending";
+			job.lastError = safeMessage;
+			job.runAt = nextRunAt;
+			job.workerId = null;
+			job.workerHeartbeat = null;
+			await this.appendJobEvent(job, {
+				type: "status",
+				level: "warn",
+				stage: "retrying",
+				message: `Retrying job after failure: ${safeMessage}`,
+				data: {
+					delay: decision.delay,
+					attempts: job.attempts
+				}
+			});
+			this.emit("job:retrying", job, error, decision.delay);
+		} else {
+			const completedAt = /* @__PURE__ */ new Date();
+			if (!await this.writeOwnedJob(jobId, {
+				status: "failed",
+				completed_at: completedAt.toISOString(),
+				last_error: safeMessage,
+				updated_at: completedAt.toISOString()
+			})) return;
+			job.status = "failed";
+			job.completedAt = completedAt;
+			job.lastError = safeMessage;
+			await this.appendJobEvent(job, {
+				type: "error",
+				level: "error",
+				stage: "failed",
+				message: safeMessage,
+				data: { attempts: job.attempts }
+			});
+			this.emit("job:failed", job, error);
+		}
+	}
+	createExecutionContext(job, contextLogger) {
+		return {
+			job: {
+				jobId: job.id ?? "",
+				tenantId: job.tenantId ?? null,
+				attempt: job.attempts,
+				queue: job.queue,
+				objectType: job.objectType,
+				method: job.method
+			},
+			logger: contextLogger,
+			event: async (input) => {
+				await this.appendJobEvent(job, input);
+			},
+			progress: async (input) => {
+				const data = {
+					...input.data ?? {},
+					...input.detail ? { detail: input.detail } : {},
+					...input.source ? { source: input.source } : {}
+				};
+				await this.appendJobEvent(job, {
+					type: "progress",
+					level: "info",
+					stage: input.stage,
+					progress: input.progress,
+					message: input.message ?? input.detail ?? `${input.stage} ${Math.round(input.progress)}%`,
+					data
+				});
+			},
+			log: async (level, message, data) => {
+				contextLogger[level](message, data);
+				await this.appendJobEvent(job, {
+					type: level === "error" ? "error" : "log",
+					level,
+					message,
+					data
+				});
+			}
+		};
+	}
+	async appendJobEvent(job, input) {
+		if (!this.eventCollection || !job.id) return null;
+		try {
+			const event = await this.eventCollection.append({
+				tenantId: job.tenantId ?? null,
+				jobId: job.id,
+				type: input.type ?? "log",
+				level: input.level ?? "info",
+				stage: input.stage ?? null,
+				progress: input.progress ?? null,
+				message: input.message ?? "",
+				data: input.data ?? {}
+			});
+			this.emit("job:event", job, event);
+			if (event.type === "progress") this.emit("job:progress", job, event);
+			return event;
+		} catch (error) {
+			const telemetryError = error instanceof Error ? error : /* @__PURE__ */ new Error(`Failed to append job telemetry: ${String(error)}`);
+			try {
+				this.emit("runner:error", telemetryError);
+			} catch {}
+			return null;
+		}
+	}
+	/**
+	* Whether the `_smrt_workers` table exists. Cached once positive — the table
+	* never disappears mid-run, so this avoids a probe query on every poll.
+	*/
+	async workersTableReady() {
+		if (this.workersTableVerified) return true;
+		const ready = await this.workerCollection?.tableReady() ?? false;
+		if (ready) this.workersTableVerified = true;
+		return ready;
+	}
+	/**
+	* Recover jobs orphaned by dead/restarted workers.
+	*
+	* A `running` job is recovered only when its owning worker is *not alive*
+	* (issue #1474): not live in this process and holding no fresh lease in
+	* `_smrt_workers`. This is independent of the handler event loop, so a worker
+	* whose handler holds the loop synchronously keeps a fresh lease (renewed off
+	* the loop by the liveness thread) or stays in this process's live set, and is
+	* never false-recovered. The live set takes precedence over a stale lease, and
+	* a runner never recovers its own active jobs.
+	*
+	* Recovery is swept at most once per lease tick (not every poll), since
+	* detection is TTL-bound anyway — this bounds the per-poll database load.
+	*/
+	async recoverStaleJobs() {
+		if (!this.db || !this.collection || !this.workerCollection) return;
+		if (!await this.workersTableReady()) return;
+		const now = Date.now();
+		if (now - this.lastRecoverySweepAt < this.config.leaseTickMs) return;
+		this.lastRecoverySweepAt = now;
+		try {
+			await this.workerCollection.pruneExpired(this.effectiveLeaseTtlMs * 10);
+		} catch {}
+		const freshLeaseKeys = await this.workerCollection.freshLeaseWorkerKeys();
+		const running = await this.collection.query(`SELECT * FROM _smrt_jobs WHERE status = 'running'`, [], { allowRawOnTenantScoped: true });
+		if (running.length === 0) return;
+		const orphans = running.filter((job) => {
+			const jobId = job.id;
+			if (jobId && this.activeJobs.has(jobId)) return false;
+			return !isWorkerAlive(job.workerId, freshLeaseKeys);
+		});
+		if (orphans.length === 0) return;
+		const orphanIds = orphans.map((job) => job.id).filter((jobId) => typeof jobId === "string");
+		if (orphanIds.length === 0) return;
+		const placeholders = orphanIds.map(() => "?").join(", ");
+		const recoveredAt = /* @__PURE__ */ new Date();
+		const errorMessage = "Recovered orphaned running job: its owning worker is no longer alive (no fresh liveness lease in _smrt_workers and not running in this process).";
+		const updated = await this.db.query(`UPDATE _smrt_jobs
+          SET status = 'failed',
+              completed_at = ?,
+              last_error = ?,
+              worker_id = NULL,
+              worker_heartbeat = NULL
+        WHERE status = 'running'
+          AND id IN (${placeholders})
+        RETURNING id`, recoveredAt.toISOString(), errorMessage, ...orphanIds);
+		const recoveredIds = new Set(updated.rows.map((row) => row.id).filter((id) => typeof id === "string"));
+		if (recoveredIds.size === 0) return;
+		for (const job of orphans) {
+			if (!job.id || !recoveredIds.has(job.id)) continue;
+			job.status = "failed";
+			job.completedAt = recoveredAt;
+			job.lastError = errorMessage;
+			job.workerId = null;
+			job.workerHeartbeat = null;
+			const error = /* @__PURE__ */ new Error(errorMessage);
+			await this.appendJobEvent(job, {
+				type: "error",
+				level: "error",
+				stage: "stale-recovery",
+				message: errorMessage
+			});
+			this.emit("job:failed", job, error);
+		}
+	}
+	/**
+	* Renew this worker's liveness lease.
+	*
+	* In Stage 1 this runs on the main event loop, so it provides cross-process
+	* detection no weaker than the old per-job heartbeat. Stage 2 moves the
+	* renewal to an off-loop worker thread so a synchronous handler can no longer
+	* starve it. Same-process correctness never depends on this timer — the
+	* in-memory live set covers it.
+	*/
+	startLeaseRenewal() {
+		this.leaseTimer = setInterval(async () => {
+			try {
+				await this.workerCollection?.renewLease(this.workerKey, this.effectiveLeaseTtlMs);
+			} catch {}
+		}, this.config.leaseTickMs);
+	}
+	/**
+	* Spawn the off-loop liveness thread. It opens its own connection and renews
+	* this worker's lease on its own thread (unstarvable by handler CPU). Returns
+	* false if the thread can't be resolved or fails to start, so the caller can
+	* fall back to main-loop renewal.
+	*/
+	async startLivenessThread() {
+		if (!this.db) return false;
+		let entry;
+		try {
+			entry = import.meta.resolve("@happyvertical/smrt-jobs/worker-liveness-thread");
+		} catch {
+			return false;
+		}
+		let worker;
+		try {
+			worker = new Worker(new URL(entry), { workerData: {
+				url: resolveUrl(this.db),
+				type: resolveEngine(this.db),
+				workerKey: this.workerKey,
+				leaseTtlMs: this.effectiveLeaseTtlMs,
+				leaseTickMs: this.config.leaseTickMs
+			} });
+		} catch {
+			return false;
+		}
+		if (!await new Promise((resolve) => {
+			const onMessage = (message) => {
+				if (message === "ready") {
+					cleanup();
+					resolve(true);
+				} else if (message && typeof message === "object" && message.type === "error") {
+					cleanup();
+					resolve(false);
+				}
+			};
+			const onFail = () => {
+				cleanup();
+				resolve(false);
+			};
+			const cleanup = () => {
+				clearTimeout(timer);
+				worker.off("message", onMessage);
+				worker.off("error", onFail);
+				worker.off("exit", onFail);
+			};
+			const timer = setTimeout(() => {
+				cleanup();
+				resolve(false);
+			}, LIVENESS_THREAD_START_TIMEOUT_MS);
+			if (typeof timer.unref === "function") timer.unref();
+			worker.on("message", onMessage);
+			worker.once("error", onFail);
+			worker.once("exit", onFail);
+		})) {
+			await worker.terminate().catch(() => {});
+			return false;
+		}
+		this.livenessWorker = worker;
+		worker.unref();
+		worker.once("error", () => this.handleLivenessThreadLoss(worker));
+		worker.once("exit", () => this.handleLivenessThreadLoss(worker));
+		return true;
+	}
+	handleLivenessThreadLoss(worker) {
+		if (this.livenessWorker !== worker) return;
+		this.livenessWorker = null;
+		if (this.running && !this.leaseTimer) this.startLeaseRenewal();
+	}
+	/** Stop the liveness thread (graceful, with a short bound), if running. */
+	async stopLivenessThread() {
+		const worker = this.livenessWorker;
+		if (!worker) return;
+		this.livenessWorker = null;
+		const stopped = new Promise((resolve) => {
+			const done = () => {
+				clearTimeout(timer);
+				worker.off("message", onMessage);
+				resolve();
+			};
+			const onMessage = (message) => {
+				if (message === "stopped") done();
+			};
+			worker.on("message", onMessage);
+			worker.once("exit", done);
+			const timer = setTimeout(done, 2e3);
+			if (typeof timer.unref === "function") timer.unref();
+		});
+		try {
+			worker.postMessage("stop");
+		} catch {}
+		await stopped;
+		await worker.terminate().catch(() => {});
+	}
+	/**
+	* Per-job heartbeat loop — telemetry only ("last activity" for the UI). It no
+	* longer gates recovery (that is the worker lease), so a blocked loop missing
+	* a heartbeat is harmless.
+	*/
+	startHeartbeat() {
+		this.heartbeatTimer = setInterval(async () => {
+			if (!this.db) return;
+			const jobIds = [...this.activeJobs.keys()];
+			if (jobIds.length === 0) return;
+			const placeholders = jobIds.map(() => "?").join(", ");
+			try {
+				await this.db.query(`UPDATE _smrt_jobs
+              SET worker_heartbeat = ?
+            WHERE status = 'running'
+              AND id IN (${placeholders})`, (/* @__PURE__ */ new Date()).toISOString(), ...jobIds);
+			} catch {}
+		}, this.config.heartbeatInterval);
+	}
+	/**
+	* Wait for active jobs to complete with timeout
+	*/
+	async waitForActiveJobs() {
+		if (this.activeJobs.size === 0) return;
+		return new Promise((resolve) => {
+			const checkInterval = setInterval(() => {
+				if (this.activeJobs.size === 0) {
+					clearInterval(checkInterval);
+					clearTimeout(timeout);
+					resolve();
+				}
+			}, 100);
+			const timeout = setTimeout(() => {
+				clearInterval(checkInterval);
+				this.logger.warn(`Shutdown timeout: ${this.activeJobs.size} jobs still active`);
+				resolve();
+			}, this.config.shutdownTimeout);
+		});
+	}
+};
+function createTaskRunner(config) {
+	return new TaskRunner(config);
+}
+//#endregion
+export { markBackgroundEligible as S, assertWithinTenantCreationCap as _, SmrtWorker as a, getBackgroundEligibleMethods as b, SmrtJobEventCollection as c, JobContextLogger as d, redactErrorForPersistence as f, TenantJobCapExceededError as g, MAX_JOB_RETRIES as h, DEFAULT_TASK_HEARTBEAT_INTERVAL_MS as i, SmrtJob as l, DEFAULT_TENANT_JOB_CAP as m, TaskRunner as n, SmrtWorkerCollection as o, redactErrorMessage as p, createTaskRunner as r, SmrtJobEvent as s, JobTimeoutError as t, SmrtJobCollection as u, backgroundEligible as v, isBackgroundEligibleMethod as x, clampRetries as y };
+
+//# sourceMappingURL=runner-DLsWfeY-.js.map