@happyvertical/smrt-jobs 0.30.0

package/dist/chunks/runner-DV8FBO0y.js

@@ -0,0 +1,1642 @@
+import { ObjectRegistry, field, smrt, SmrtObject, SmrtCollection, ensureJobsSystemTableCompatibility, detectEngine, foreignKey, ensureJobEventsSystemTableCompatibility, getClassConfigResolvers, resolveLazyConfig } from "@happyvertical/smrt-core";
+import { EventEmitter } from "node:events";
+import { Worker } from "node:worker_threads";
+import { fromConfig } from "@happyvertical/jobs";
+import { createLogger } from "@happyvertical/logger";
+import { tenantId, TenantScoped, getTenantId, TenantContext } from "@happyvertical/smrt-tenancy";
+import { createId } from "@happyvertical/utils";
+import { c as createWorkerKey, a as resolveEngine, t as tuneSqliteForConcurrency, r as registerLiveWorker, o as offLoopEligible, u as unregisterLiveWorker, i as isWorkerAlive, b as resolveUrl } from "./worker-liveness-DOTjoIjr.js";
+ObjectRegistry.registerPackageManifest(
+  new URL("./manifest.json", import.meta.url)
+);
+const MAX_JOB_RETRIES = 25;
+const DEFAULT_TENANT_JOB_CAP = 1e4;
+function clampRetries(requested) {
+  if (Number.isNaN(requested) || requested < 0) {
+    return 0;
+  }
+  if (requested === Number.POSITIVE_INFINITY) {
+    return MAX_JOB_RETRIES;
+  }
+  return Math.min(Math.floor(requested), MAX_JOB_RETRIES);
+}
+class TenantJobCapExceededError extends Error {
+  constructor(tenantId2, cap, current) {
+    super(
+      `Tenant "${tenantId2}" has reached its background-job cap (${current}/${cap} in-flight). Refusing to enqueue another job.`
+    );
+    this.tenantId = tenantId2;
+    this.cap = cap;
+    this.current = current;
+    this.name = "TenantJobCapExceededError";
+  }
+  tenantId;
+  cap;
+  current;
+}
+function assertWithinTenantCreationCap(tenantId2, current, cap) {
+  if (!tenantId2 || cap <= 0) return;
+  if (current >= cap) {
+    throw new TenantJobCapExceededError(tenantId2, cap, current);
+  }
+}
+function markBackgroundEligible(ctor, ...methods) {
+  const target = ctor;
+  const existing = target.backgroundEligibleMethods;
+  const set = existing instanceof Set ? new Set(existing) : new Set(existing ?? []);
+  for (const method of methods) set.add(method);
+  target.backgroundEligibleMethods = set;
+}
+function backgroundEligible() {
+  return (target, propertyKey, descriptor) => {
+    const ctor = target.constructor;
+    markBackgroundEligible(ctor, String(propertyKey));
+    return descriptor;
+  };
+}
+function getBackgroundEligibleMethods(ctor) {
+  const declared = ctor?.backgroundEligibleMethods;
+  if (declared == null) return null;
+  return declared instanceof Set ? declared : new Set(declared);
+}
+function isBackgroundEligibleMethod(ctor, method) {
+  const allow = getBackgroundEligibleMethods(ctor);
+  if (allow == null) return true;
+  return allow.has(method);
+}
+const REDACTED = "***REDACTED***";
+const CREDENTIAL_URL_RE = /([a-z][a-z0-9+.-]*:\/\/)[^/?#@\s]+@/gi;
+const BEARER_TOKEN_RE = /\b(bearer|token)\s+[A-Za-z0-9._\-+/=]{8,}/gi;
+const AUTHORIZATION_HEADER_RE = /\bauthorization\s*[:=]\s*(?:[A-Za-z]+\s+)?[^\s,;)]+/gi;
+const SECRET_KEY_VALUE_RE = /\b([A-Za-z0-9_-]*(?:password|passwd|pwd|secret|api[_-]?key|access[_-]?key|secret[_-]?key|private[_-]?key|client[_-]?secret|token|credential|auth)[A-Za-z0-9_-]*)\s*([:=])\s*("[^"]*"|'[^']*'|[^\s,;)]+)/gi;
+const JSON_SECRET_KEY_VALUE_RE = /("(?:[A-Za-z0-9_-]*(?:password|passwd|pwd|secret|api[_-]?key|access[_-]?key|secret[_-]?key|private[_-]?key|client[_-]?secret|token|credential|auth)[A-Za-z0-9_-]*)")\s*:\s*"[^"]*"/gi;
+const STANDALONE_SECRET_RE = /\b(sk-[A-Za-z0-9_-]{14,}[A-Za-z0-9]|sk_(?:live|test)_[A-Za-z0-9_-]{14,}[A-Za-z0-9]|AKIA[0-9A-Z]{12,}|gh[pousr]_[A-Za-z0-9]{20,}|AIza[A-Za-z0-9_-]{20,}|xox[bp]-[A-Za-z0-9-]{10,})\b/g;
+function redactErrorMessage(message) {
+  if (typeof message !== "string" || message.length === 0) {
+    return message;
+  }
+  return message.replace(CREDENTIAL_URL_RE, `$1${REDACTED}@`).replace(AUTHORIZATION_HEADER_RE, `authorization=${REDACTED}`).replace(BEARER_TOKEN_RE, `$1 ${REDACTED}`).replace(JSON_SECRET_KEY_VALUE_RE, `$1:"${REDACTED}"`).replace(SECRET_KEY_VALUE_RE, `$1$2${REDACTED}`).replace(STANDALONE_SECRET_RE, REDACTED);
+}
+function redactErrorForPersistence(error) {
+  if (error instanceof Error) {
+    return redactErrorMessage(error.message);
+  }
+  return redactErrorMessage(String(error));
+}
+class JobContextLogger {
+  constructor(baseLogger, jobContext) {
+    this.baseLogger = baseLogger;
+    this.jobContext = jobContext;
+  }
+  baseLogger;
+  jobContext;
+  addContext(data) {
+    return {
+      ...data,
+      _job: {
+        id: this.jobContext.jobId,
+        attempt: this.jobContext.attempt,
+        queue: this.jobContext.queue,
+        objectType: this.jobContext.objectType,
+        method: this.jobContext.method
+      }
+    };
+  }
+  debug(message, data) {
+    this.baseLogger.debug(message, this.addContext(data));
+  }
+  info(message, data) {
+    this.baseLogger.info(message, this.addContext(data));
+  }
+  warn(message, data) {
+    this.baseLogger.warn(message, this.addContext(data));
+  }
+  error(message, data) {
+    this.baseLogger.error(message, this.addContext(data));
+  }
+}
+var __defProp$2 = Object.defineProperty;
+var __getOwnPropDesc$2 = Object.getOwnPropertyDescriptor;
+var __decorateClass$2 = (decorators, target, key, kind) => {
+  var result = kind > 1 ? void 0 : kind ? __getOwnPropDesc$2(target, key) : target;
+  for (var i = decorators.length - 1, decorator; i >= 0; i--)
+    if (decorator = decorators[i])
+      result = (kind ? decorator(target, key, result) : decorator(result)) || result;
+  if (kind && result) __defProp$2(target, key, result);
+  return result;
+};
+let SmrtJob = class extends SmrtObject {
+  tenantId = void 0;
+  queue = "default";
+  objectType = "";
+  objectId = null;
+  method = "";
+  args = {};
+  runAt = /* @__PURE__ */ new Date();
+  priority = 50;
+  status = "pending";
+  attempts = 0;
+  maxAttempts = 3;
+  timeout = 3e5;
+  timeoutBehavior = "fail";
+  startedAt = null;
+  completedAt = null;
+  lastError = null;
+  resultPointer = null;
+  retryStrategy = {
+    type: "exponential",
+    config: { initialDelay: 1e3, multiplier: 2, maxDelay: 3e5 }
+  };
+  workerId = null;
+  workerHeartbeat = null;
+  /**
+   * Capture ambient tenant context when a job is saved inside withTenant().
+   *
+   * Scheduled jobs can also set this explicitly from their owning schedule.
+   */
+  async save() {
+    if (this.tenantId === void 0) {
+      const contextTenantId = getTenantId();
+      if (contextTenantId) {
+        this.tenantId = contextTenantId;
+      }
+    }
+    return super.save();
+  }
+  /**
+   * Mark the job for retry
+   */
+  async retry() {
+    if (this.status === "completed") {
+      throw new Error("Cannot retry a completed job");
+    }
+    this.status = "pending";
+    this.attempts = 0;
+    this.lastError = null;
+    this.startedAt = null;
+    this.completedAt = null;
+    this.workerId = null;
+    this.workerHeartbeat = null;
+    await this.save();
+  }
+  /**
+   * Cancel the job
+   */
+  async cancel() {
+    if (this.status === "completed" || this.status === "cancelled") {
+      throw new Error(`Cannot cancel job with status: ${this.status}`);
+    }
+    this.status = "cancelled";
+    this.completedAt = /* @__PURE__ */ new Date();
+    await this.save();
+  }
+  /**
+   * Get a human-readable description of the job
+   */
+  getDescription() {
+    const target = this.objectId ? `${this.objectType}#${this.objectId}` : this.objectType;
+    return `${target}.${this.method}()`;
+  }
+};
+__decorateClass$2([
+  tenantId({ nullable: true })
+], SmrtJob.prototype, "tenantId", 2);
+__decorateClass$2([
+  field({ type: "text", required: true, default: "default" })
+], SmrtJob.prototype, "queue", 2);
+__decorateClass$2([
+  field({ type: "text", required: true })
+], SmrtJob.prototype, "objectType", 2);
+__decorateClass$2([
+  field({ type: "text", nullable: true })
+], SmrtJob.prototype, "objectId", 2);
+__decorateClass$2([
+  field({ type: "text", required: true })
+], SmrtJob.prototype, "method", 2);
+__decorateClass$2([
+  field({ type: "json" })
+], SmrtJob.prototype, "args", 2);
+__decorateClass$2([
+  field({ type: "datetime", required: true })
+], SmrtJob.prototype, "runAt", 2);
+__decorateClass$2([
+  field({ type: "integer", required: true, default: 50 })
+], SmrtJob.prototype, "priority", 2);
+__decorateClass$2([
+  field({ type: "text", required: true, default: "pending" })
+], SmrtJob.prototype, "status", 2);
+__decorateClass$2([
+  field({ type: "integer", required: true, default: 0 })
+], SmrtJob.prototype, "attempts", 2);
+__decorateClass$2([
+  field({ type: "integer", required: true, default: 3 })
+], SmrtJob.prototype, "maxAttempts", 2);
+__decorateClass$2([
+  field({ type: "integer", required: true, default: 3e5 })
+], SmrtJob.prototype, "timeout", 2);
+__decorateClass$2([
+  field({ type: "text", required: true, default: "fail" })
+], SmrtJob.prototype, "timeoutBehavior", 2);
+__decorateClass$2([
+  field({ type: "datetime", nullable: true })
+], SmrtJob.prototype, "startedAt", 2);
+__decorateClass$2([
+  field({ type: "datetime", nullable: true })
+], SmrtJob.prototype, "completedAt", 2);
+__decorateClass$2([
+  field({ type: "text", nullable: true })
+], SmrtJob.prototype, "lastError", 2);
+__decorateClass$2([
+  field({ type: "text", nullable: true })
+], SmrtJob.prototype, "resultPointer", 2);
+__decorateClass$2([
+  field({ type: "json" })
+], SmrtJob.prototype, "retryStrategy", 2);
+__decorateClass$2([
+  field({ type: "text", nullable: true })
+], SmrtJob.prototype, "workerId", 2);
+__decorateClass$2([
+  field({ type: "datetime", nullable: true })
+], SmrtJob.prototype, "workerHeartbeat", 2);
+SmrtJob = __decorateClass$2([
+  smrt({
+    tableName: "_smrt_jobs",
+    // Fail closed: `_smrt_jobs` is an internal operational queue table. Generated
+    // REST/MCP list/get only filter by tenant when a tenant context is active;
+    // reached without context (a tenant-less/admin principal, or any non-SvelteKit
+    // surface) an `optional`-mode class returns UNFILTERED rows, leaking every
+    // tenant's jobs. Workers read this table via the collection directly
+    // (allowRawOnTenantScoped), never through generated routes, so nothing
+    // internal needs the read surface — so we do not generate one (S5 audit #1402).
+    api: false,
+    // retry/cancel are operator commands invoked in-process via the CLI;
+    // they intentionally aren't exposed over HTTP.
+    cli: {
+      include: ["list", "get", "retry", "cancel"],
+      skipApiCheck: true,
+      http: false
+    },
+    mcp: false
+  }),
+  TenantScoped({ mode: "optional" })
+], SmrtJob);
+class SmrtJobCollection extends SmrtCollection {
+  static _itemClass = SmrtJob;
+  async initialize() {
+    await super.initialize();
+    await ensureJobsSystemTableCompatibility(this.db);
+    return this;
+  }
+  /**
+   * List jobs by status
+   */
+  async listByStatus(status, options = {}) {
+    const where = {
+      status: Array.isArray(status) ? status : [status]
+    };
+    if (options.queue) {
+      where.queue = options.queue;
+    }
+    return this.list({
+      where,
+      orderBy: ["priority DESC", "run_at ASC"],
+      limit: options.limit
+    });
+  }
+  /**
+   * List pending jobs ready to run
+   */
+  async listReady(options = {}) {
+    const now = (/* @__PURE__ */ new Date()).toISOString();
+    const whereConditions = ["status = 'pending'", "run_at <= ?"];
+    const params = [now];
+    if (options.queues?.length) {
+      const placeholders = options.queues.map(() => "?").join(", ");
+      whereConditions.push(`queue IN (${placeholders})`);
+      params.push(...options.queues);
+    }
+    params.push(options.limit || 100);
+    return this.query(
+      `SELECT * FROM _smrt_jobs WHERE ${whereConditions.join(" AND ")} ORDER BY priority DESC, run_at ASC LIMIT ?`,
+      params,
+      { allowRawOnTenantScoped: true }
+    );
+  }
+  /**
+   * Atomically claim pending jobs ready to run for a worker.
+   *
+   * The claim is performed as one conditional UPDATE so concurrent workers
+   * cannot receive the same pending row. PostgreSQL additionally skips rows
+   * locked by other workers instead of waiting behind them.
+   */
+  async claimReady(options) {
+    const limit = options.limit ?? 100;
+    if (limit <= 0) return [];
+    const now = options.now ?? /* @__PURE__ */ new Date();
+    const nowIso = now.toISOString();
+    const whereConditions = ["status = 'pending'", "run_at <= ?"];
+    const whereParams = [nowIso];
+    if (options.queues?.length) {
+      const placeholders = options.queues.map(() => "?").join(", ");
+      whereConditions.push(`queue IN (${placeholders})`);
+      whereParams.push(...options.queues);
+    }
+    const lockClause = getDatabaseEngine(this.db) === "postgres" ? " FOR UPDATE SKIP LOCKED" : "";
+    const candidateSelect = `
+      SELECT id
+        FROM _smrt_jobs
+       WHERE ${whereConditions.join(" AND ")}
+       ORDER BY priority DESC, run_at ASC, created_at ASC, id ASC
+       LIMIT ?${lockClause}
+    `;
+    const claimed = await this.query(
+      `UPDATE _smrt_jobs
+          SET status = 'running',
+              worker_id = ?,
+              worker_heartbeat = ?,
+              started_at = ?,
+              attempts = attempts + 1,
+              updated_at = ?
+        WHERE id IN (${candidateSelect})
+          AND status = 'pending'
+        RETURNING *`,
+      [options.workerId, nowIso, nowIso, nowIso, ...whereParams, limit],
+      // Worker-internal cross-tenant claim; tenant context is restored
+      // per-job at execution (SmrtJob is now @TenantScoped, S5 #1402).
+      { allowRawOnTenantScoped: true }
+    );
+    return claimed.toSorted(compareClaimOrder);
+  }
+  /**
+   * Count non-terminal (pending/running) jobs owned by a tenant.
+   *
+   * Used to enforce the per-tenant creation cap so one tenant cannot exhaust
+   * the shared worker pool (S5 audit #1402). Reads `_smrt_jobs` directly so it
+   * works regardless of ambient tenant context.
+   *
+   * @param tenantId - Tenant to count for. `null` counts global (NULL-tenant)
+   *   jobs.
+   */
+  async countInFlightForTenant(tenantId2) {
+    const predicate = tenantId2 === null ? "tenant_id IS NULL" : "tenant_id = ?";
+    const params = tenantId2 === null ? [] : [tenantId2];
+    const result = await this.db.query(
+      `SELECT COUNT(*) AS count
+         FROM _smrt_jobs
+        WHERE status IN ('pending', 'running')
+          AND ${predicate}`,
+      ...params
+    );
+    const row = result.rows[0];
+    return Number(row?.count ?? 0);
+  }
+  /**
+   * The single creation path for queued jobs.
+   *
+   * Centralizes the two creation-time security guards from the S5 audit (#1402)
+   * so every enqueue — the fluent {@link "./job-builder".JobBuilder} *and* the
+   * ScheduleRunner's cron-triggered jobs — goes through one place:
+   *
+   * 1. `maxAttempts` is clamped to {@link MAX_JOB_RETRIES} so a misconfigured
+   *    caller cannot pin a worker on a poison job indefinitely.
+   * 2. A per-tenant in-flight cap bounds how many non-terminal jobs one tenant
+   *    may hold, so one tenant cannot exhaust the shared worker pool
+   *    (cross-tenant denial of service). The cap applies to the row's effective
+   *    tenant (explicit `data.tenantId` or, when absent, the ambient context);
+   *    global (null-tenant) jobs are exempt.
+   *
+   * Atomicity note (best-effort soft cap, by design): the cap is a
+   * count-then-insert, NOT a hard transactional invariant. It is intentionally
+   * left non-atomic. A plain transaction would not help — under the adapters'
+   * default isolation two concurrent same-tenant enqueues would each read the
+   * same COUNT and both insert, so serializing them would require either a
+   * per-tenant lock row (`SELECT ... FOR UPDATE`) or SERIALIZABLE-isolation
+   * retry loops. That cross-process locking is fragile (lock-row contention,
+   * adapter-specific isolation behavior, the `transaction` adapter method being
+   * optional) and out of proportion to the threat: this cap is defense in depth
+   * against runaway/accidental creation exhausting the shared worker pool, not a
+   * billing/quota boundary. So under truly simultaneous enqueues a tenant may
+   * momentarily overshoot by the number of in-flight creators; the bound still
+   * prevents unbounded growth and closes the prior ScheduleRunner bypass. If a
+   * hard guarantee is ever needed, enforce it with a DB CHECK/trigger or a
+   * dedicated counter row, not an application-level lock.
+   */
+  async enqueueJob(data, options = {}) {
+    const cap = options.tenantJobCap ?? DEFAULT_TENANT_JOB_CAP;
+    const explicitTenant = typeof data.tenantId === "string" && data.tenantId.length > 0 ? data.tenantId : data.tenantId === null ? null : void 0;
+    const effectiveTenant = explicitTenant !== void 0 ? explicitTenant : getTenantId() ?? null;
+    if (effectiveTenant && cap > 0) {
+      const current = await this.countInFlightForTenant(effectiveTenant);
+      assertWithinTenantCreationCap(effectiveTenant, current, cap);
+    }
+    const job = await this.create({
+      ...data,
+      // Clamp here so neither the builder nor the schedule runner can bypass the
+      // retry ceiling (S5 audit #1402).
+      maxAttempts: clampRetries(data.maxAttempts ?? 3)
+    });
+    await job.save();
+    return job;
+  }
+  /**
+   * Get job statistics
+   */
+  async stats(queue) {
+    const query = queue ? "SELECT status, COUNT(*) as count FROM _smrt_jobs WHERE queue = ? GROUP BY status" : "SELECT status, COUNT(*) as count FROM _smrt_jobs GROUP BY status";
+    const params = queue ? [queue] : [];
+    const result = await this._db.query(query, ...params);
+    const counts = {};
+    for (const row of result.rows) {
+      counts[row.status] = row.count;
+    }
+    return {
+      pending: counts.pending ?? 0,
+      running: counts.running ?? 0,
+      completed: counts.completed ?? 0,
+      failed: counts.failed ?? 0,
+      cancelled: counts.cancelled ?? 0
+    };
+  }
+  /**
+   * Cleanup old completed/failed jobs
+   */
+  async cleanup(options) {
+    const conditions = [];
+    const params = [];
+    if (options.completedBefore) {
+      conditions.push("(status = 'completed' AND completed_at < ?)");
+      params.push(options.completedBefore.toISOString());
+    }
+    if (options.failedBefore) {
+      conditions.push("(status = 'failed' AND completed_at < ?)");
+      params.push(options.failedBefore.toISOString());
+    }
+    if (options.cancelledBefore) {
+      conditions.push("(status = 'cancelled' AND completed_at < ?)");
+      params.push(options.cancelledBefore.toISOString());
+    }
+    if (conditions.length === 0) return 0;
+    let query = `DELETE FROM _smrt_jobs WHERE (${conditions.join(" OR ")})`;
+    if (options.limit) {
+      query = `
+        DELETE FROM _smrt_jobs
+        WHERE id IN (
+          SELECT id FROM _smrt_jobs
+          WHERE (${conditions.join(" OR ")})
+          LIMIT ?
+        )
+      `;
+      params.push(options.limit);
+    }
+    const result = await this._db.query(query, ...params);
+    return result.rowCount ?? 0;
+  }
+}
+function getDatabaseEngine(db) {
+  const dbWithConfig = db;
+  return detectEngine(
+    db.url || dbWithConfig.config?.url || "",
+    dbWithConfig.type || dbWithConfig.config?.type
+  );
+}
+function compareClaimOrder(left, right) {
+  const priority = right.priority - left.priority;
+  if (priority !== 0) return priority;
+  const runAt = left.runAt.getTime() - right.runAt.getTime();
+  if (runAt !== 0) return runAt;
+  const createdAt = timestamp(left.created_at) - timestamp(right.created_at);
+  if (createdAt !== 0) return createdAt;
+  return (left.id ?? "").localeCompare(right.id ?? "");
+}
+function timestamp(value) {
+  return value?.getTime() ?? 0;
+}
+var __defProp$1 = Object.defineProperty;
+var __getOwnPropDesc$1 = Object.getOwnPropertyDescriptor;
+var __decorateClass$1 = (decorators, target, key, kind) => {
+  var result = kind > 1 ? void 0 : kind ? __getOwnPropDesc$1(target, key) : target;
+  for (var i = decorators.length - 1, decorator; i >= 0; i--)
+    if (decorator = decorators[i])
+      result = (kind ? decorator(target, key, result) : decorator(result)) || result;
+  if (kind && result) __defProp$1(target, key, result);
+  return result;
+};
+const JOB_EVENT_STORAGE_COLUMNS = [
+  "id",
+  "slug",
+  "context",
+  "created_at",
+  "updated_at",
+  "tenant_id",
+  "job_id",
+  "type",
+  "level",
+  "stage",
+  "progress",
+  "message",
+  "data"
+].join(", ");
+let SmrtJobEvent = class extends SmrtObject {
+  tenantId = void 0;
+  jobId = "";
+  type = "log";
+  level = "info";
+  stage = null;
+  progress = null;
+  message = "";
+  data = {};
+  createdAt = /* @__PURE__ */ new Date();
+  toCursor() {
+    const createdAt = this.createdAt instanceof Date ? this.createdAt.toISOString() : String(this.createdAt);
+    return `${createdAt}|${this.id ?? ""}`;
+  }
+};
+__decorateClass$1([
+  tenantId({ nullable: true })
+], SmrtJobEvent.prototype, "tenantId", 2);
+__decorateClass$1([
+  foreignKey("SmrtJob", { required: true })
+], SmrtJobEvent.prototype, "jobId", 2);
+__decorateClass$1([
+  field({ type: "text", required: true, default: "log" })
+], SmrtJobEvent.prototype, "type", 2);
+__decorateClass$1([
+  field({ type: "text", required: true, default: "info" })
+], SmrtJobEvent.prototype, "level", 2);
+__decorateClass$1([
+  field({ type: "text", nullable: true })
+], SmrtJobEvent.prototype, "stage", 2);
+__decorateClass$1([
+  field({ type: "integer", nullable: true })
+], SmrtJobEvent.prototype, "progress", 2);
+__decorateClass$1([
+  field({ type: "text", required: true, default: "" })
+], SmrtJobEvent.prototype, "message", 2);
+__decorateClass$1([
+  field({ type: "json" })
+], SmrtJobEvent.prototype, "data", 2);
+__decorateClass$1([
+  field({ type: "datetime", required: true })
+], SmrtJobEvent.prototype, "createdAt", 2);
+SmrtJobEvent = __decorateClass$1([
+  smrt({
+    tableName: "_smrt_job_events",
+    // Fail closed: same reasoning as SmrtJob. `_smrt_job_events` carries job
+    // progress/log/error payloads for every tenant; an `optional`-mode generated
+    // read reached without tenant context returns UNFILTERED rows. Consumers read
+    // events through the collection's tenant-aware methods (listByJob /
+    // listSinceCursor, which require an explicit tenantId or ambient context), not
+    // through generated routes — so we do not generate a read surface here
+    // (S5 audit #1402).
+    api: false,
+    // In-process operator commands only (http: false). skipApiCheck acknowledges
+    // that these CLI reads intentionally have no HTTP/API route now that api is
+    // disabled (S5 audit #1402).
+    cli: { include: ["list", "get"], http: false, skipApiCheck: true },
+    mcp: false
+  }),
+  TenantScoped({ mode: "optional" })
+], SmrtJobEvent);
+function normalizeLimit(limit) {
+  const numeric = typeof limit === "number" && Number.isFinite(limit) ? limit : 250;
+  return Math.max(1, Math.min(1e3, Math.floor(numeric)));
+}
+function normalizeProgress(progress) {
+  if (typeof progress !== "number" || !Number.isFinite(progress)) {
+    return null;
+  }
+  return Math.max(0, Math.min(100, Math.round(progress)));
+}
+function parseCursor(cursor) {
+  if (typeof cursor !== "string") return cursor;
+  const separator = cursor.lastIndexOf("|");
+  if (separator === -1) {
+    return { createdAt: cursor, id: "" };
+  }
+  return {
+    createdAt: cursor.slice(0, separator),
+    id: cursor.slice(separator + 1)
+  };
+}
+function normalizeCursorDate(value) {
+  if (value instanceof Date) {
+    return value.toISOString();
+  }
+  const parsed = new Date(value);
+  if (!Number.isNaN(parsed.getTime())) {
+    return parsed.toISOString();
+  }
+  return value;
+}
+function usesSqliteDateFunctions(dbUrl) {
+  const normalized = dbUrl.toLowerCase();
+  return !(normalized.startsWith("postgres:") || normalized.startsWith("postgresql:"));
+}
+function getQueryRows(result) {
+  if (Array.isArray(result)) {
+    return result;
+  }
+  return result.rows ?? [];
+}
+class SmrtJobEventCollection extends SmrtCollection {
+  static _itemClass = SmrtJobEvent;
+  async initialize() {
+    await super.initialize();
+    await ensureJobEventsSystemTableCompatibility(this.db);
+    return this;
+  }
+  async append(input) {
+    return this.create({
+      tenantId: input.tenantId,
+      jobId: input.jobId,
+      type: input.type ?? "log",
+      level: input.level ?? "info",
+      stage: input.stage ?? null,
+      progress: normalizeProgress(input.progress),
+      message: input.message ?? "",
+      data: input.data ?? {},
+      createdAt: input.createdAt ?? /* @__PURE__ */ new Date()
+    });
+  }
+  async listByJob(jobId, options = {}) {
+    return this.listSinceCursor({
+      ...options,
+      jobId
+    });
+  }
+  async listSinceCursor(options = {}) {
+    const where = [];
+    const params = [];
+    if (options.jobId) {
+      where.push("job_id = ?");
+      params.push(options.jobId);
+    }
+    this.addTenantPredicate(where, params, options);
+    if (options.cursor) {
+      const cursor = parseCursor(options.cursor);
+      const createdAt = await this.resolveCursorCreatedAt(cursor, options);
+      const createdAtExpression = this.createdAtComparableExpression();
+      where.push(
+        `(${createdAtExpression} > ? OR (${createdAtExpression} = ? AND id > ?))`
+      );
+      params.push(createdAt, createdAt, cursor.id);
+    } else if (options.since) {
+      where.push(`${this.createdAtComparableExpression()} > ?`);
+      params.push(normalizeCursorDate(options.since));
+    }
+    if (options.afterId) {
+      where.push("id > ?");
+      params.push(options.afterId);
+    }
+    params.push(normalizeLimit(options.limit));
+    const whereSql = where.length ? `WHERE ${where.join(" AND ")}` : "";
+    return this.query(
+      `SELECT ${JOB_EVENT_STORAGE_COLUMNS}
+         FROM _smrt_job_events
+        ${whereSql}
+        ORDER BY ${this.createdAtComparableExpression()} ASC, id ASC
+        LIMIT ?`,
+      params,
+      { allowRawOnTenantScoped: true }
+    );
+  }
+  async latestProgressByJobIds(jobIds, options = {}) {
+    const uniqueJobIds = [...new Set(jobIds.filter(Boolean))];
+    const latestByJobId = /* @__PURE__ */ new Map();
+    if (uniqueJobIds.length === 0) return latestByJobId;
+    const placeholders = uniqueJobIds.map(() => "?").join(", ");
+    const where = [
+      `job_id IN (${placeholders})`,
+      "type = 'progress'"
+    ];
+    const params = [...uniqueJobIds];
+    this.addTenantPredicate(where, params, options);
+    const createdAtExpression = this.createdAtComparableExpression();
+    const events = await this.query(
+      `SELECT ${JOB_EVENT_STORAGE_COLUMNS}
+         FROM (
+           SELECT ${JOB_EVENT_STORAGE_COLUMNS},
+                  ${createdAtExpression} AS smrt_created_at_sort,
+                  ROW_NUMBER() OVER (
+                    PARTITION BY job_id
+                    ORDER BY ${createdAtExpression} DESC, id DESC
+                  ) AS smrt_rank
+             FROM _smrt_job_events
+            WHERE ${where.join(" AND ")}
+         ) ranked
+        WHERE smrt_rank = 1
+        ORDER BY smrt_created_at_sort DESC, id DESC`,
+      params,
+      { allowRawOnTenantScoped: true }
+    );
+    for (const event of events) {
+      latestByJobId.set(event.jobId, event);
+    }
+    return latestByJobId;
+  }
+  addTenantPredicate(where, params, options) {
+    if (options.tenantId === null) {
+      where.push("tenant_id IS NULL");
+      return;
+    }
+    const tenantId2 = typeof options.tenantId === "string" ? options.tenantId : getTenantId();
+    if (tenantId2) {
+      where.push("tenant_id = ?");
+      params.push(tenantId2);
+      return;
+    }
+    throw new Error(
+      "Tenant-scoped job event queries require tenantId, tenantId: null, or an ambient tenant context."
+    );
+  }
+  createdAtComparableExpression() {
+    if (usesSqliteDateFunctions(this.db.url)) {
+      return "strftime('%Y-%m-%dT%H:%M:%fZ', created_at)";
+    }
+    return "created_at";
+  }
+  async resolveCursorCreatedAt(cursor, options) {
+    if (!cursor.id) {
+      return normalizeCursorDate(cursor.createdAt);
+    }
+    const where = ["id = ?"];
+    const params = [cursor.id];
+    if (options.jobId) {
+      where.push("job_id = ?");
+      params.push(options.jobId);
+    }
+    this.addTenantPredicate(where, params, options);
+    const result = await this.db.query(
+      `SELECT ${this.createdAtComparableExpression()} AS cursor_created_at
+         FROM _smrt_job_events
+        WHERE ${where.join(" AND ")}
+        LIMIT 1`,
+      ...params
+    );
+    const cursorCreatedAt = getQueryRows(result)[0]?.cursor_created_at;
+    return typeof cursorCreatedAt === "string" && cursorCreatedAt.trim() ? cursorCreatedAt : normalizeCursorDate(cursor.createdAt);
+  }
+}
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __decorateClass = (decorators, target, key, kind) => {
+  var result = kind > 1 ? void 0 : kind ? __getOwnPropDesc(target, key) : target;
+  for (var i = decorators.length - 1, decorator; i >= 0; i--)
+    if (decorator = decorators[i])
+      result = (kind ? decorator(target, key, result) : decorator(result)) || result;
+  if (kind && result) __defProp(target, key, result);
+  return result;
+};
+let SmrtWorker = class extends SmrtObject {
+  workerId = "";
+  pid = null;
+  hostname = null;
+  startedAt = null;
+  heartbeatAt = null;
+  leaseExpiresAt = null;
+  status = "running";
+};
+__decorateClass([
+  field({ type: "text", required: true })
+], SmrtWorker.prototype, "workerId", 2);
+__decorateClass([
+  field({ type: "integer", nullable: true })
+], SmrtWorker.prototype, "pid", 2);
+__decorateClass([
+  field({ type: "text", nullable: true })
+], SmrtWorker.prototype, "hostname", 2);
+__decorateClass([
+  field({ type: "datetime", nullable: true })
+], SmrtWorker.prototype, "startedAt", 2);
+__decorateClass([
+  field({ type: "datetime", nullable: true })
+], SmrtWorker.prototype, "heartbeatAt", 2);
+__decorateClass([
+  field({ type: "datetime", nullable: true })
+], SmrtWorker.prototype, "leaseExpiresAt", 2);
+__decorateClass([
+  field({ type: "text", required: true, default: "running" })
+], SmrtWorker.prototype, "status", 2);
+SmrtWorker = __decorateClass([
+  smrt({
+    tableName: "_smrt_workers",
+    conflictColumns: ["worker_id"],
+    api: false,
+    cli: false,
+    mcp: false
+  })
+], SmrtWorker);
+class SmrtWorkerCollection extends SmrtCollection {
+  static _itemClass = SmrtWorker;
+  /**
+   * Fail fast if the `_smrt_workers` table has not been migrated.
+   *
+   * The framework never creates application/system tables at runtime; the
+   * table is created by `smrt db:migrate` (or `getTestDatabase`). A consumer
+   * that upgrades smrt-jobs without migrating must get a clear, actionable
+   * error at `start()` rather than a confusing recovery failure later.
+   */
+  async assertReady() {
+    try {
+      await this.db.query("SELECT 1 FROM _smrt_workers LIMIT 1");
+    } catch (error) {
+      throw new Error(
+        `The _smrt_workers table is missing. Run \`smrt db:migrate\` to create job-system tables before starting a TaskRunner/ScheduleRunner. (underlying error: ${error.message})`
+      );
+    }
+  }
+  /** Whether the `_smrt_workers` table exists (recovery skips lease checks if not). */
+  async tableReady() {
+    try {
+      await this.db.query("SELECT 1 FROM _smrt_workers LIMIT 1");
+      return true;
+    } catch {
+      return false;
+    }
+  }
+  /** Register a worker incarnation with its lease seeded to `now + ttl`. */
+  async registerWorker(input) {
+    const now = /* @__PURE__ */ new Date();
+    await this.create({
+      workerId: input.workerKey,
+      pid: input.pid ?? null,
+      hostname: input.hostname ?? null,
+      startedAt: now,
+      heartbeatAt: now,
+      // Seed the lease in the same write so the worker is immediately "alive",
+      // closing the window between registration and the first claimReady().
+      leaseExpiresAt: new Date(now.getTime() + input.leaseTtlMs),
+      status: "running"
+    });
+  }
+  /** Renew a worker's lease to `now + ttl`. */
+  async renewLease(workerKey, leaseTtlMs) {
+    const now = /* @__PURE__ */ new Date();
+    await this.db.query(
+      `UPDATE _smrt_workers
+          SET lease_expires_at = ?,
+              heartbeat_at = ?
+        WHERE worker_id = ?`,
+      new Date(now.getTime() + leaseTtlMs).toISOString(),
+      now.toISOString(),
+      workerKey
+    );
+  }
+  /** Remove a worker incarnation (graceful shutdown). */
+  async expireWorker(workerKey) {
+    await this.db.query(
+      "DELETE FROM _smrt_workers WHERE worker_id = ?",
+      workerKey
+    );
+  }
+  /** Worker keys whose database lease is still fresh (alive cross-process). */
+  async freshLeaseWorkerKeys() {
+    const result = await this.db.query(
+      `SELECT worker_id
+         FROM _smrt_workers
+        WHERE lease_expires_at IS NOT NULL
+          AND lease_expires_at >= ?`,
+      (/* @__PURE__ */ new Date()).toISOString()
+    );
+    const keys = /* @__PURE__ */ new Set();
+    for (const row of result.rows) {
+      if (typeof row.worker_id === "string") keys.add(row.worker_id);
+    }
+    return keys;
+  }
+  /** Delete worker rows whose lease expired more than `graceMs` ago. */
+  async pruneExpired(graceMs) {
+    const cutoff = new Date(Date.now() - Math.max(0, graceMs)).toISOString();
+    await this.db.query(
+      `DELETE FROM _smrt_workers
+        WHERE lease_expires_at IS NOT NULL
+          AND lease_expires_at < ?`,
+      cutoff
+    );
+  }
+}
+const DEFAULT_TASK_HEARTBEAT_INTERVAL_MS = 3e4;
+const DEFAULT_LEASE_TICK_MS = 1e4;
+const DEFAULT_LEASE_TTL_MS = 3e4;
+const LEASE_TTL_GRACE_MULTIPLIER = 3;
+function getEffectiveLeaseTtlMs(leaseTtlMs, leaseTickMs) {
+  return Math.max(leaseTtlMs, leaseTickMs * LEASE_TTL_GRACE_MULTIPLIER);
+}
+const LIVENESS_THREAD_START_TIMEOUT_MS = 1e4;
+const DEFAULT_CONFIG = {
+  id: "",
+  concurrency: 5,
+  queues: ["default"],
+  pollInterval: 1e3,
+  heartbeatInterval: DEFAULT_TASK_HEARTBEAT_INTERVAL_MS,
+  shutdownTimeout: 3e4,
+  staleJobThresholdMs: 9e4,
+  leaseTtlMs: DEFAULT_LEASE_TTL_MS,
+  leaseTickMs: DEFAULT_LEASE_TICK_MS
+};
+class TaskRunner extends EventEmitter {
+  id;
+  /**
+   * Per-incarnation-unique worker key. Stored as the `worker_id` on claimed
+   * jobs and in `_smrt_workers`, so a restart of a runner sharing the same
+   * configured `id` does not look like it still owns the previous
+   * incarnation's orphaned jobs. The human-facing {@link id} stays stable for
+   * events/logs.
+   */
+  workerKey;
+  config;
+  effectiveLeaseTtlMs;
+  collection = null;
+  eventCollection = null;
+  workerCollection = null;
+  workersTableVerified = false;
+  lastRecoverySweepAt = 0;
+  running = false;
+  activeJobs = /* @__PURE__ */ new Map();
+  pollTimer = null;
+  heartbeatTimer = null;
+  leaseTimer = null;
+  livenessWorker = null;
+  shutdownPromise = null;
+  db = null;
+  logger = createLogger(true);
+  constructor(config = {}) {
+    super();
+    this.config = {
+      ...DEFAULT_CONFIG,
+      ...config,
+      id: config.id || `runner_${createId().slice(0, 8)}`
+    };
+    this.id = this.config.id;
+    this.workerKey = createWorkerKey(this.id);
+    this.effectiveLeaseTtlMs = getEffectiveLeaseTtlMs(
+      this.config.leaseTtlMs,
+      this.config.leaseTickMs
+    );
+  }
+  /**
+   * Initialize the runner with database connection
+   */
+  async initialize(db) {
+    this.db = db;
+    this.collection = await SmrtJobCollection.create({ db });
+    this.eventCollection = await SmrtJobEventCollection.create({ db });
+    this.workerCollection = await SmrtWorkerCollection.create({ db });
+  }
+  /**
+   * Start processing jobs
+   */
+  async start() {
+    if (this.running) return;
+    if (!this.collection || !this.workerCollection) {
+      throw new Error("TaskRunner not initialized. Call initialize() first.");
+    }
+    await this.workerCollection.assertReady();
+    if (this.db && resolveEngine(this.db) === "sqlite") {
+      await tuneSqliteForConcurrency(this.db);
+    }
+    await this.workerCollection.registerWorker({
+      workerKey: this.workerKey,
+      pid: typeof process !== "undefined" ? process.pid : null,
+      hostname: typeof process !== "undefined" ? process.env.HOSTNAME ?? null : null,
+      leaseTtlMs: this.effectiveLeaseTtlMs
+    });
+    registerLiveWorker(this.workerKey);
+    this.running = true;
+    if (offLoopEligible(this.db)) {
+      const threadStarted = await this.startLivenessThread();
+      if (!threadStarted) this.startLeaseRenewal();
+    } else {
+      this.startLeaseRenewal();
+    }
+    this.startPolling();
+    this.startHeartbeat();
+    this.emit("runner:started");
+  }
+  /**
+   * Stop processing jobs (graceful shutdown)
+   */
+  async stop() {
+    if (!this.running) return;
+    if (this.shutdownPromise) return this.shutdownPromise;
+    this.running = false;
+    if (this.pollTimer) {
+      clearTimeout(this.pollTimer);
+      this.pollTimer = null;
+    }
+    if (this.heartbeatTimer) {
+      clearInterval(this.heartbeatTimer);
+      this.heartbeatTimer = null;
+    }
+    this.shutdownPromise = this.waitForActiveJobs();
+    try {
+      await this.shutdownPromise;
+    } finally {
+      this.shutdownPromise = null;
+      await this.stopLivenessThread();
+      if (this.leaseTimer) {
+        clearInterval(this.leaseTimer);
+        this.leaseTimer = null;
+      }
+      unregisterLiveWorker(this.workerKey);
+      if (this.activeJobs.size === 0) {
+        try {
+          await this.workerCollection?.expireWorker(this.workerKey);
+        } catch {
+        }
+      }
+      this.emit("runner:stopped");
+    }
+  }
+  /**
+   * Check if runner is running
+   */
+  isRunning() {
+    return this.running;
+  }
+  /**
+   * Get count of active jobs
+   */
+  activeJobCount() {
+    return this.activeJobs.size;
+  }
+  /**
+   * Start the polling loop
+   */
+  startPolling() {
+    const poll = async () => {
+      if (!this.running) return;
+      try {
+        await this.poll();
+      } catch (error) {
+        this.emit("runner:error", error);
+      }
+      if (this.running) {
+        this.pollTimer = setTimeout(poll, this.config.pollInterval);
+      }
+    };
+    poll();
+  }
+  /**
+   * Poll for and process jobs
+   */
+  async poll() {
+    if (!this.collection || !this.db) return;
+    await this.recoverStaleJobs();
+    const available = this.config.concurrency - this.activeJobs.size;
+    if (available <= 0) return;
+    const jobs = await this.collection.claimReady({
+      workerId: this.workerKey,
+      queues: this.config.queues,
+      limit: available
+    });
+    for (const job of jobs) {
+      const jobId = job.id;
+      if (!jobId) continue;
+      this.processJob(job);
+    }
+  }
+  /**
+   * Process a single job
+   */
+  async processJob(job) {
+    const jobId = job.id;
+    if (!jobId) {
+      this.emit("runner:error", new Error("Job has no ID"));
+      return;
+    }
+    this.activeJobs.set(jobId, job);
+    this.emit("job:started", job);
+    await this.appendJobEvent(job, {
+      type: "status",
+      level: "info",
+      stage: "started",
+      progress: 0,
+      message: `Started job: ${job.getDescription()}`
+    });
+    try {
+      const timeoutPromise = new Promise((_, reject) => {
+        setTimeout(() => {
+          reject(new Error(`Job timeout after ${job.timeout}ms`));
+        }, job.timeout);
+      });
+      const result = await Promise.race([this.executeJob(job), timeoutPromise]);
+      const completedAt = /* @__PURE__ */ new Date();
+      const applied = await this.writeOwnedJob(jobId, {
+        status: "completed",
+        completed_at: completedAt.toISOString(),
+        result_pointer: result?.resultPointer ?? null,
+        updated_at: completedAt.toISOString()
+      });
+      if (applied) {
+        job.status = "completed";
+        job.completedAt = completedAt;
+        job.resultPointer = result?.resultPointer ?? null;
+        await this.appendJobEvent(job, {
+          type: "progress",
+          level: "info",
+          stage: "completed",
+          progress: 100,
+          message: `Completed job: ${job.getDescription()}`
+        });
+        this.emit("job:completed", job, result);
+      }
+    } catch (error) {
+      await this.handleJobError(job, error);
+    } finally {
+      this.activeJobs.delete(jobId);
+    }
+  }
+  /**
+   * Apply a terminal/retry state transition to a job only if this worker still
+   * owns it and it is still `running`. Returns whether the write applied.
+   *
+   * This closes the completion-vs-recovery race: if recovery already failed a
+   * job out from under a finishing handler (a genuine zombie), the handler's
+   * outcome is dropped rather than resurrecting the row.
+   */
+  async writeOwnedJob(jobId, assignments) {
+    if (!this.db) return false;
+    const columns = Object.keys(assignments);
+    const setSql = columns.map((column) => `${column} = ?`).join(", ");
+    const values = columns.map((column) => assignments[column]);
+    const result = await this.db.query(
+      `UPDATE _smrt_jobs
+          SET ${setSql}
+        WHERE id = ? AND worker_id = ? AND status = 'running'
+        RETURNING id`,
+      ...values,
+      jobId,
+      this.workerKey
+    );
+    return (result.rows?.length ?? 0) > 0;
+  }
+  /**
+   * Execute a job by invoking the method on the SmrtObject
+   */
+  async executeJob(job) {
+    const runJob = async () => {
+      const registeredClass = ObjectRegistry.getClass(job.objectType);
+      if (!registeredClass) {
+        throw new Error(`Unknown object type: ${job.objectType}`);
+      }
+      const ObjectClass = registeredClass.constructor;
+      const rawArgs = job.args ?? {};
+      const persistedAgentConfig = rawArgs._agentConfig ?? {};
+      const { _agentConfig: _, _scheduleId: __, ...methodArgs } = rawArgs;
+      const classResolvers = getClassConfigResolvers(ObjectClass);
+      const agentConfig = await resolveLazyConfig(persistedAgentConfig, {
+        classResolvers,
+        onError: "throw"
+      });
+      let instance;
+      if (job.objectId) {
+        instance = new ObjectClass({ db: this.db, ...agentConfig });
+        await instance.initialize();
+        await instance.loadFromId(job.objectId);
+      } else {
+        instance = new ObjectClass({ db: this.db, ...agentConfig });
+        await instance.initialize();
+      }
+      const jobId = job.id;
+      if (!jobId) {
+        throw new Error("Job has no ID");
+      }
+      const baseLogger = createLogger(true);
+      const contextLogger = new JobContextLogger(baseLogger, {
+        jobId,
+        attempt: job.attempts,
+        queue: job.queue,
+        objectType: job.objectType,
+        method: job.method
+      });
+      contextLogger.info(`Starting job: ${job.getDescription()}`);
+      const executionContext = this.createExecutionContext(job, contextLogger);
+      const method = instance[job.method];
+      if (typeof method !== "function") {
+        throw new Error(`Method not found: ${job.objectType}.${job.method}`);
+      }
+      if (!isBackgroundEligibleMethod(ObjectClass, job.method)) {
+        throw new Error(
+          `Method not background-eligible: ${job.objectType}.${job.method}`
+        );
+      }
+      const result = await method.call(instance, methodArgs, executionContext);
+      return { result };
+    };
+    if (job.tenantId) {
+      return TenantContext.runWithJobContext(
+        { tenantId: job.tenantId },
+        runJob
+      );
+    }
+    return runJob();
+  }
+  /**
+   * Handle job execution error
+   */
+  async handleJobError(job, error) {
+    const strategy = fromConfig(job.retryStrategy);
+    const decision = strategy.shouldRetry(job.attempts, error);
+    const jobId = job.id;
+    if (!jobId) return;
+    const safeMessage = redactErrorForPersistence(error);
+    if (decision.shouldRetry && job.attempts < job.maxAttempts) {
+      const nextRunAt = new Date(Date.now() + decision.delay);
+      const applied = await this.writeOwnedJob(jobId, {
+        status: "pending",
+        last_error: safeMessage,
+        run_at: nextRunAt.toISOString(),
+        worker_id: null,
+        worker_heartbeat: null,
+        updated_at: (/* @__PURE__ */ new Date()).toISOString()
+      });
+      if (!applied) return;
+      job.status = "pending";
+      job.lastError = safeMessage;
+      job.runAt = nextRunAt;
+      job.workerId = null;
+      job.workerHeartbeat = null;
+      await this.appendJobEvent(job, {
+        type: "status",
+        level: "warn",
+        stage: "retrying",
+        message: `Retrying job after failure: ${safeMessage}`,
+        data: { delay: decision.delay, attempts: job.attempts }
+      });
+      this.emit("job:retrying", job, error, decision.delay);
+    } else {
+      const completedAt = /* @__PURE__ */ new Date();
+      const applied = await this.writeOwnedJob(jobId, {
+        status: "failed",
+        completed_at: completedAt.toISOString(),
+        last_error: safeMessage,
+        updated_at: completedAt.toISOString()
+      });
+      if (!applied) return;
+      job.status = "failed";
+      job.completedAt = completedAt;
+      job.lastError = safeMessage;
+      await this.appendJobEvent(job, {
+        type: "error",
+        level: "error",
+        stage: "failed",
+        message: safeMessage,
+        data: { attempts: job.attempts }
+      });
+      this.emit("job:failed", job, error);
+    }
+  }
+  createExecutionContext(job, contextLogger) {
+    const jobContext = {
+      jobId: job.id ?? "",
+      tenantId: job.tenantId ?? null,
+      attempt: job.attempts,
+      queue: job.queue,
+      objectType: job.objectType,
+      method: job.method
+    };
+    return {
+      job: jobContext,
+      logger: contextLogger,
+      event: async (input) => {
+        await this.appendJobEvent(job, input);
+      },
+      progress: async (input) => {
+        const data = {
+          ...input.data ?? {},
+          ...input.detail ? { detail: input.detail } : {},
+          ...input.source ? { source: input.source } : {}
+        };
+        await this.appendJobEvent(job, {
+          type: "progress",
+          level: "info",
+          stage: input.stage,
+          progress: input.progress,
+          message: input.message ?? input.detail ?? `${input.stage} ${Math.round(input.progress)}%`,
+          data
+        });
+      },
+      log: async (level, message, data) => {
+        contextLogger[level](message, data);
+        await this.appendJobEvent(job, {
+          type: level === "error" ? "error" : "log",
+          level,
+          message,
+          data
+        });
+      }
+    };
+  }
+  async appendJobEvent(job, input) {
+    if (!this.eventCollection || !job.id) {
+      return null;
+    }
+    try {
+      const event = await this.eventCollection.append({
+        tenantId: job.tenantId ?? null,
+        jobId: job.id,
+        type: input.type ?? "log",
+        level: input.level ?? "info",
+        stage: input.stage ?? null,
+        progress: input.progress ?? null,
+        message: input.message ?? "",
+        data: input.data ?? {}
+      });
+      this.emit("job:event", job, event);
+      if (event.type === "progress") {
+        this.emit("job:progress", job, event);
+      }
+      return event;
+    } catch (error) {
+      const telemetryError = error instanceof Error ? error : new Error(`Failed to append job telemetry: ${String(error)}`);
+      try {
+        this.emit("runner:error", telemetryError);
+      } catch {
+      }
+      return null;
+    }
+  }
+  /**
+   * Whether the `_smrt_workers` table exists. Cached once positive — the table
+   * never disappears mid-run, so this avoids a probe query on every poll.
+   */
+  async workersTableReady() {
+    if (this.workersTableVerified) return true;
+    const ready = await this.workerCollection?.tableReady() ?? false;
+    if (ready) this.workersTableVerified = true;
+    return ready;
+  }
+  /**
+   * Recover jobs orphaned by dead/restarted workers.
+   *
+   * A `running` job is recovered only when its owning worker is *not alive*
+   * (issue #1474): not live in this process and holding no fresh lease in
+   * `_smrt_workers`. This is independent of the handler event loop, so a worker
+   * whose handler holds the loop synchronously keeps a fresh lease (renewed off
+   * the loop by the liveness thread) or stays in this process's live set, and is
+   * never false-recovered. The live set takes precedence over a stale lease, and
+   * a runner never recovers its own active jobs.
+   *
+   * Recovery is swept at most once per lease tick (not every poll), since
+   * detection is TTL-bound anyway — this bounds the per-poll database load.
+   */
+  async recoverStaleJobs() {
+    if (!this.db || !this.collection || !this.workerCollection) return;
+    if (!await this.workersTableReady()) return;
+    const now = Date.now();
+    if (now - this.lastRecoverySweepAt < this.config.leaseTickMs) return;
+    this.lastRecoverySweepAt = now;
+    try {
+      await this.workerCollection.pruneExpired(this.effectiveLeaseTtlMs * 10);
+    } catch {
+    }
+    const freshLeaseKeys = await this.workerCollection.freshLeaseWorkerKeys();
+    const running = await this.collection.query(
+      `SELECT * FROM _smrt_jobs WHERE status = 'running'`,
+      [],
+      { allowRawOnTenantScoped: true }
+    );
+    if (running.length === 0) return;
+    const orphans = running.filter((job) => {
+      const jobId = job.id;
+      if (jobId && this.activeJobs.has(jobId)) return false;
+      return !isWorkerAlive(job.workerId, freshLeaseKeys);
+    });
+    if (orphans.length === 0) return;
+    const orphanIds = orphans.map((job) => job.id).filter((jobId) => typeof jobId === "string");
+    if (orphanIds.length === 0) return;
+    const placeholders = orphanIds.map(() => "?").join(", ");
+    const recoveredAt = /* @__PURE__ */ new Date();
+    const errorMessage = "Recovered orphaned running job: its owning worker is no longer alive (no fresh liveness lease in _smrt_workers and not running in this process).";
+    const updated = await this.db.query(
+      `UPDATE _smrt_jobs
+          SET status = 'failed',
+              completed_at = ?,
+              last_error = ?,
+              worker_id = NULL,
+              worker_heartbeat = NULL
+        WHERE status = 'running'
+          AND id IN (${placeholders})
+        RETURNING id`,
+      recoveredAt.toISOString(),
+      errorMessage,
+      ...orphanIds
+    );
+    const recoveredIds = new Set(
+      updated.rows.map((row) => row.id).filter((id) => typeof id === "string")
+    );
+    if (recoveredIds.size === 0) return;
+    for (const job of orphans) {
+      if (!job.id || !recoveredIds.has(job.id)) continue;
+      job.status = "failed";
+      job.completedAt = recoveredAt;
+      job.lastError = errorMessage;
+      job.workerId = null;
+      job.workerHeartbeat = null;
+      const error = new Error(errorMessage);
+      await this.appendJobEvent(job, {
+        type: "error",
+        level: "error",
+        stage: "stale-recovery",
+        message: errorMessage
+      });
+      this.emit("job:failed", job, error);
+    }
+  }
+  /**
+   * Renew this worker's liveness lease.
+   *
+   * In Stage 1 this runs on the main event loop, so it provides cross-process
+   * detection no weaker than the old per-job heartbeat. Stage 2 moves the
+   * renewal to an off-loop worker thread so a synchronous handler can no longer
+   * starve it. Same-process correctness never depends on this timer — the
+   * in-memory live set covers it.
+   */
+  startLeaseRenewal() {
+    this.leaseTimer = setInterval(async () => {
+      try {
+        await this.workerCollection?.renewLease(
+          this.workerKey,
+          this.effectiveLeaseTtlMs
+        );
+      } catch {
+      }
+    }, this.config.leaseTickMs);
+  }
+  /**
+   * Spawn the off-loop liveness thread. It opens its own connection and renews
+   * this worker's lease on its own thread (unstarvable by handler CPU). Returns
+   * false if the thread can't be resolved or fails to start, so the caller can
+   * fall back to main-loop renewal.
+   */
+  async startLivenessThread() {
+    if (!this.db) return false;
+    let entry;
+    try {
+      entry = import.meta.resolve("@happyvertical/smrt-jobs/worker-liveness-thread");
+    } catch {
+      return false;
+    }
+    let worker;
+    try {
+      worker = new Worker(new URL(entry), {
+        workerData: {
+          url: resolveUrl(this.db),
+          type: resolveEngine(this.db),
+          workerKey: this.workerKey,
+          leaseTtlMs: this.effectiveLeaseTtlMs,
+          leaseTickMs: this.config.leaseTickMs
+        }
+      });
+    } catch {
+      return false;
+    }
+    const ready = await new Promise((resolve) => {
+      const onMessage = (message) => {
+        if (message === "ready") {
+          cleanup();
+          resolve(true);
+        } else if (message && typeof message === "object" && message.type === "error") {
+          cleanup();
+          resolve(false);
+        }
+      };
+      const onFail = () => {
+        cleanup();
+        resolve(false);
+      };
+      const cleanup = () => {
+        clearTimeout(timer);
+        worker.off("message", onMessage);
+        worker.off("error", onFail);
+        worker.off("exit", onFail);
+      };
+      const timer = setTimeout(() => {
+        cleanup();
+        resolve(false);
+      }, LIVENESS_THREAD_START_TIMEOUT_MS);
+      if (typeof timer.unref === "function") timer.unref();
+      worker.on("message", onMessage);
+      worker.once("error", onFail);
+      worker.once("exit", onFail);
+    });
+    if (!ready) {
+      await worker.terminate().catch(() => {
+      });
+      return false;
+    }
+    this.livenessWorker = worker;
+    worker.unref();
+    worker.once("error", () => this.handleLivenessThreadLoss(worker));
+    worker.once("exit", () => this.handleLivenessThreadLoss(worker));
+    return true;
+  }
+  handleLivenessThreadLoss(worker) {
+    if (this.livenessWorker !== worker) return;
+    this.livenessWorker = null;
+    if (this.running && !this.leaseTimer) {
+      this.startLeaseRenewal();
+    }
+  }
+  /** Stop the liveness thread (graceful, with a short bound), if running. */
+  async stopLivenessThread() {
+    const worker = this.livenessWorker;
+    if (!worker) return;
+    this.livenessWorker = null;
+    const stopped = new Promise((resolve) => {
+      const done = () => {
+        clearTimeout(timer);
+        worker.off("message", onMessage);
+        resolve();
+      };
+      const onMessage = (message) => {
+        if (message === "stopped") done();
+      };
+      worker.on("message", onMessage);
+      worker.once("exit", done);
+      const timer = setTimeout(done, 2e3);
+      if (typeof timer.unref === "function") timer.unref();
+    });
+    try {
+      worker.postMessage("stop");
+    } catch {
+    }
+    await stopped;
+    await worker.terminate().catch(() => {
+    });
+  }
+  /**
+   * Per-job heartbeat loop — telemetry only ("last activity" for the UI). It no
+   * longer gates recovery (that is the worker lease), so a blocked loop missing
+   * a heartbeat is harmless.
+   */
+  startHeartbeat() {
+    this.heartbeatTimer = setInterval(async () => {
+      if (!this.db) return;
+      const jobIds = [...this.activeJobs.keys()];
+      if (jobIds.length === 0) return;
+      const placeholders = jobIds.map(() => "?").join(", ");
+      try {
+        await this.db.query(
+          `UPDATE _smrt_jobs
+              SET worker_heartbeat = ?
+            WHERE status = 'running'
+              AND id IN (${placeholders})`,
+          (/* @__PURE__ */ new Date()).toISOString(),
+          ...jobIds
+        );
+      } catch {
+      }
+    }, this.config.heartbeatInterval);
+  }
+  /**
+   * Wait for active jobs to complete with timeout
+   */
+  async waitForActiveJobs() {
+    if (this.activeJobs.size === 0) return;
+    return new Promise((resolve) => {
+      const checkInterval = setInterval(() => {
+        if (this.activeJobs.size === 0) {
+          clearInterval(checkInterval);
+          clearTimeout(timeout);
+          resolve();
+        }
+      }, 100);
+      const timeout = setTimeout(() => {
+        clearInterval(checkInterval);
+        this.logger.warn(
+          `Shutdown timeout: ${this.activeJobs.size} jobs still active`
+        );
+        resolve();
+      }, this.config.shutdownTimeout);
+    });
+  }
+}
+function createTaskRunner(config) {
+  return new TaskRunner(config);
+}
+export {
+  DEFAULT_TENANT_JOB_CAP as D,
+  JobContextLogger as J,
+  MAX_JOB_RETRIES as M,
+  SmrtJobCollection as S,
+  TaskRunner as T,
+  DEFAULT_TASK_HEARTBEAT_INTERVAL_MS as a,
+  SmrtWorkerCollection as b,
+  clampRetries as c,
+  redactErrorForPersistence as d,
+  SmrtJob as e,
+  SmrtJobEvent as f,
+  SmrtJobEventCollection as g,
+  SmrtWorker as h,
+  TenantJobCapExceededError as i,
+  assertWithinTenantCreationCap as j,
+  backgroundEligible as k,
+  createTaskRunner as l,
+  getBackgroundEligibleMethods as m,
+  isBackgroundEligibleMethod as n,
+  markBackgroundEligible as o,
+  redactErrorMessage as r
+};
+//# sourceMappingURL=runner-DV8FBO0y.js.map