@softure-ai/jobs 0.1.2 → 0.1.3

package/CHANGELOG.md

@@ -4,6 +4,29 @@ Newest first. Each version lists what changed for an app that uses `@softure-ai/
 production, the version gets a line `verified in: <app>@<commit>` ([docs/05](../../docs/05-adoption-playbook.md),
 "Definition of done").
 
+## 0.1.3
+
+- A health check of the worker's loop (#610): `runJobWorkerProcess({ healthFile })` (and `runJobWorker`) touches the
+  file after each round whose claims reached the database, removes a stale one at start and its own after a clean
+  stop; started with `--health`, the same entry checks the file instead of running (exit 0 when no older than
+  `healthMaxAgeMs`, default five poll intervals and at least 30 s; 1 when missing or older, or without `healthFile`).
+  New exports from `@softure-ai/jobs/worker`: `checkJobWorkerHealth`, `getJobWorkerHealthMaxAgeMs`, `JobWorkerHealth`.
+  Off by default: an app that passes no `healthFile` sees no change.
+- Migration for an app whose worker runs in Docker with a liveness check of PID 1 (`node -e "process.kill(1, 0)"`):
+  after upgrading to this version, pass `healthFile: "/tmp/softure-jobs-worker.alive"` in `scripts/worker.ts` and set
+  the worker service's `healthcheck.test` to `["CMD", "node", "worker.mjs", "--health"]` in the production compose
+  file (and the integration one, if it has a worker). Do it in that order: an older worker does not know `--health`.
+- `getQueueStats` reports a scheduled queue's last failure (#603): each `schedule` gains `lastFailedAt` (when a job of
+  the queue last ended `failed`: attempts used up, `failJob`, an invalid payload or an expired lease with no attempts
+  left; a retried attempt does not count), `lastFailureCode` (that job's error code, never an error's text) and
+  `isFailing` (`lastFailedAt` later than `lastSucceededAt`, or a failure and no success yet). Stored in
+  `jobs.schedules` in the same transaction as the failed job, so it outlives pruning; migration
+  `0003_add_schedule_failures.sql` adds two nullable columns.
+- Migration for an app: run the module migrations (`0003` adds the columns; failures before it are not back-filled).
+  An app that kept a log of its own runs only to tell whether the latest run failed reads
+  `stats.schedule.isFailing`, `lastFailedAt` and `lastFailureCode` from `getQueueStats` instead and can drop that log.
+  Code or tests that compare a whole `schedule` with `toEqual` see the three new fields.
+
 ## 0.1.2
 
 - Schedules at local times of day (#554): `schedules: { "<queue>": { at: ["09:00", "21:00"], timeZone: "Europe/Warsaw" } }`

package/README.md

@@ -151,6 +151,31 @@ On SIGTERM the worker stops claiming, waits up to `shutdownGraceMs` for running
 releases their jobs (pending again, due now, the attempt given back). Database errors while claiming or recording are
 logged and retried on the next round; a database restart does not end the worker.
 
+**Health check of the loop.** A process that is alive is not a loop that runs: a hung claim query or a database
+connection the worker never gets back leaves the process up and the jobs untouched. With `healthFile`, the loop
+touches that file after each round in which every claim reached the database (it removes a file left by an earlier
+run at start, and removes its own after a clean stop), and the same entry answers `--health` instead of running:
+
+```ts
+// scripts/worker.ts
+await runJobWorkerProcess({ config, handlers, healthFile: "/tmp/softure-jobs-worker.alive" });
+```
+
+```yaml
+# docker compose: the worker service
+healthcheck:
+  test: ["CMD", "node", "worker.mjs", "--health"]
+```
+
+`node worker.mjs --health` opens no database and starts no loop: it exits 0 when the file is no older than
+`healthMaxAgeMs` (default five poll intervals, at least 30 s: `getJobWorkerHealthMaxAgeMs(config)`) and 1 when it is
+missing or older, with one line saying which. `--health` without `healthFile` exits 1 rather than start a second
+worker; an older jobs version has no `--health` and would start one, so switch the check only after upgrading. An app
+that runs the loop itself passes `healthFile` to `runJobWorker` and checks with
+`checkJobWorkerHealth({ healthFile, maxAgeMs })`. `docker compose up --wait` then waits for a loop that reached the
+database, and the container reads `unhealthy` while it is stuck; Docker's `restart:` policy does not act on that
+status (a monitor or an orchestrator does). `@softure-ai/deploy` writes both lines from `init`.
+
 `runJobWorker({ db, config, handlers, signal })` is the loop without the process part, for an app that runs it inside
 a process of its own.
 
@@ -201,9 +226,24 @@ export const handlers: JobHandlers = { ...authJobHandlers, ...securityJobHandler
   optional peer.
 
 `getQueueStats` reports each scheduled queue's `schedule`: `everySeconds` (for an `at` schedule the longest gap
-between two of its times, 24h for one time), `lastEnqueuedAt` (the start of the latest slot a run was enqueued for), `lastSucceededAt` (any succeeded job of the queue, a run enqueued by hand too) and
-`isLate`: no success within two periods, counted from when a worker first saw the schedule, or no worker has seen it
-at all. The module's health check does not fail on a late schedule (a late cleanup must not take the app out of
+between two of its times, 24h for one time), `lastEnqueuedAt` (the start of the latest slot a run was enqueued for), `lastSucceededAt` (any succeeded job of the queue, a run enqueued by hand too),
+`lastFailedAt` and `lastFailureCode` (the latest job of the queue that ended `failed`, with its error code: the
+handler's code, `jobs.handler_threw`, `jobs.invalid_payload` or `jobs.lease_expired`, never an error's text; a retried
+attempt is not a failure until the attempts are used up), `isFailing` and `isLate`. A queue is failing when its latest
+outcome is a failure: `isFailing` is `lastFailedAt !== null && (lastSucceededAt === null || lastFailedAt >
+lastSucceededAt)`. It is late when no run succeeded within two periods, counted from when a worker first saw the
+schedule, or no worker has seen it at all. Successes and failures are recorded once a worker has taken the schedule's
+first slot (its row in `jobs.schedules` exists), and they outlive the pruned jobs.
+
+An app that shows how fresh a synced source is needs nothing else:
+
+```ts
+const sync = (await getQueueStats(ctx)).find((stats) => stats.queue === "app.sync")?.schedule;
+// "Updated at {lastSucceededAt}", plus "the last update failed ({lastFailureCode})" while isFailing.
+const freshness = { updatedAt: sync?.lastSucceededAt ?? null, isFailing: sync?.isFailing ?? false };
+```
+
+The module's health check does not fail on a late schedule (a late cleanup must not take the app out of
 rotation); an app that wants an alarm adds a check of its own:
 
 ```ts
@@ -246,7 +286,8 @@ Nothing to mount: the module has no routes or pages. The worker is a process (§
 
 ## 5. Migrations and tables
 
-Schema `jobs`, two tables, `migrations/0001_create_jobs.sql` and `migrations/0002_create_schedules.sql`:
+Schema `jobs`, two tables, `migrations/0001_create_jobs.sql`, `migrations/0002_create_schedules.sql` and
+`migrations/0003_add_schedule_failures.sql`:
 
 `jobs.jobs`: `id` (bigint identity; a string in the API), `queue`, `payload` (jsonb), `status` (`pending`, `running`,
 `succeeded`, `failed`, `cancelled`), `attempts`, `max_attempts`, `run_at`, `locked_until`, `locked_by` (the worker,
@@ -255,7 +296,8 @@ Schema `jobs`, two tables, `migrations/0001_create_jobs.sql` and `migrations/000
 for finished jobs; a partial unique index allows one live job per queue and dedup key.
 
 `jobs.schedules`: `name` (the schedule's queue, primary key), `last_slot_at` (the start of the latest period a run
-was enqueued for), `last_succeeded_at`, `created_at`, `updated_at`. One row per schedule a worker has seen; a row of
+was enqueued for), `last_succeeded_at`, `last_failed_at`, `last_failure_code` (an error code, never a message),
+`created_at`, `updated_at`. One row per schedule a worker has seen; a row of
 a schedule removed from the config stays and is ignored.
 
 The app role needs `SELECT, INSERT, UPDATE, DELETE` on the tables, which the `@softure-ai/ops` roles recipe grants on

package/dist/index.js

@@ -13,7 +13,7 @@ export const MODULE_ID = "jobs";
 export const jobs = defineModule({
     manifest: {
         id: MODULE_ID,
-        version: "0.1.2",
+        version: "0.1.3",
         dependsOn: {},
         dbSchema: "jobs",
         tables: ["jobs", "schedules"],

package/dist/schema.d.ts

@@ -317,6 +317,40 @@ export declare const schedules: import("drizzle-orm/pg-core").PgTableWithColumns
             identity: undefined;
             generated: undefined;
         }, {}, {}>;
+        lastFailedAt: import("drizzle-orm/pg-core").PgColumn<{
+            name: "last_failed_at";
+            tableName: "schedules";
+            dataType: "date";
+            columnType: "PgTimestamp";
+            data: Date;
+            driverParam: string;
+            notNull: false;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: undefined;
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {}>;
+        lastFailureCode: import("drizzle-orm/pg-core").PgColumn<{
+            name: "last_failure_code";
+            tableName: "schedules";
+            dataType: "string";
+            columnType: "PgText";
+            data: string;
+            driverParam: string;
+            notNull: false;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: [string, ...string[]];
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {}>;
         createdAt: import("drizzle-orm/pg-core").PgColumn<{
             name: "created_at";
             tableName: "schedules";

package/dist/schema.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,UAAU,gDAAmB,CAAC;AAE3C,eAAO,MAAM,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAgBf,CAAC;AAEH,MAAM,MAAM,MAAM,GAAG,OAAO,IAAI,CAAC,YAAY,CAAC;AAE9C,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAMpB,CAAC"}
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,UAAU,gDAAmB,CAAC;AAE3C,eAAO,MAAM,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAgBf,CAAC;AAEH,MAAM,MAAM,MAAM,GAAG,OAAO,IAAI,CAAC,YAAY,CAAC;AAE9C,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAQpB,CAAC"}

package/dist/schema.js

@@ -23,6 +23,8 @@ export const schedules = jobsSchema.table("schedules", {
     name: text("name").primaryKey(),
     lastSlotAt: timestamp("last_slot_at", { withTimezone: true }).notNull(),
     lastSucceededAt: timestamp("last_succeeded_at", { withTimezone: true }),
+    lastFailedAt: timestamp("last_failed_at", { withTimezone: true }),
+    lastFailureCode: text("last_failure_code"),
     createdAt: timestamp("created_at", { withTimezone: true }).notNull(),
     updatedAt: timestamp("updated_at", { withTimezone: true }).notNull(),
 });

package/dist/schema.js.map

@@ -1 +1 @@
-{"version":3,"file":"schema.js","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,kGAAkG;AAClG,+BAA+B;AAC/B,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AAE9F,MAAM,CAAC,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;AAE3C,MAAM,CAAC,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE;IAC3C,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,yBAAyB,EAAE;IAC7E,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE;IAC9B,OAAO,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE;IACnC,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC,SAAS,EAAE,SAAS,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE;IACtG,QAAQ,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,OAAO,EAAE;IACvC,WAAW,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC,OAAO,EAAE;IAC9C,KAAK,EAAE,SAAS,CAAC,QAAQ,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE;IAC5D,WAAW,EAAE,SAAS,CAAC,cAAc,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;IAC9D,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC;IAC3B,UAAU,EAAE,IAAI,CAAC,aAAa,CAAC;IAC/B,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC;IAC3B,SAAS,EAAE,IAAI,CAAC,YAAY,CAAC;IAC7B,SAAS,EAAE,SAAS,CAAC,YAAY,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE;IACpE,SAAS,EAAE,SAAS,CAAC,YAAY,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE;IACpE,UAAU,EAAE,SAAS,CAAC,aAAa,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;CAC7D,CAAC,CAAC;AAIH,MAAM,CAAC,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,CAAC,WAAW,EAAE;IACrD,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE;IAC/B,UAAU,EAAE,SAAS,CAAC,cAAc,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE;IACvE,eAAe,EAAE,SAAS,CAAC,mBAAmB,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;IACvE,SAAS,EAAE,SAAS,CAAC,YAAY,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE;IACpE,SAAS,EAAE,SAAS,CAAC,YAAY,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE;CACrE,CAAC,CAAC"}
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,kGAAkG;AAClG,+BAA+B;AAC/B,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AAE9F,MAAM,CAAC,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;AAE3C,MAAM,CAAC,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE;IAC3C,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,yBAAyB,EAAE;IAC7E,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE;IAC9B,OAAO,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE;IACnC,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC,SAAS,EAAE,SAAS,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE;IACtG,QAAQ,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,OAAO,EAAE;IACvC,WAAW,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC,OAAO,EAAE;IAC9C,KAAK,EAAE,SAAS,CAAC,QAAQ,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE;IAC5D,WAAW,EAAE,SAAS,CAAC,cAAc,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;IAC9D,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC;IAC3B,UAAU,EAAE,IAAI,CAAC,aAAa,CAAC;IAC/B,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC;IAC3B,SAAS,EAAE,IAAI,CAAC,YAAY,CAAC;IAC7B,SAAS,EAAE,SAAS,CAAC,YAAY,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE;IACpE,SAAS,EAAE,SAAS,CAAC,YAAY,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE;IACpE,UAAU,EAAE,SAAS,CAAC,aAAa,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;CAC7D,CAAC,CAAC;AAIH,MAAM,CAAC,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,CAAC,WAAW,EAAE;IACrD,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE;IAC/B,UAAU,EAAE,SAAS,CAAC,cAAc,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE;IACvE,eAAe,EAAE,SAAS,CAAC,mBAAmB,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;IACvE,YAAY,EAAE,SAAS,CAAC,gBAAgB,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;IACjE,eAAe,EAAE,IAAI,CAAC,mBAAmB,CAAC;IAC1C,SAAS,EAAE,SAAS,CAAC,YAAY,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE;IACpE,SAAS,EAAE,SAAS,CAAC,YAAY,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE;CACrE,CAAC,CAAC"}

package/dist/server/claim.d.ts

@@ -33,7 +33,8 @@ export declare function claimJobs(ctx: JobsContext, input: ClaimJobsInput): Prom
 /** Pushes the lease of a held job forward. False when the job is no longer this claim's: stop working on it. */
 export declare function extendJobLease(ctx: JobsContext, claimed: ClaimedJob): Promise<boolean>;
 /**
- * Records the end of an attempt; a success of a scheduled queue also updates its schedule, in the same transaction.
+ * Records the end of an attempt; a success or a terminal failure of a scheduled queue also updates its schedule, in
+ * the same transaction.
  * False when the job is no longer this claim's: nothing was written.
  */
 export declare function completeJob(ctx: JobsContext, claimed: ClaimedJob, completion: Completion): Promise<boolean>;

package/dist/server/claim.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"claim.d.ts","sourceRoot":"","sources":["../../src/server/claim.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAC;AAE1C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAKhD,iFAAiF;AACjF,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAC;IAClB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,gGAAgG;IAChG,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED,uCAAuC;AACvC,MAAM,MAAM,UAAU,GAClB;IAAE,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAA;CAAE,GAC9B;IAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAA;CAAE,GACxE;IAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAIxD;;;;;GAKG;AACH,wBAAsB,SAAS,CAAC,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAsC9F;AAED,gHAAgH;AAChH,wBAAsB,cAAc,CAAC,GAAG,EAAE,WAAW,EAAE,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,CAS5F;AAED;;;GAGG;AACH,wBAAsB,WAAW,CAAC,GAAG,EAAE,WAAW,EAAE,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,CAejH;AAED;;;GAGG;AACH,wBAAsB,UAAU,CAAC,GAAG,EAAE,WAAW,EAAE,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,CAgBxF"}
+{"version":3,"file":"claim.d.ts","sourceRoot":"","sources":["../../src/server/claim.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAC;AAE1C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAKhD,iFAAiF;AACjF,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAC;IAClB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,gGAAgG;IAChG,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED,uCAAuC;AACvC,MAAM,MAAM,UAAU,GAClB;IAAE,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAA;CAAE,GAC9B;IAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAA;CAAE,GACxE;IAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAIxD;;;;;GAKG;AACH,wBAAsB,SAAS,CAAC,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAsC9F;AAED,gHAAgH;AAChH,wBAAsB,cAAc,CAAC,GAAG,EAAE,WAAW,EAAE,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,CAS5F;AAED;;;;GAIG;AACH,wBAAsB,WAAW,CAAC,GAAG,EAAE,WAAW,EAAE,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,CAgBjH;AAED;;;GAGG;AACH,wBAAsB,UAAU,CAAC,GAAG,EAAE,WAAW,EAAE,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,CAgBxF"}

package/dist/server/claim.js

@@ -4,7 +4,7 @@ import { and, asc, eq, gte, inArray, lte, or, sql } from "drizzle-orm";
 import { jobs } from "../schema.js";
 import { toJob } from "./job-row.js";
 import { getJobsOptions, getQueue } from "./options.js";
-import { recordScheduleSuccess } from "./schedules.js";
+import { recordScheduleFailure, recordScheduleSuccess } from "./schedules.js";
 const LEASE_EXPIRED = "jobs.lease_expired";
 /**
  * Claims up to `limit` due jobs of one queue, oldest due first: pending jobs whose `run_at` has come and running jobs
@@ -53,7 +53,8 @@ export async function extendJobLease(ctx, claimed) {
     return rows.length > 0;
 }
 /**
- * Records the end of an attempt; a success of a scheduled queue also updates its schedule, in the same transaction.
+ * Records the end of an attempt; a success or a terminal failure of a scheduled queue also updates its schedule, in
+ * the same transaction.
  * False when the job is no longer this claim's: nothing was written.
  */
 export async function completeJob(ctx, claimed, completion) {
@@ -70,6 +71,8 @@ export async function completeJob(ctx, claimed, completion) {
             return false;
         if (completion.kind === "succeeded")
             await recordScheduleSuccess({ ...ctx, db: tx }, claimed.job.queue);
+        if (completion.kind === "failed")
+            await recordScheduleFailure({ ...ctx, db: tx }, claimed.job.queue, completion.error);
         return true;
     });
 }
@@ -95,18 +98,23 @@ export async function releaseJob(ctx, claimed) {
     return rows.length > 0;
 }
 async function failExhaustedLeases(ctx, queue, now) {
-    await ctx.db
-        .update(jobs)
-        .set({
-        status: "failed",
-        lastError: LEASE_EXPIRED,
-        finishedAt: now,
-        lockedUntil: null,
-        lockedBy: null,
-        claimToken: null,
-        updatedAt: now,
-    })
-        .where(and(eq(jobs.queue, queue), eq(jobs.status, "running"), lte(jobs.lockedUntil, now), gte(jobs.attempts, jobs.maxAttempts)));
+    await ctx.db.transaction(async (tx) => {
+        const failed = await tx
+            .update(jobs)
+            .set({
+            status: "failed",
+            lastError: LEASE_EXPIRED,
+            finishedAt: now,
+            lockedUntil: null,
+            lockedBy: null,
+            claimToken: null,
+            updatedAt: now,
+        })
+            .where(and(eq(jobs.queue, queue), eq(jobs.status, "running"), lte(jobs.lockedUntil, now), gte(jobs.attempts, jobs.maxAttempts)))
+            .returning({ id: jobs.id });
+        if (failed.length > 0)
+            await recordScheduleFailure({ ...ctx, db: tx }, queue, LEASE_EXPIRED);
+    });
 }
 function isHeld(claimed) {
     return and(eq(jobs.id, BigInt(claimed.job.id)), eq(jobs.status, "running"), eq(jobs.claimToken, claimed.claimToken));

package/dist/server/claim.js.map

@@ -1 +1 @@
-{"version":3,"file":"claim.js","sourceRoot":"","sources":["../../src/server/claim.ts"],"names":[],"mappings":"AAAA,kHAAkH;AAClH,qGAAqG;AACrG,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AAEvE,OAAO,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEpC,OAAO,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC;AACrC,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AACxD,OAAO,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAsBvD,MAAM,aAAa,GAAG,oBAAoB,CAAC;AAE3C;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,GAAgB,EAAE,KAAqB;IACrE,MAAM,KAAK,GAAG,QAAQ,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;IAChE,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC;QAAE,OAAO,EAAE,CAAC;IAC/B,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;IAC5B,MAAM,mBAAmB,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAEjD,MAAM,GAAG,GAAG,GAAG,CAAC,EAAE;SACf,MAAM,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC;SACvB,IAAI,CAAC,IAAI,CAAC;SACV,KAAK,CACJ,GAAG,CACD,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,EAC3B,EAAE,CACA,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,EACrD,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,GAAG,CAAA,GAAG,IAAI,CAAC,QAAQ,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC,CACzG,CACF,CACF;SACA,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;SACtC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC;SAClB,GAAG,CAAC,QAAQ,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;IAEvC,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,EAAE;SACtB,MAAM,CAAC,IAAI,CAAC;SACZ,GAAG,CAAC;QACH,MAAM,EAAE,SAAS;QACjB,QAAQ,EAAE,GAAG,CAAA,GAAG,IAAI,CAAC,QAAQ,MAAM;QACnC,WAAW,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC;QAChE,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,UAAU,EAAE,GAAG,CAAA,mBAAmB;QAClC,SAAS,EAAE,GAAG;KACf,CAAC;SACD,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;SAC5B,SAAS,EAAE,CAAC;IAEf,OAAO,IAAI;SACR,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC;SACjG,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,UAAU,EAAE,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC;AACnF,CAAC;AAED,gHAAgH;AAChH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,GAAgB,EAAE,OAAmB;IACxE,MAAM,KAAK,GAAG,QAAQ,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACtE,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;IAC5B,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,EAAE;SACtB,MAAM,CAAC,IAAI,CAAC;SACZ,GAAG,CAAC,EAAE,WAAW,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC;SACzF,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;SACtB,SAAS,EAAE,CAAC;IACf,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;AACzB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,GAAgB,EAAE,OAAmB,EAAE,UAAsB;IAC7F,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;IAC5B,MAAM,QAAQ,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC;IACzF,MAAM,GAAG,GACP,UAAU,CAAC,IAAI,KAAK,WAAW;QAC7B,CAAC,CAAC,EAAE,GAAG,QAAQ,EAAE,MAAM,EAAE,WAAoB,EAAE,UAAU,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE;QACjF,CAAC,CAAC,UAAU,CAAC,IAAI,KAAK,OAAO;YAC3B,CAAC,CAAC,EAAE,GAAG,QAAQ,EAAE,MAAM,EAAE,SAAkB,EAAE,KAAK,EAAE,UAAU,CAAC,KAAK,EAAE,SAAS,EAAE,UAAU,CAAC,KAAK,EAAE;YACnG,CAAC,CAAC,EAAE,GAAG,QAAQ,EAAE,MAAM,EAAE,QAAiB,EAAE,UAAU,EAAE,GAAG,EAAE,SAAS,EAAE,UAAU,CAAC,KAAK,EAAE,CAAC;IACjG,OAAO,GAAG,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;QACrC,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;QAC/E,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QACpC,IAAI,UAAU,CAAC,IAAI,KAAK,WAAW;YAAE,MAAM,qBAAqB,CAAC,EAAE,GAAG,GAAG,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACxG,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,GAAgB,EAAE,OAAmB;IACpE,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;IAC5B,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,EAAE;SACtB,MAAM,CAAC,IAAI,CAAC;SACZ,GAAG,CAAC;QACH,MAAM,EAAE,SAAS;QACjB,QAAQ,EAAE,GAAG,CAAA,GAAG,IAAI,CAAC,QAAQ,MAAM;QACnC,KAAK,EAAE,GAAG;QACV,WAAW,EAAE,IAAI;QACjB,QAAQ,EAAE,IAAI;QACd,UAAU,EAAE,IAAI;QAChB,SAAS,EAAE,GAAG;KACf,CAAC;SACD,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;SACtB,SAAS,EAAE,CAAC;IACf,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;AACzB,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,GAAgB,EAAE,KAAa,EAAE,GAAS;IAC3E,MAAM,GAAG,CAAC,EAAE;SACT,MAAM,CAAC,IAAI,CAAC;SACZ,GAAG,CAAC;QACH,MAAM,EAAE,QAAQ;QAChB,SAAS,EAAE,aAAa;QACxB,UAAU,EAAE,GAAG;QACf,WAAW,EAAE,IAAI;QACjB,QAAQ,EAAE,IAAI;QACd,UAAU,EAAE,IAAI;QAChB,SAAS,EAAE,GAAG;KACf,CAAC;SACD,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;AACrI,CAAC;AAED,SAAS,MAAM,CAAC,OAAmB;IACjC,OAAO,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;AACvH,CAAC;AAED,SAAS,YAAY,CAAC,KAAoB;IACxC,IAAI,KAAK,KAAK,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IAC1F,OAAO,KAAK,CAAC;AACf,CAAC"}
+{"version":3,"file":"claim.js","sourceRoot":"","sources":["../../src/server/claim.ts"],"names":[],"mappings":"AAAA,kHAAkH;AAClH,qGAAqG;AACrG,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AAEvE,OAAO,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEpC,OAAO,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC;AACrC,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AACxD,OAAO,EAAE,qBAAqB,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAsB9E,MAAM,aAAa,GAAG,oBAAoB,CAAC;AAE3C;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,GAAgB,EAAE,KAAqB;IACrE,MAAM,KAAK,GAAG,QAAQ,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;IAChE,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC;QAAE,OAAO,EAAE,CAAC;IAC/B,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;IAC5B,MAAM,mBAAmB,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAEjD,MAAM,GAAG,GAAG,GAAG,CAAC,EAAE;SACf,MAAM,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC;SACvB,IAAI,CAAC,IAAI,CAAC;SACV,KAAK,CACJ,GAAG,CACD,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,EAC3B,EAAE,CACA,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,EACrD,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,GAAG,CAAA,GAAG,IAAI,CAAC,QAAQ,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC,CACzG,CACF,CACF;SACA,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;SACtC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC;SAClB,GAAG,CAAC,QAAQ,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;IAEvC,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,EAAE;SACtB,MAAM,CAAC,IAAI,CAAC;SACZ,GAAG,CAAC;QACH,MAAM,EAAE,SAAS;QACjB,QAAQ,EAAE,GAAG,CAAA,GAAG,IAAI,CAAC,QAAQ,MAAM;QACnC,WAAW,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC;QAChE,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,UAAU,EAAE,GAAG,CAAA,mBAAmB;QAClC,SAAS,EAAE,GAAG;KACf,CAAC;SACD,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;SAC5B,SAAS,EAAE,CAAC;IAEf,OAAO,IAAI;SACR,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC;SACjG,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,UAAU,EAAE,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC;AACnF,CAAC;AAED,gHAAgH;AAChH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,GAAgB,EAAE,OAAmB;IACxE,MAAM,KAAK,GAAG,QAAQ,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACtE,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;IAC5B,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,EAAE;SACtB,MAAM,CAAC,IAAI,CAAC;SACZ,GAAG,CAAC,EAAE,WAAW,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC;SACzF,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;SACtB,SAAS,EAAE,CAAC;IACf,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;AACzB,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,GAAgB,EAAE,OAAmB,EAAE,UAAsB;IAC7F,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;IAC5B,MAAM,QAAQ,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC;IACzF,MAAM,GAAG,GACP,UAAU,CAAC,IAAI,KAAK,WAAW;QAC7B,CAAC,CAAC,EAAE,GAAG,QAAQ,EAAE,MAAM,EAAE,WAAoB,EAAE,UAAU,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE;QACjF,CAAC,CAAC,UAAU,CAAC,IAAI,KAAK,OAAO;YAC3B,CAAC,CAAC,EAAE,GAAG,QAAQ,EAAE,MAAM,EAAE,SAAkB,EAAE,KAAK,EAAE,UAAU,CAAC,KAAK,EAAE,SAAS,EAAE,UAAU,CAAC,KAAK,EAAE;YACnG,CAAC,CAAC,EAAE,GAAG,QAAQ,EAAE,MAAM,EAAE,QAAiB,EAAE,UAAU,EAAE,GAAG,EAAE,SAAS,EAAE,UAAU,CAAC,KAAK,EAAE,CAAC;IACjG,OAAO,GAAG,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;QACrC,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;QAC/E,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QACpC,IAAI,UAAU,CAAC,IAAI,KAAK,WAAW;YAAE,MAAM,qBAAqB,CAAC,EAAE,GAAG,GAAG,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACxG,IAAI,UAAU,CAAC,IAAI,KAAK,QAAQ;YAAE,MAAM,qBAAqB,CAAC,EAAE,GAAG,GAAG,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC;QACvH,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,GAAgB,EAAE,OAAmB;IACpE,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;IAC5B,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,EAAE;SACtB,MAAM,CAAC,IAAI,CAAC;SACZ,GAAG,CAAC;QACH,MAAM,EAAE,SAAS;QACjB,QAAQ,EAAE,GAAG,CAAA,GAAG,IAAI,CAAC,QAAQ,MAAM;QACnC,KAAK,EAAE,GAAG;QACV,WAAW,EAAE,IAAI;QACjB,QAAQ,EAAE,IAAI;QACd,UAAU,EAAE,IAAI;QAChB,SAAS,EAAE,GAAG;KACf,CAAC;SACD,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;SACtB,SAAS,EAAE,CAAC;IACf,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;AACzB,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,GAAgB,EAAE,KAAa,EAAE,GAAS;IAC3E,MAAM,GAAG,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;QACpC,MAAM,MAAM,GAAG,MAAM,EAAE;aACpB,MAAM,CAAC,IAAI,CAAC;aACZ,GAAG,CAAC;YACH,MAAM,EAAE,QAAQ;YAChB,SAAS,EAAE,aAAa;YACxB,UAAU,EAAE,GAAG;YACf,WAAW,EAAE,IAAI;YACjB,QAAQ,EAAE,IAAI;YACd,UAAU,EAAE,IAAI;YAChB,SAAS,EAAE,GAAG;SACf,CAAC;aACD,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;aAC/H,SAAS,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;QAC9B,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;YAAE,MAAM,qBAAqB,CAAC,EAAE,GAAG,GAAG,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,aAAa,CAAC,CAAC;IAC/F,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,MAAM,CAAC,OAAmB;IACjC,OAAO,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;AACvH,CAAC;AAED,SAAS,YAAY,CAAC,KAAoB;IACxC,IAAI,KAAK,KAAK,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IAC1F,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/server/schedules.d.ts

@@ -13,11 +13,17 @@ export interface ScheduleState {
     readonly lastEnqueuedAt: Date | null;
     /** When a job of the schedule's queue last succeeded; null when none has yet. */
     readonly lastSucceededAt: Date | null;
+    /** When a job of the schedule's queue last ended `failed` (attempts used up, or failed for good); null when none has. */
+    readonly lastFailedAt: Date | null;
+    /** That job's error code (the handler's code, `jobs.handler_threw`, `jobs.lease_expired`…); null when none failed. */
+    readonly lastFailureCode: string | null;
     /**
      * True when no run succeeded within two periods (counted from when a worker first saw the schedule), or no worker
      * has seen it at all: no worker runs its queue, or its runs keep failing.
      */
     readonly isLate: boolean;
+    /** True when the latest outcome is a failure: `lastFailedAt` is later than `lastSucceededAt`, or nothing succeeded. */
+    readonly isFailing: boolean;
 }
 export interface EnqueueDueSchedulesInput {
     /** Only the schedules of these queues; default every schedule. */
@@ -41,6 +47,8 @@ export declare function enqueueDueSchedules(ctx: JobsContext, input?: EnqueueDue
 export declare function enqueueScheduledRun(ctx: JobsContext, name: string, slot: Date): Promise<boolean>;
 /** Notes a succeeded job of a scheduled queue (any job of it, a run enqueued by hand too). */
 export declare function recordScheduleSuccess(ctx: JobsContext, queue: string): Promise<void>;
+/** Notes a failed job of a scheduled queue (terminal, not a retried attempt) with its error code. */
+export declare function recordScheduleFailure(ctx: JobsContext, queue: string, code: string): Promise<void>;
 /** The state of every configured schedule, by queue name. */
 export declare function getScheduleStates(ctx: JobsContext): Promise<Map<string, ScheduleState>>;
 /** The schedules among `queues` (default all), in configuration order. */

package/dist/server/schedules.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"schedules.d.ts","sourceRoot":"","sources":["../../src/server/schedules.ts"],"names":[],"mappings":"AAIA,OAAO,EAAqB,KAAK,kBAAkB,EAAE,KAAK,WAAW,EAAiC,MAAM,eAAe,CAAC;AAG5H,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAIhD;;;GAGG;AACH,eAAO,MAAM,kBAAkB,aAAa,CAAC;AAE7C,yDAAyD;AACzD,MAAM,WAAW,aAAa;IAC5B,gHAAgH;IAChH,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,sGAAsG;IACtG,QAAQ,CAAC,cAAc,EAAE,IAAI,GAAG,IAAI,CAAC;IACrC,iFAAiF;IACjF,QAAQ,CAAC,eAAe,EAAE,IAAI,GAAG,IAAI,CAAC;IACtC;;;OAGG;IACH,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,wBAAwB;IACvC,kEAAkE;IAClE,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACrC;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,kBAAkB,EAAE,GAAG,EAAE,IAAI,GAAG,IAAI,CAI7E;AAED;;;;GAIG;AACH,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,WAAW,EAAE,KAAK,GAAE,wBAA6B,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAQnH;AAED;;;GAGG;AACH,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,CAiBtG;AAED,8FAA8F;AAC9F,wBAAsB,qBAAqB,CAAC,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAI1F;AAED,6DAA6D;AAC7D,wBAAsB,iBAAiB,CAAC,GAAG,EAAE,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAe7F;AAED,0EAA0E;AAC1E,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,SAAS,GAAG,MAAM,EAAE,CAEtG;AAED,wBAAgB,WAAW,CAAC,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,GAAG,kBAAkB,CAIlF"}
+{"version":3,"file":"schedules.d.ts","sourceRoot":"","sources":["../../src/server/schedules.ts"],"names":[],"mappings":"AAIA,OAAO,EAAqB,KAAK,kBAAkB,EAAE,KAAK,WAAW,EAAiC,MAAM,eAAe,CAAC;AAG5H,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAIhD;;;GAGG;AACH,eAAO,MAAM,kBAAkB,aAAa,CAAC;AAE7C,yDAAyD;AACzD,MAAM,WAAW,aAAa;IAC5B,gHAAgH;IAChH,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,sGAAsG;IACtG,QAAQ,CAAC,cAAc,EAAE,IAAI,GAAG,IAAI,CAAC;IACrC,iFAAiF;IACjF,QAAQ,CAAC,eAAe,EAAE,IAAI,GAAG,IAAI,CAAC;IACtC,yHAAyH;IACzH,QAAQ,CAAC,YAAY,EAAE,IAAI,GAAG,IAAI,CAAC;IACnC,sHAAsH;IACtH,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC;;;OAGG;IACH,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,uHAAuH;IACvH,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;CAC7B;AAED,MAAM,WAAW,wBAAwB;IACvC,kEAAkE;IAClE,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACrC;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,kBAAkB,EAAE,GAAG,EAAE,IAAI,GAAG,IAAI,CAI7E;AAED;;;;GAIG;AACH,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,WAAW,EAAE,KAAK,GAAE,wBAA6B,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAQnH;AAED;;;GAGG;AACH,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,CAiBtG;AAED,8FAA8F;AAC9F,wBAAsB,qBAAqB,CAAC,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAI1F;AAED,qGAAqG;AACrG,wBAAsB,qBAAqB,CAAC,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAIxG;AAED,6DAA6D;AAC7D,wBAAsB,iBAAiB,CAAC,GAAG,EAAE,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAmB7F;AAED,0EAA0E;AAC1E,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,SAAS,GAAG,MAAM,EAAE,CAEtG;AAED,wBAAgB,WAAW,CAAC,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,GAAG,kBAAkB,CAIlF"}

package/dist/server/schedules.js

@@ -67,6 +67,13 @@ export async function recordScheduleSuccess(ctx, queue) {
     const now = ctx.clock.now();
     await ctx.db.update(schedules).set({ lastSucceededAt: now, updatedAt: now }).where(eq(schedules.name, queue));
 }
+/** Notes a failed job of a scheduled queue (terminal, not a retried attempt) with its error code. */
+export async function recordScheduleFailure(ctx, queue, code) {
+    if (!Object.hasOwn(getJobsOptions(ctx.config).schedules, queue))
+        return;
+    const now = ctx.clock.now();
+    await ctx.db.update(schedules).set({ lastFailedAt: now, lastFailureCode: code, updatedAt: now }).where(eq(schedules.name, queue));
+}
 /** The state of every configured schedule, by queue name. */
 export async function getScheduleStates(ctx) {
     const options = getJobsOptions(ctx.config);
@@ -80,7 +87,11 @@ export async function getScheduleStates(ctx) {
         const row = rows.find((candidate) => candidate.name === name);
         const since = row?.lastSucceededAt ?? row?.createdAt ?? null;
         const isLate = since === null || now - since.getTime() > 2 * everySeconds * 1000;
-        return [name, { everySeconds, lastEnqueuedAt: row?.lastSlotAt ?? null, lastSucceededAt: row?.lastSucceededAt ?? null, isLate }];
+        const lastSucceededAt = row?.lastSucceededAt ?? null;
+        const lastFailedAt = row?.lastFailedAt ?? null;
+        const isFailing = lastFailedAt !== null && (lastSucceededAt === null || lastFailedAt > lastSucceededAt);
+        const lastFailureCode = row?.lastFailureCode ?? null;
+        return [name, { everySeconds, lastEnqueuedAt: row?.lastSlotAt ?? null, lastSucceededAt, lastFailedAt, lastFailureCode, isLate, isFailing }];
     }));
 }
 /** The schedules among `queues` (default all), in configuration order. */

package/dist/server/schedules.js.map

@@ -1 +1 @@
-{"version":3,"file":"schedules.js","sourceRoot":"","sources":["../../src/server/schedules.ts"],"names":[],"mappings":"AAAA,iHAAiH;AACjH,6LAA6L;AAC7L,6FAA6F;AAC7F,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAA4E,MAAM,eAAe,CAAC;AAC5H,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEjE,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE9C;;;GAGG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,UAAU,CAAC;AAsB7C;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,QAA4B,EAAE,GAAS;IACrE,IAAI,IAAI,IAAI,QAAQ;QAAE,OAAO,IAAI,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IACjF,MAAM,QAAQ,GAAG,eAAe,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC;IAClD,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,QAAQ,CAAC,GAAG,QAAQ,CAAC,CAAC;AACnE,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,GAAgB,EAAE,QAAkC,EAAE;IAC9F,MAAM,OAAO,GAAG,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,KAAK,MAAM,IAAI,IAAI,gBAAgB,CAAC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3D,MAAM,IAAI,GAAG,eAAe,CAAC,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;QAC1E,IAAI,MAAM,mBAAmB,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC;YAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtE,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,GAAgB,EAAE,IAAY,EAAE,IAAU;IAClF,MAAM,QAAQ,GAAG,WAAW,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC;IAC/D,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;IAC5B,OAAO,GAAG,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;QACrC,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,EAAE;aACrB,MAAM,CAAC,SAAS,CAAC;aACjB,MAAM,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC;aAClE,kBAAkB,CAAC;YAClB,MAAM,EAAE,SAAS,CAAC,IAAI;YACtB,GAAG,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE;YACzC,QAAQ,EAAE,EAAE,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,CAAC;SACzC,CAAC;aACD,SAAS,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;QACvC,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO,KAAK,CAAC;QACtC,MAAM,UAAU,CAAC,EAAE,GAAG,GAAG,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,QAAQ,EAAE,kBAAkB,EAAE,CAAC,CAAC;QAC/G,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAED,8FAA8F;AAC9F,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,GAAgB,EAAE,KAAa;IACzE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC;QAAE,OAAO;IACxE,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;IAC5B,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,EAAE,eAAe,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;AAChH,CAAC;AAED,6DAA6D;AAC7D,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,GAAgB;IACtD,MAAM,OAAO,GAAG,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC7C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,GAAG,EAAE,CAAC;IACzC,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;IACzF,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;IACtC,OAAO,IAAI,GAAG,CACZ,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACjB,MAAM,YAAY,GAAG,eAAe,CAAC,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;QACjE,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;QAC9D,MAAM,KAAK,GAAG,GAAG,EAAE,eAAe,IAAI,GAAG,EAAE,SAAS,IAAI,IAAI,CAAC;QAC7D,MAAM,MAAM,GAAG,KAAK,KAAK,IAAI,IAAI,GAAG,GAAG,KAAK,CAAC,OAAO,EAAE,GAAG,CAAC,GAAG,YAAY,GAAG,IAAI,CAAC;QACjF,OAAO,CAAC,IAAI,EAAE,EAAE,YAAY,EAAE,cAAc,EAAE,GAAG,EAAE,UAAU,IAAI,IAAI,EAAE,eAAe,EAAE,GAAG,EAAE,eAAe,IAAI,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;IAClI,CAAC,CAAC,CACH,CAAC;AACJ,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,gBAAgB,CAAC,OAAoB,EAAE,MAAqC;IAC1F,OAAO,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;AACxG,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,OAAoB,EAAE,IAAY;IAC5D,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC9F,IAAI,QAAQ,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,IAAI,gCAAgC,CAAC,CAAC;IACxG,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,eAAe,CAAC,QAA4B;IACnD,IAAI,IAAI,IAAI,QAAQ;QAAE,OAAO,oBAAoB,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC/D,MAAM,OAAO,GAAG,iBAAiB,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAClD,yDAAyD;IACzD,IAAI,OAAO,KAAK,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAC;IACvG,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,gBAAgB,CAAC,QAAkC,EAAE,GAAW;IACvE,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACnD,6FAA6F;IAC7F,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;IACjF,MAAM,IAAI,GAAG;QACX,EAAE,IAAI,EAAE,SAAS,CAAC,cAAc,EAAE,EAAE,KAAK,EAAE,SAAS,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,SAAS,CAAC,UAAU,EAAE,EAAE;QACrG,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE;KACzD,CAAC;IACF,IAAI,IAAI,GAAG,MAAM,CAAC,iBAAiB,CAAC;IACpC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;YAC/B,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACnD,MAAM,OAAO,GAAG,eAAe,CAAC,EAAE,GAAG,GAAG,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACrG,IAAI,OAAO,IAAI,GAAG,IAAI,OAAO,GAAG,IAAI;gBAAE,IAAI,GAAG,OAAO,CAAC;QACvD,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,uGAAuG;AACvG,SAAS,oBAAoB,CAAC,KAAwB;IACpD,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACjC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACnD,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,GAAG,MAAM,GAAG,CAAC,KAAK,KAAK,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACvJ,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;AAChC,CAAC"}
+{"version":3,"file":"schedules.js","sourceRoot":"","sources":["../../src/server/schedules.ts"],"names":[],"mappings":"AAAA,iHAAiH;AACjH,6LAA6L;AAC7L,6FAA6F;AAC7F,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAA4E,MAAM,eAAe,CAAC;AAC5H,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEjE,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE9C;;;GAGG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,UAAU,CAAC;AA4B7C;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,QAA4B,EAAE,GAAS;IACrE,IAAI,IAAI,IAAI,QAAQ;QAAE,OAAO,IAAI,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IACjF,MAAM,QAAQ,GAAG,eAAe,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC;IAClD,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,QAAQ,CAAC,GAAG,QAAQ,CAAC,CAAC;AACnE,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,GAAgB,EAAE,QAAkC,EAAE;IAC9F,MAAM,OAAO,GAAG,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,KAAK,MAAM,IAAI,IAAI,gBAAgB,CAAC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3D,MAAM,IAAI,GAAG,eAAe,CAAC,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;QAC1E,IAAI,MAAM,mBAAmB,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC;YAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtE,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,GAAgB,EAAE,IAAY,EAAE,IAAU;IAClF,MAAM,QAAQ,GAAG,WAAW,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC;IAC/D,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;IAC5B,OAAO,GAAG,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;QACrC,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,EAAE;aACrB,MAAM,CAAC,SAAS,CAAC;aACjB,MAAM,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC;aAClE,kBAAkB,CAAC;YAClB,MAAM,EAAE,SAAS,CAAC,IAAI;YACtB,GAAG,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE;YACzC,QAAQ,EAAE,EAAE,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,CAAC;SACzC,CAAC;aACD,SAAS,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;QACvC,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO,KAAK,CAAC;QACtC,MAAM,UAAU,CAAC,EAAE,GAAG,GAAG,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,QAAQ,EAAE,kBAAkB,EAAE,CAAC,CAAC;QAC/G,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAED,8FAA8F;AAC9F,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,GAAgB,EAAE,KAAa;IACzE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC;QAAE,OAAO;IACxE,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;IAC5B,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,EAAE,eAAe,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;AAChH,CAAC;AAED,qGAAqG;AACrG,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,GAAgB,EAAE,KAAa,EAAE,IAAY;IACvF,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC;QAAE,OAAO;IACxE,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;IAC5B,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,EAAE,YAAY,EAAE,GAAG,EAAE,eAAe,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;AACpI,CAAC;AAED,6DAA6D;AAC7D,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,GAAgB;IACtD,MAAM,OAAO,GAAG,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC7C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,GAAG,EAAE,CAAC;IACzC,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;IACzF,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;IACtC,OAAO,IAAI,GAAG,CACZ,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACjB,MAAM,YAAY,GAAG,eAAe,CAAC,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;QACjE,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;QAC9D,MAAM,KAAK,GAAG,GAAG,EAAE,eAAe,IAAI,GAAG,EAAE,SAAS,IAAI,IAAI,CAAC;QAC7D,MAAM,MAAM,GAAG,KAAK,KAAK,IAAI,IAAI,GAAG,GAAG,KAAK,CAAC,OAAO,EAAE,GAAG,CAAC,GAAG,YAAY,GAAG,IAAI,CAAC;QACjF,MAAM,eAAe,GAAG,GAAG,EAAE,eAAe,IAAI,IAAI,CAAC;QACrD,MAAM,YAAY,GAAG,GAAG,EAAE,YAAY,IAAI,IAAI,CAAC;QAC/C,MAAM,SAAS,GAAG,YAAY,KAAK,IAAI,IAAI,CAAC,eAAe,KAAK,IAAI,IAAI,YAAY,GAAG,eAAe,CAAC,CAAC;QACxG,MAAM,eAAe,GAAG,GAAG,EAAE,eAAe,IAAI,IAAI,CAAC;QACrD,OAAO,CAAC,IAAI,EAAE,EAAE,YAAY,EAAE,cAAc,EAAE,GAAG,EAAE,UAAU,IAAI,IAAI,EAAE,eAAe,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IAC9I,CAAC,CAAC,CACH,CAAC;AACJ,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,gBAAgB,CAAC,OAAoB,EAAE,MAAqC;IAC1F,OAAO,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;AACxG,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,OAAoB,EAAE,IAAY;IAC5D,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC9F,IAAI,QAAQ,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,IAAI,gCAAgC,CAAC,CAAC;IACxG,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,eAAe,CAAC,QAA4B;IACnD,IAAI,IAAI,IAAI,QAAQ;QAAE,OAAO,oBAAoB,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC/D,MAAM,OAAO,GAAG,iBAAiB,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAClD,yDAAyD;IACzD,IAAI,OAAO,KAAK,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAC;IACvG,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,gBAAgB,CAAC,QAAkC,EAAE,GAAW;IACvE,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACnD,6FAA6F;IAC7F,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;IACjF,MAAM,IAAI,GAAG;QACX,EAAE,IAAI,EAAE,SAAS,CAAC,cAAc,EAAE,EAAE,KAAK,EAAE,SAAS,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,SAAS,CAAC,UAAU,EAAE,EAAE;QACrG,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE;KACzD,CAAC;IACF,IAAI,IAAI,GAAG,MAAM,CAAC,iBAAiB,CAAC;IACpC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;YAC/B,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACnD,MAAM,OAAO,GAAG,eAAe,CAAC,EAAE,GAAG,GAAG,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACrG,IAAI,OAAO,IAAI,GAAG,IAAI,OAAO,GAAG,IAAI;gBAAE,IAAI,GAAG,OAAO,CAAC;QACvD,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,uGAAuG;AACvG,SAAS,oBAAoB,CAAC,KAAwB;IACpD,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACjC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACnD,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,GAAG,MAAM,GAAG,CAAC,KAAK,KAAK,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACvJ,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;AAChC,CAAC"}

package/dist/worker/health.d.ts

@@ -0,0 +1,30 @@
+import type { SoftureConfig } from "@softure-ai/core";
+import type { JobsLogger } from "../server/logger.js";
+export type JobWorkerHealth = {
+    ok: true;
+    ageMs: number;
+} | {
+    ok: false;
+    reason: "missing";
+} | {
+    ok: false;
+    reason: "stale";
+    ageMs: number;
+};
+export interface CheckJobWorkerHealthInput {
+    readonly healthFile: string;
+    /** The oldest heartbeat that still counts; `getJobWorkerHealthMaxAgeMs(config)` by default in the process. */
+    readonly maxAgeMs: number;
+    /** Epoch ms to compare the file's modification time with; default `Date.now()`. */
+    readonly now?: number;
+}
+/** The default age limit: five poll intervals, at least 30 s. */
+export declare function getJobWorkerHealthMaxAgeMs(config: SoftureConfig): number;
+/** Whether the heartbeat file exists and is no older than `maxAgeMs`. Reads the file system only. */
+export declare function checkJobWorkerHealth(input: CheckJobWorkerHealthInput): JobWorkerHealth;
+/** Writes and removes the heartbeat file; a failure is logged once until a write succeeds again, never thrown. */
+export declare function createHealthFile(healthFile: string, logger: JobsLogger): {
+    touch: () => void;
+    remove: () => void;
+};
+//# sourceMappingURL=health.d.ts.map

package/dist/worker/health.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"health.d.ts","sourceRoot":"","sources":["../../src/worker/health.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAOtD,MAAM,MAAM,eAAe,GAAG;IAAE,EAAE,EAAE,IAAI,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,MAAM,EAAE,SAAS,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,MAAM,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAE7I,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,8GAA8G;IAC9G,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,mFAAmF;IACnF,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,iEAAiE;AACjE,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,aAAa,GAAG,MAAM,CAExE;AAED,qGAAqG;AACrG,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,yBAAyB,GAAG,eAAe,CAStF;AAED,kHAAkH;AAClH,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,GAAG;IAAE,KAAK,EAAE,MAAM,IAAI,CAAC;IAAC,MAAM,EAAE,MAAM,IAAI,CAAA;CAAE,CAwBlH"}

package/dist/worker/health.js

@@ -0,0 +1,53 @@
+// The worker's heartbeat: a file the loop touches after each round whose claims reached the database, and the check
+// an image runs (`node worker.mjs --health`) to tell a running loop from a process that only stays alive.
+import { rmSync, statSync, writeFileSync } from "node:fs";
+import { getJobsOptions } from "../server/options.js";
+/** The age limit never drops below this, so a slow round (an hourly prune) does not read as a stuck loop. */
+const MIN_HEALTH_MAX_AGE_MS = 30_000;
+const POLL_INTERVALS_PER_MAX_AGE = 5;
+/** The default age limit: five poll intervals, at least 30 s. */
+export function getJobWorkerHealthMaxAgeMs(config) {
+    return Math.max(MIN_HEALTH_MAX_AGE_MS, POLL_INTERVALS_PER_MAX_AGE * getJobsOptions(config).pollIntervalMs);
+}
+/** Whether the heartbeat file exists and is no older than `maxAgeMs`. Reads the file system only. */
+export function checkJobWorkerHealth(input) {
+    let mtimeMs;
+    try {
+        mtimeMs = statSync(input.healthFile).mtimeMs;
+    }
+    catch {
+        return { ok: false, reason: "missing" };
+    }
+    const ageMs = Math.max(0, Math.round((input.now ?? Date.now()) - mtimeMs));
+    return ageMs > input.maxAgeMs ? { ok: false, reason: "stale", ageMs } : { ok: true, ageMs };
+}
+/** Writes and removes the heartbeat file; a failure is logged once until a write succeeds again, never thrown. */
+export function createHealthFile(healthFile, logger) {
+    let hasFailed = false;
+    const report = (operation, error) => {
+        if (hasFailed)
+            return;
+        hasFailed = true;
+        logger.error(`could not ${operation} the worker's health file`, { healthFile, error });
+    };
+    return {
+        touch: () => {
+            try {
+                writeFileSync(healthFile, new Date().toISOString());
+                hasFailed = false;
+            }
+            catch (error) {
+                report("write", error);
+            }
+        },
+        remove: () => {
+            try {
+                rmSync(healthFile, { force: true });
+            }
+            catch (error) {
+                report("remove", error);
+            }
+        },
+    };
+}
+//# sourceMappingURL=health.js.map

package/dist/worker/health.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"health.js","sourceRoot":"","sources":["../../src/worker/health.ts"],"names":[],"mappings":"AAAA,oHAAoH;AACpH,0GAA0G;AAC1G,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAG1D,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAEtD,6GAA6G;AAC7G,MAAM,qBAAqB,GAAG,MAAM,CAAC;AACrC,MAAM,0BAA0B,GAAG,CAAC,CAAC;AAYrC,iEAAiE;AACjE,MAAM,UAAU,0BAA0B,CAAC,MAAqB;IAC9D,OAAO,IAAI,CAAC,GAAG,CAAC,qBAAqB,EAAE,0BAA0B,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,cAAc,CAAC,CAAC;AAC7G,CAAC;AAED,qGAAqG;AACrG,MAAM,UAAU,oBAAoB,CAAC,KAAgC;IACnE,IAAI,OAAe,CAAC;IACpB,IAAI,CAAC;QACH,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC;IAC/C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IAC1C,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAC3E,OAAO,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;AAC9F,CAAC;AAED,kHAAkH;AAClH,MAAM,UAAU,gBAAgB,CAAC,UAAkB,EAAE,MAAkB;IACrE,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,MAAM,MAAM,GAAG,CAAC,SAAiB,EAAE,KAAc,EAAE,EAAE;QACnD,IAAI,SAAS;YAAE,OAAO;QACtB,SAAS,GAAG,IAAI,CAAC;QACjB,MAAM,CAAC,KAAK,CAAC,aAAa,SAAS,2BAA2B,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;IACzF,CAAC,CAAC;IACF,OAAO;QACL,KAAK,EAAE,GAAG,EAAE;YACV,IAAI,CAAC;gBACH,aAAa,CAAC,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;gBACpD,SAAS,GAAG,KAAK,CAAC;YACpB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;QACD,MAAM,EAAE,GAAG,EAAE;YACX,IAAI,CAAC;gBACH,MAAM,CAAC,UAAU,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACtC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/worker/index.d.ts

@@ -1,3 +1,4 @@
+export { checkJobWorkerHealth, getJobWorkerHealthMaxAgeMs, type CheckJobWorkerHealthInput, type JobWorkerHealth } from "./health.js";
 export { runJobWorkerProcess, type RunJobWorkerProcessOptions } from "./process.js";
 export { runJobWorker, type RunJobWorkerOptions } from "./run-job-worker.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/worker/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/worker/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,mBAAmB,EAAE,KAAK,0BAA0B,EAAE,MAAM,cAAc,CAAC;AACpF,OAAO,EAAE,YAAY,EAAE,KAAK,mBAAmB,EAAE,MAAM,qBAAqB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/worker/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,KAAK,yBAAyB,EAAE,KAAK,eAAe,EAAE,MAAM,aAAa,CAAC;AACrI,OAAO,EAAE,mBAAmB,EAAE,KAAK,0BAA0B,EAAE,MAAM,cAAc,CAAC;AACpF,OAAO,EAAE,YAAY,EAAE,KAAK,mBAAmB,EAAE,MAAM,qBAAqB,CAAC"}

package/dist/worker/index.js

@@ -1,5 +1,6 @@
-// The worker of @softure-ai/jobs: the loop (`runJobWorker`) and the process entry an app's `scripts/worker.ts` calls
-// (`runJobWorkerProcess`).
+// The worker of @softure-ai/jobs: the loop (`runJobWorker`), the process entry an app's `scripts/worker.ts` calls
+// (`runJobWorkerProcess`) and the check of its heartbeat (`checkJobWorkerHealth`).
+export { checkJobWorkerHealth, getJobWorkerHealthMaxAgeMs } from "./health.js";
 export { runJobWorkerProcess } from "./process.js";
 export { runJobWorker } from "./run-job-worker.js";
 //# sourceMappingURL=index.js.map

package/dist/worker/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/worker/index.ts"],"names":[],"mappings":"AAAA,qHAAqH;AACrH,2BAA2B;AAC3B,OAAO,EAAE,mBAAmB,EAAmC,MAAM,cAAc,CAAC;AACpF,OAAO,EAAE,YAAY,EAA4B,MAAM,qBAAqB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/worker/index.ts"],"names":[],"mappings":"AAAA,kHAAkH;AAClH,mFAAmF;AACnF,OAAO,EAAE,oBAAoB,EAAE,0BAA0B,EAAwD,MAAM,aAAa,CAAC;AACrI,OAAO,EAAE,mBAAmB,EAAmC,MAAM,cAAc,CAAC;AACpF,OAAO,EAAE,YAAY,EAA4B,MAAM,qBAAqB,CAAC"}

package/dist/worker/process.d.ts

@@ -3,7 +3,7 @@ import { type Database } from "@softure-ai/db";
 import type { JobHandlers } from "../handlers.js";
 import { type JobsLogger } from "../server/logger.js";
 import { type RunJobWorkerOptions } from "./run-job-worker.js";
-export interface RunJobWorkerProcessOptions extends Partial<Pick<RunJobWorkerOptions, "queues" | "clock" | "workerId" | "random" | "heartbeatMs">> {
+export interface RunJobWorkerProcessOptions extends Partial<Pick<RunJobWorkerOptions, "queues" | "clock" | "workerId" | "random" | "heartbeatMs" | "healthFile">> {
     readonly config: SoftureConfig;
     readonly handlers: JobHandlers;
     readonly logger?: JobsLogger;
@@ -11,10 +11,16 @@ export interface RunJobWorkerProcessOptions extends Partial<Pick<RunJobWorkerOpt
     readonly database?: Database;
     /** Stops the worker instead of SIGTERM and SIGINT (tests). */
     readonly signal?: AbortSignal;
+    /** The oldest heartbeat `--health` accepts; default `getJobWorkerHealthMaxAgeMs(config)` (five poll intervals, at least 30 s). */
+    readonly healthMaxAgeMs?: number;
+    /** The process arguments; default `process.argv.slice(2)`. `--health` checks `healthFile` instead of running. */
+    readonly argv?: readonly string[];
 }
 /**
  * Runs the worker until SIGTERM or SIGINT, then stops gracefully. Resolves to the exit code and sets
- * `process.exitCode`: 0 after a clean stop, 1 when it could not start or the database could not be opened.
+ * `process.exitCode`: 0 after a clean stop, 1 when it could not start or the database could not be opened. Started
+ * with `--health`, it opens nothing and checks the running worker's `healthFile` instead: 0 when its loop ran within
+ * `healthMaxAgeMs`, else 1.
  *
  *   // scripts/worker.ts
  *   import { runJobWorkerProcess } from "@softure-ai/jobs/worker";

package/dist/worker/process.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"process.d.ts","sourceRoot":"","sources":["../../src/worker/process.ts"],"names":[],"mappings":"AAEA,OAAO,EAAiB,KAAK,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACrE,OAAO,EAAuB,KAAK,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AACpE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAqB,KAAK,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEzE,OAAO,EAAgB,KAAK,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAM7E,MAAM,WAAW,0BAA2B,SAAQ,OAAO,CAAC,IAAI,CAAC,mBAAmB,EAAE,QAAQ,GAAG,OAAO,GAAG,UAAU,GAAG,QAAQ,GAAG,aAAa,CAAC,CAAC;IAChJ,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC;IAC/B,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC;IAC/B,QAAQ,CAAC,MAAM,CAAC,EAAE,UAAU,CAAC;IAC7B,gFAAgF;IAChF,QAAQ,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC;IAC7B,8DAA8D;IAC9D,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;CAC/B;AAED;;;;;;;;;GASG;AACH,wBAAsB,mBAAmB,CAAC,OAAO,EAAE,0BAA0B,GAAG,OAAO,CAAC,MAAM,CAAC,CAI9F"}
+{"version":3,"file":"process.d.ts","sourceRoot":"","sources":["../../src/worker/process.ts"],"names":[],"mappings":"AAGA,OAAO,EAAiB,KAAK,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACrE,OAAO,EAAuB,KAAK,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AACpE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAqB,KAAK,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAGzE,OAAO,EAAgB,KAAK,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAO7E,MAAM,WAAW,0BACf,SAAQ,OAAO,CAAC,IAAI,CAAC,mBAAmB,EAAE,QAAQ,GAAG,OAAO,GAAG,UAAU,GAAG,QAAQ,GAAG,aAAa,GAAG,YAAY,CAAC,CAAC;IACrH,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC;IAC/B,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC;IAC/B,QAAQ,CAAC,MAAM,CAAC,EAAE,UAAU,CAAC;IAC7B,gFAAgF;IAChF,QAAQ,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC;IAC7B,8DAA8D;IAC9D,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IAC9B,kIAAkI;IAClI,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC,iHAAiH;IACjH,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACnC;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,mBAAmB,CAAC,OAAO,EAAE,0BAA0B,GAAG,OAAO,CAAC,MAAM,CAAC,CAK9F"}

package/dist/worker/process.js

@@ -1,16 +1,21 @@
 // The worker as a process: what an app's `scripts/worker.ts` calls. It opens its own database pool, turns SIGTERM and
-// SIGINT into a graceful stop, closes the pool and reports an exit code.
+// SIGINT into a graceful stop, closes the pool and reports an exit code. With `--health` it checks the heartbeat file
+// of a running worker instead.
 import { errorLogLabel } from "@softure-ai/core";
 import { openCommandDatabase } from "@softure-ai/db";
 import { consoleJobsLogger } from "../server/logger.js";
 import { getJobsOptions } from "../server/options.js";
+import { checkJobWorkerHealth, getJobWorkerHealthMaxAgeMs } from "./health.js";
 import { runJobWorker } from "./run-job-worker.js";
 const EXIT_OK = 0;
 const EXIT_FAILED = 1;
 const STOP_SIGNALS = ["SIGTERM", "SIGINT"];
+const HEALTH_FLAG = "--health";
 /**
  * Runs the worker until SIGTERM or SIGINT, then stops gracefully. Resolves to the exit code and sets
- * `process.exitCode`: 0 after a clean stop, 1 when it could not start or the database could not be opened.
+ * `process.exitCode`: 0 after a clean stop, 1 when it could not start or the database could not be opened. Started
+ * with `--health`, it opens nothing and checks the running worker's `healthFile` instead: 0 when its loop ran within
+ * `healthMaxAgeMs`, else 1.
  *
  *   // scripts/worker.ts
  *   import { runJobWorkerProcess } from "@softure-ai/jobs/worker";
@@ -19,10 +24,29 @@ const STOP_SIGNALS = ["SIGTERM", "SIGINT"];
  *   await runJobWorkerProcess({ config, handlers });
  */
 export async function runJobWorkerProcess(options) {
-    const exitCode = await runProcess(options);
+    const argv = options.argv ?? process.argv.slice(2);
+    const exitCode = argv.includes(HEALTH_FLAG) ? checkHealth(options) : await runProcess(options);
     process.exitCode = exitCode;
     return exitCode;
 }
+function checkHealth(options) {
+    const logger = options.logger ?? consoleJobsLogger;
+    if (options.healthFile === undefined) {
+        logger.error(`${HEALTH_FLAG} needs the worker's healthFile option; pass healthFile to runJobWorkerProcess`);
+        return EXIT_FAILED;
+    }
+    const maxAgeMs = options.healthMaxAgeMs ?? getJobWorkerHealthMaxAgeMs(options.config);
+    const health = checkJobWorkerHealth({ healthFile: options.healthFile, maxAgeMs });
+    if (health.ok) {
+        logger.info(`the worker's loop ran ${String(health.ageMs)} ms ago`);
+        return EXIT_OK;
+    }
+    if (health.reason === "missing")
+        logger.error(`the worker's health file is missing: ${options.healthFile}`);
+    else
+        logger.error(`the worker's loop last ran ${String(health.ageMs)} ms ago (limit ${String(maxAgeMs)} ms)`);
+    return EXIT_FAILED;
+}
 async function runProcess(options) {
     const logger = options.logger ?? consoleJobsLogger;
     const database = options.config.database;

package/dist/worker/process.js.map

@@ -1 +1 @@
-{"version":3,"file":"process.js","sourceRoot":"","sources":["../../src/worker/process.ts"],"names":[],"mappings":"AAAA,sHAAsH;AACtH,yEAAyE;AACzE,OAAO,EAAE,aAAa,EAAsB,MAAM,kBAAkB,CAAC;AACrE,OAAO,EAAE,mBAAmB,EAAiB,MAAM,gBAAgB,CAAC;AAEpE,OAAO,EAAE,iBAAiB,EAAmB,MAAM,qBAAqB,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,YAAY,EAA4B,MAAM,qBAAqB,CAAC;AAE7E,MAAM,OAAO,GAAG,CAAC,CAAC;AAClB,MAAM,WAAW,GAAG,CAAC,CAAC;AACtB,MAAM,YAAY,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAU,CAAC;AAYpD;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,OAAmC;IAC3E,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAC;IAC3C,OAAO,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC5B,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,OAAmC;IAC3D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,iBAAiB,CAAC;IACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC;IACzC,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;QACxD,MAAM,CAAC,KAAK,CAAC,gEAAgE,CAAC,CAAC;QAC/E,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,IAAI,MAAoD,CAAC;IACzD,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IACjD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,6BAA6B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACvD,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,IAAI,GAAG,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IACtC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IAChC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,KAAK,MAAM,MAAM,IAAI,YAAY;YAAE,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAChE,CAAC;SAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;QAC5B,IAAI,EAAE,CAAC;IACT,CAAC;SAAM,CAAC;QACN,QAAQ,CAAC,gBAAgB,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3D,CAAC;IAED,IAAI,CAAC;QACH,MAAM,YAAY,CAAC,EAAE,GAAG,OAAO,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QACrF,OAAO,OAAO,CAAC;IACjB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,sBAAsB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/G,OAAO,WAAW,CAAC;IACrB,CAAC;YAAS,CAAC;QACT,KAAK,MAAM,MAAM,IAAI,YAAY;YAAE,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACxE,QAAQ,EAAE,mBAAmB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QAC7C,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;IAC1G,CAAC;AACH,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,OAAmC,EACnC,QAAmC;IAEnC,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS;QAAE,OAAO,EAAE,EAAE,EAAE,OAAO,CAAC,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;IACpG,IAAI,QAAQ,KAAK,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC;IACtD,8EAA8E;IAC9E,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IAC5H,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,WAAW,GAAG,CAAC,EAAE,CAAC,CAAC;IAC9E,OAAO,EAAE,EAAE,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;AACzD,CAAC"}
+{"version":3,"file":"process.js","sourceRoot":"","sources":["../../src/worker/process.ts"],"names":[],"mappings":"AAAA,sHAAsH;AACtH,sHAAsH;AACtH,+BAA+B;AAC/B,OAAO,EAAE,aAAa,EAAsB,MAAM,kBAAkB,CAAC;AACrE,OAAO,EAAE,mBAAmB,EAAiB,MAAM,gBAAgB,CAAC;AAEpE,OAAO,EAAE,iBAAiB,EAAmB,MAAM,qBAAqB,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,aAAa,CAAC;AAC/E,OAAO,EAAE,YAAY,EAA4B,MAAM,qBAAqB,CAAC;AAE7E,MAAM,OAAO,GAAG,CAAC,CAAC;AAClB,MAAM,WAAW,GAAG,CAAC,CAAC;AACtB,MAAM,YAAY,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAU,CAAC;AACpD,MAAM,WAAW,GAAG,UAAU,CAAC;AAiB/B;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,OAAmC;IAC3E,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,UAAU,CAAC,OAAO,CAAC,CAAC;IAC/F,OAAO,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC5B,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,WAAW,CAAC,OAAmC;IACtD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,iBAAiB,CAAC;IACnD,IAAI,OAAO,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QACrC,MAAM,CAAC,KAAK,CAAC,GAAG,WAAW,+EAA+E,CAAC,CAAC;QAC5G,OAAO,WAAW,CAAC;IACrB,CAAC;IACD,MAAM,QAAQ,GAAG,OAAO,CAAC,cAAc,IAAI,0BAA0B,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACtF,MAAM,MAAM,GAAG,oBAAoB,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,QAAQ,EAAE,CAAC,CAAC;IAClF,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;QACd,MAAM,CAAC,IAAI,CAAC,yBAAyB,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QACpE,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS;QAAE,MAAM,CAAC,KAAK,CAAC,wCAAwC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;;QACvG,MAAM,CAAC,KAAK,CAAC,8BAA8B,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC9G,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,OAAmC;IAC3D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,iBAAiB,CAAC;IACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC;IACzC,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;QACxD,MAAM,CAAC,KAAK,CAAC,gEAAgE,CAAC,CAAC;QAC/E,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,IAAI,MAAoD,CAAC;IACzD,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IACjD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,6BAA6B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACvD,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,IAAI,GAAG,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IACtC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IAChC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,KAAK,MAAM,MAAM,IAAI,YAAY;YAAE,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAChE,CAAC;SAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;QAC5B,IAAI,EAAE,CAAC;IACT,CAAC;SAAM,CAAC;QACN,QAAQ,CAAC,gBAAgB,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3D,CAAC;IAED,IAAI,CAAC;QACH,MAAM,YAAY,CAAC,EAAE,GAAG,OAAO,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QACrF,OAAO,OAAO,CAAC;IACjB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,sBAAsB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/G,OAAO,WAAW,CAAC;IACrB,CAAC;YAAS,CAAC;QACT,KAAK,MAAM,MAAM,IAAI,YAAY;YAAE,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACxE,QAAQ,EAAE,mBAAmB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QAC7C,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;IAC1G,CAAC;AACH,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,OAAmC,EACnC,QAAmC;IAEnC,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS;QAAE,OAAO,EAAE,EAAE,EAAE,OAAO,CAAC,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;IACpG,IAAI,QAAQ,KAAK,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC;IACtD,8EAA8E;IAC9E,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IAC5H,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,WAAW,GAAG,CAAC,EAAE,CAAC,CAAC;IAC9E,OAAO,EAAE,EAAE,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;AACzD,CAAC"}

package/dist/worker/run-job-worker.d.ts

@@ -19,6 +19,11 @@ export interface RunJobWorkerOptions {
     readonly random?: () => number;
     /** How often a running job's lease is extended; default a third of the queue's `leaseSeconds`. */
     readonly heartbeatMs?: number;
+    /**
+     * A file the loop touches after each round whose claims reached the database (removed at start and after a clean
+     * stop), for a health check of the loop: `checkJobWorkerHealth`. Default none.
+     */
+    readonly healthFile?: string;
 }
 /**
  * Runs until `signal` aborts, then resolves once every job finished or was released. Throws before starting on a

package/dist/worker/run-job-worker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run-job-worker.d.ts","sourceRoot":"","sources":["../../src/worker/run-job-worker.ts"],"names":[],"mappings":"AAIA,OAAO,EAAe,KAAK,KAAK,EAAE,KAAK,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAC/E,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAIlD,OAAO,EAAqB,KAAK,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAUzE,MAAM,WAAW,mBAAmB;IAClC,oHAAoH;IACpH,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC;IAC/B,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC;IAC/B,gGAAgG;IAChG,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,wEAAwE;IACxE,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACpC,QAAQ,CAAC,MAAM,CAAC,EAAE,UAAU,CAAC;IAC7B,QAAQ,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC;IACvB,yDAAyD;IACzD,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,6CAA6C;IAC7C,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,MAAM,CAAC;IAC/B,kGAAkG;IAClG,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CAC/B;AAQD;;;;GAIG;AACH,wBAAsB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC,CA0F9E"}
+{"version":3,"file":"run-job-worker.d.ts","sourceRoot":"","sources":["../../src/worker/run-job-worker.ts"],"names":[],"mappings":"AAIA,OAAO,EAAe,KAAK,KAAK,EAAE,KAAK,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAC/E,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAIlD,OAAO,EAAqB,KAAK,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAWzE,MAAM,WAAW,mBAAmB;IAClC,oHAAoH;IACpH,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC;IAC/B,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC;IAC/B,gGAAgG;IAChG,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,wEAAwE;IACxE,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACpC,QAAQ,CAAC,MAAM,CAAC,EAAE,UAAU,CAAC;IAC7B,QAAQ,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC;IACvB,yDAAyD;IACzD,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,6CAA6C;IAC7C,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,MAAM,CAAC;IAC/B,kGAAkG;IAClG,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B;;;OAGG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAC9B;AAQD;;;;GAIG;AACH,wBAAsB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC,CAiG9E"}

package/dist/worker/run-job-worker.js

@@ -10,6 +10,7 @@ import { pruneJobs } from "../server/maintenance.js";
 import { getJobsOptions, getQueue } from "../server/options.js";
 import { runClaimedJob } from "../server/process.js";
 import { enqueueScheduledRun, getSchedule, getScheduleNames, getScheduleSlot } from "../server/schedules.js";
+import { createHealthFile } from "./health.js";
 const PRUNE_INTERVAL_MS = 60 * 60 * 1000;
 /** A lease is extended three times per lease period, so one slow write does not lose it. */
 const HEARTBEATS_PER_LEASE = 3;
@@ -38,6 +39,9 @@ export async function runJobWorker(options) {
     if (idle.length > 0 && options.queues === undefined) {
         logger.warn("configured queues without a handler are not run by this worker", { queues: idle.join(", ") });
     }
+    const healthFile = options.healthFile === undefined ? undefined : createHealthFile(options.healthFile, logger);
+    // A file left by an earlier run of the same container must not pass the check before this loop ran a round.
+    healthFile?.remove();
     logger.info("worker started", { workerId, queues: queues.join(", ") });
     const start = (claimed) => {
         const controller = new AbortController();
@@ -78,6 +82,7 @@ export async function runJobWorker(options) {
                 logger.error("could not enqueue a scheduled run; trying again next round", { queue: name, error });
             }
         }
+        let isClaimFailed = false;
         for (const queue of queues) {
             const slots = running.get(queue) ?? new Set();
             const free = getQueue(jobsOptions, queue).concurrency - slots.size;
@@ -89,6 +94,7 @@ export async function runJobWorker(options) {
                     slots.add(start(job));
             }
             catch (error) {
+                isClaimFailed = true;
                 logger.error("could not claim jobs", { queue, error });
             }
         }
@@ -103,9 +109,12 @@ export async function runJobWorker(options) {
                 logger.error("could not prune finished jobs", { error });
             }
         }
+        if (!isClaimFailed)
+            healthFile?.touch();
         await wakeUp.wait(jobsOptions.pollIntervalMs, options.signal);
     }
     await stop(running, { ctx, logger, graceMs: jobsOptions.shutdownGraceMs });
+    healthFile?.remove();
     logger.info("worker stopped", { workerId });
 }
 /**

package/dist/worker/run-job-worker.js.map

@@ -1 +1 @@
-{"version":3,"file":"run-job-worker.js","sourceRoot":"","sources":["../../src/worker/run-job-worker.ts"],"names":[],"mappings":"AAAA,oHAAoH;AACpH,sHAAsH;AACtH,sFAAsF;AACtF,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,WAAW,EAAkC,MAAM,kBAAkB,CAAC;AAG/E,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,UAAU,EAAmB,MAAM,oBAAoB,CAAC;AAE5F,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,iBAAiB,EAAmB,MAAM,qBAAqB,CAAC;AACzE,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChE,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,mBAAmB,EAAE,WAAW,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAE7G,MAAM,iBAAiB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AACzC,4FAA4F;AAC5F,MAAM,oBAAoB,GAAG,CAAC,CAAC;AA2B/B;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAA4B;IAC7D,MAAM,WAAW,GAAG,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,aAAa,CAAC,WAAW,EAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAC5E,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,oGAAoG,CAAC,CAAC;IACxH,CAAC;IACD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,iBAAiB,CAAC;IACnD,MAAM,GAAG,GAAgB,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;IACzG,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,GAAG,QAAQ,EAAE,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;IAC5E,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC;IAC7C,MAAM,OAAO,GAAG,IAAI,GAAG,CAA0B,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5F,MAAM,MAAM,GAAG,YAAY,EAAE,CAAC;IAC9B,IAAI,WAAW,GAAG,MAAM,CAAC,iBAAiB,CAAC;IAC3C,wHAAwH;IACxH,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC/C,MAAM,aAAa,GAAG,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IAE5D,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;IACxF,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QACpD,MAAM,CAAC,IAAI,CAAC,gEAAgE,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC7G,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEvE,MAAM,KAAK,GAAG,CAAC,OAAmB,EAAc,EAAE;QAChD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,oBAAoB,CAAC;QACjI,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE;YACjC,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,IAAI,CAC/B,CAAC,MAAM,EAAE,EAAE;gBACT,IAAI,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;oBAC1C,MAAM,CAAC,IAAI,CAAC,wCAAwC,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;oBAC3G,UAAU,CAAC,KAAK,EAAE,CAAC;gBACrB,CAAC;YACH,CAAC,EACD,CAAC,KAAc,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAC7H,CAAC;QACJ,CAAC,EAAE,WAAW,CAAC,CAAC;QAChB,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC;aAChH,IAAI,CACH,CAAC,OAAO,EAAE,EAAE;YACV,IAAI,OAAO,CAAC,IAAI,KAAK,OAAO;gBAAE,MAAM,CAAC,IAAI,CAAC,8DAA8D,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;QACjK,CAAC,EACD,CAAC,KAAc,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,mEAAmE,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAClK;aACA,OAAO,CAAC,GAAG,EAAE;YACZ,aAAa,CAAC,SAAS,CAAC,CAAC;YACzB,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;YAC9C,MAAM,CAAC,IAAI,EAAE,CAAC;QAChB,CAAC,CAAC,CAAC;QACL,MAAM,KAAK,GAAe,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;QACxD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;IAEF,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,eAAe,CAAC,WAAW,CAAC,WAAW,EAAE,IAAI,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;YAC9E,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,iBAAiB,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE;gBAAE,SAAS;YACrF,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,mBAAmB,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;gBAC9D,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;gBACvC,IAAI,UAAU;oBAAE,MAAM,CAAC,IAAI,CAAC,wBAAwB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;YACnG,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,4DAA4D,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YACrG,CAAC;QACH,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,GAAG,EAAc,CAAC;YAC1D,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC;YACnE,IAAI,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO;gBAAE,SAAS;YAClD,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;gBACvE,KAAK,MAAM,GAAG,IAAI,OAAO;oBAAE,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;YACnD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,sBAAsB,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;YACzD,CAAC;QACH,CAAC;QACD,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,GAAG,WAAW,IAAI,iBAAiB,EAAE,CAAC;YACjE,WAAW,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;YACxC,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,CAAC;gBACrC,IAAI,OAAO,GAAG,CAAC;oBAAE,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACpE,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC;QACD,MAAM,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,cAAc,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,IAAI,CAAC,OAAO,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,CAAC,eAAe,EAAE,CAAC,CAAC;IAC3E,MAAM,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;AAC9C,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,IAAI,CAAC,OAA6C,EAAE,OAAkE;IACnI,MAAM,IAAI,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;IAClE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAC9B,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,2CAA2C,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IACrH,IAAI,KAAiC,CAAC;IACtC,MAAM,KAAK,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAC1C,KAAK,GAAG,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;IACtE,YAAY,CAAC,KAAK,CAAC,CAAC;IAEpB,MAAM,UAAU,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;IACxE,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,GAAG,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;YAC9D,IAAI,UAAU;gBAAE,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,iCAAiC,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;QACtI,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,4DAA4D,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACzJ,CAAC;IACH,CAAC;AACH,CAAC;AAED,gFAAgF;AAChF,SAAS,YAAY;IACnB,IAAI,OAAiC,CAAC;IACtC,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,OAAO;QACL,IAAI,EAAE,GAAG,EAAE;YACT,OAAO,GAAG,IAAI,CAAC;YACf,OAAO,EAAE,EAAE,CAAC;QACd,CAAC;QACD,IAAI,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,CACnB,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAC5B,IAAI,OAAO,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBAC9B,OAAO,GAAG,KAAK,CAAC;gBAChB,OAAO,EAAE,CAAC;gBACV,OAAO;YACT,CAAC;YACD,MAAM,MAAM,GAAG,GAAG,EAAE;gBAClB,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;gBAC5C,OAAO,GAAG,SAAS,CAAC;gBACpB,OAAO,GAAG,KAAK,CAAC;gBAChB,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC;YACF,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACrC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YACzD,OAAO,GAAG,MAAM,CAAC;QACnB,CAAC,CAAC;KACL,CAAC;AACJ,CAAC"}
+{"version":3,"file":"run-job-worker.js","sourceRoot":"","sources":["../../src/worker/run-job-worker.ts"],"names":[],"mappings":"AAAA,oHAAoH;AACpH,sHAAsH;AACtH,sFAAsF;AACtF,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,WAAW,EAAkC,MAAM,kBAAkB,CAAC;AAG/E,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,UAAU,EAAmB,MAAM,oBAAoB,CAAC;AAE5F,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,iBAAiB,EAAmB,MAAM,qBAAqB,CAAC;AACzE,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChE,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,mBAAmB,EAAE,WAAW,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAC7G,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAE/C,MAAM,iBAAiB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AACzC,4FAA4F;AAC5F,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAgC/B;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAA4B;IAC7D,MAAM,WAAW,GAAG,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,aAAa,CAAC,WAAW,EAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAC5E,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,oGAAoG,CAAC,CAAC;IACxH,CAAC;IACD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,iBAAiB,CAAC;IACnD,MAAM,GAAG,GAAgB,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;IACzG,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,GAAG,QAAQ,EAAE,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;IAC5E,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC;IAC7C,MAAM,OAAO,GAAG,IAAI,GAAG,CAA0B,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5F,MAAM,MAAM,GAAG,YAAY,EAAE,CAAC;IAC9B,IAAI,WAAW,GAAG,MAAM,CAAC,iBAAiB,CAAC;IAC3C,wHAAwH;IACxH,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC/C,MAAM,aAAa,GAAG,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IAE5D,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;IACxF,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QACpD,MAAM,CAAC,IAAI,CAAC,gEAAgE,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC7G,CAAC;IACD,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,gBAAgB,CAAC,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IAC/G,4GAA4G;IAC5G,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,MAAM,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEvE,MAAM,KAAK,GAAG,CAAC,OAAmB,EAAc,EAAE;QAChD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,oBAAoB,CAAC;QACjI,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE;YACjC,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,IAAI,CAC/B,CAAC,MAAM,EAAE,EAAE;gBACT,IAAI,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;oBAC1C,MAAM,CAAC,IAAI,CAAC,wCAAwC,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;oBAC3G,UAAU,CAAC,KAAK,EAAE,CAAC;gBACrB,CAAC;YACH,CAAC,EACD,CAAC,KAAc,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAC7H,CAAC;QACJ,CAAC,EAAE,WAAW,CAAC,CAAC;QAChB,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC;aAChH,IAAI,CACH,CAAC,OAAO,EAAE,EAAE;YACV,IAAI,OAAO,CAAC,IAAI,KAAK,OAAO;gBAAE,MAAM,CAAC,IAAI,CAAC,8DAA8D,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;QACjK,CAAC,EACD,CAAC,KAAc,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,mEAAmE,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAClK;aACA,OAAO,CAAC,GAAG,EAAE;YACZ,aAAa,CAAC,SAAS,CAAC,CAAC;YACzB,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;YAC9C,MAAM,CAAC,IAAI,EAAE,CAAC;QAChB,CAAC,CAAC,CAAC;QACL,MAAM,KAAK,GAAe,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;QACxD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;IAEF,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,eAAe,CAAC,WAAW,CAAC,WAAW,EAAE,IAAI,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;YAC9E,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,iBAAiB,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE;gBAAE,SAAS;YACrF,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,mBAAmB,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;gBAC9D,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;gBACvC,IAAI,UAAU;oBAAE,MAAM,CAAC,IAAI,CAAC,wBAAwB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;YACnG,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,4DAA4D,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YACrG,CAAC;QACH,CAAC;QACD,IAAI,aAAa,GAAG,KAAK,CAAC;QAC1B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,GAAG,EAAc,CAAC;YAC1D,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC;YACnE,IAAI,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO;gBAAE,SAAS;YAClD,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;gBACvE,KAAK,MAAM,GAAG,IAAI,OAAO;oBAAE,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;YACnD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,aAAa,GAAG,IAAI,CAAC;gBACrB,MAAM,CAAC,KAAK,CAAC,sBAAsB,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;YACzD,CAAC;QACH,CAAC;QACD,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,GAAG,WAAW,IAAI,iBAAiB,EAAE,CAAC;YACjE,WAAW,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;YACxC,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,CAAC;gBACrC,IAAI,OAAO,GAAG,CAAC;oBAAE,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACpE,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC;QACD,IAAI,CAAC,aAAa;YAAE,UAAU,EAAE,KAAK,EAAE,CAAC;QACxC,MAAM,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,cAAc,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,IAAI,CAAC,OAAO,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,CAAC,eAAe,EAAE,CAAC,CAAC;IAC3E,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,MAAM,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;AAC9C,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,IAAI,CAAC,OAA6C,EAAE,OAAkE;IACnI,MAAM,IAAI,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;IAClE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAC9B,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,2CAA2C,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IACrH,IAAI,KAAiC,CAAC;IACtC,MAAM,KAAK,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAC1C,KAAK,GAAG,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;IACtE,YAAY,CAAC,KAAK,CAAC,CAAC;IAEpB,MAAM,UAAU,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;IACxE,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,GAAG,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;YAC9D,IAAI,UAAU;gBAAE,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,iCAAiC,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;QACtI,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,4DAA4D,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACzJ,CAAC;IACH,CAAC;AACH,CAAC;AAED,gFAAgF;AAChF,SAAS,YAAY;IACnB,IAAI,OAAiC,CAAC;IACtC,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,OAAO;QACL,IAAI,EAAE,GAAG,EAAE;YACT,OAAO,GAAG,IAAI,CAAC;YACf,OAAO,EAAE,EAAE,CAAC;QACd,CAAC;QACD,IAAI,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,CACnB,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAC5B,IAAI,OAAO,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBAC9B,OAAO,GAAG,KAAK,CAAC;gBAChB,OAAO,EAAE,CAAC;gBACV,OAAO;YACT,CAAC;YACD,MAAM,MAAM,GAAG,GAAG,EAAE;gBAClB,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;gBAC5C,OAAO,GAAG,SAAS,CAAC;gBACpB,OAAO,GAAG,KAAK,CAAC;gBAChB,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC;YACF,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACrC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YACzD,OAAO,GAAG,MAAM,CAAC;QACnB,CAAC,CAAC;KACL,CAAC;AACJ,CAAC"}

package/migrations/0003_add_schedule_failures.sql

@@ -0,0 +1,8 @@
+-- The last terminal failure of a scheduled queue, so getQueueStats can tell whether it fails since its last success.
+-- Rollback: ALTER TABLE jobs.schedules DROP COLUMN last_failed_at, DROP COLUMN last_failure_code; then
+--   DELETE FROM softure.migrations WHERE module = 'jobs' AND version = 3;
+ALTER TABLE schedules
+  -- When a job of the schedule's queue last ended `failed`.
+  ADD COLUMN last_failed_at timestamptz,
+  -- That job's error code (`last_error`), never an error's text.
+  ADD COLUMN last_failure_code text;

package/module.json

@@ -1,6 +1,6 @@
 {
   "id": "jobs",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "dependsOn": {},
   "dbSchema": "jobs",
   "tables": ["jobs", "schedules"],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@softure-ai/jobs",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "description": "SOFTURE AI jobs module: background jobs in Postgres. Enqueue inside the caller's transaction with an optional dedup key, claim with FOR UPDATE SKIP LOCKED under a lease, fenced completion, retry with exponential backoff and a terminal failed state, a worker process with graceful shutdown, and an in-process runner for tests.",
   "license": "MIT",
   "type": "module",

package/src/index.ts

@@ -15,7 +15,7 @@ export const MODULE_ID = "jobs";
 export const jobs = defineModule({
   manifest: {
     id: MODULE_ID,
-    version: "0.1.2",
+    version: "0.1.3",
     dependsOn: {},
     dbSchema: "jobs",
     tables: ["jobs", "schedules"],

package/src/schema.ts

@@ -28,6 +28,8 @@ export const schedules = jobsSchema.table("schedules", {
   name: text("name").primaryKey(),
   lastSlotAt: timestamp("last_slot_at", { withTimezone: true }).notNull(),
   lastSucceededAt: timestamp("last_succeeded_at", { withTimezone: true }),
+  lastFailedAt: timestamp("last_failed_at", { withTimezone: true }),
+  lastFailureCode: text("last_failure_code"),
   createdAt: timestamp("created_at", { withTimezone: true }).notNull(),
   updatedAt: timestamp("updated_at", { withTimezone: true }).notNull(),
 });

package/src/server/claim.ts

@@ -6,7 +6,7 @@ import { jobs } from "../schema.js";
 import type { JobsContext } from "./context.js";
 import { toJob } from "./job-row.js";
 import { getJobsOptions, getQueue } from "./options.js";
-import { recordScheduleSuccess } from "./schedules.js";
+import { recordScheduleFailure, recordScheduleSuccess } from "./schedules.js";
 
 /** A job this worker holds: the job and the token its writes are fenced with. */
 export interface ClaimedJob {
@@ -89,7 +89,8 @@ export async function extendJobLease(ctx: JobsContext, claimed: ClaimedJob): Pro
 }
 
 /**
- * Records the end of an attempt; a success of a scheduled queue also updates its schedule, in the same transaction.
+ * Records the end of an attempt; a success or a terminal failure of a scheduled queue also updates its schedule, in
+ * the same transaction.
  * False when the job is no longer this claim's: nothing was written.
  */
 export async function completeJob(ctx: JobsContext, claimed: ClaimedJob, completion: Completion): Promise<boolean> {
@@ -105,6 +106,7 @@ export async function completeJob(ctx: JobsContext, claimed: ClaimedJob, complet
     const rows = await tx.update(jobs).set(set).where(isHeld(claimed)).returning();
     if (rows.length === 0) return false;
     if (completion.kind === "succeeded") await recordScheduleSuccess({ ...ctx, db: tx }, claimed.job.queue);
+    if (completion.kind === "failed") await recordScheduleFailure({ ...ctx, db: tx }, claimed.job.queue, completion.error);
     return true;
   });
 }
@@ -132,18 +134,22 @@ export async function releaseJob(ctx: JobsContext, claimed: ClaimedJob): Promise
 }
 
 async function failExhaustedLeases(ctx: JobsContext, queue: string, now: Date): Promise<void> {
-  await ctx.db
-    .update(jobs)
-    .set({
-      status: "failed",
-      lastError: LEASE_EXPIRED,
-      finishedAt: now,
-      lockedUntil: null,
-      lockedBy: null,
-      claimToken: null,
-      updatedAt: now,
-    })
-    .where(and(eq(jobs.queue, queue), eq(jobs.status, "running"), lte(jobs.lockedUntil, now), gte(jobs.attempts, jobs.maxAttempts)));
+  await ctx.db.transaction(async (tx) => {
+    const failed = await tx
+      .update(jobs)
+      .set({
+        status: "failed",
+        lastError: LEASE_EXPIRED,
+        finishedAt: now,
+        lockedUntil: null,
+        lockedBy: null,
+        claimToken: null,
+        updatedAt: now,
+      })
+      .where(and(eq(jobs.queue, queue), eq(jobs.status, "running"), lte(jobs.lockedUntil, now), gte(jobs.attempts, jobs.maxAttempts)))
+      .returning({ id: jobs.id });
+    if (failed.length > 0) await recordScheduleFailure({ ...ctx, db: tx }, queue, LEASE_EXPIRED);
+  });
 }
 
 function isHeld(claimed: ClaimedJob) {

package/src/server/schedules.ts

@@ -23,11 +23,17 @@ export interface ScheduleState {
   readonly lastEnqueuedAt: Date | null;
   /** When a job of the schedule's queue last succeeded; null when none has yet. */
   readonly lastSucceededAt: Date | null;
+  /** When a job of the schedule's queue last ended `failed` (attempts used up, or failed for good); null when none has. */
+  readonly lastFailedAt: Date | null;
+  /** That job's error code (the handler's code, `jobs.handler_threw`, `jobs.lease_expired`…); null when none failed. */
+  readonly lastFailureCode: string | null;
   /**
    * True when no run succeeded within two periods (counted from when a worker first saw the schedule), or no worker
    * has seen it at all: no worker runs its queue, or its runs keep failing.
    */
   readonly isLate: boolean;
+  /** True when the latest outcome is a failure: `lastFailedAt` is later than `lastSucceededAt`, or nothing succeeded. */
+  readonly isFailing: boolean;
 }
 
 export interface EnqueueDueSchedulesInput {
@@ -90,6 +96,13 @@ export async function recordScheduleSuccess(ctx: JobsContext, queue: string): Pr
   await ctx.db.update(schedules).set({ lastSucceededAt: now, updatedAt: now }).where(eq(schedules.name, queue));
 }
 
+/** Notes a failed job of a scheduled queue (terminal, not a retried attempt) with its error code. */
+export async function recordScheduleFailure(ctx: JobsContext, queue: string, code: string): Promise<void> {
+  if (!Object.hasOwn(getJobsOptions(ctx.config).schedules, queue)) return;
+  const now = ctx.clock.now();
+  await ctx.db.update(schedules).set({ lastFailedAt: now, lastFailureCode: code, updatedAt: now }).where(eq(schedules.name, queue));
+}
+
 /** The state of every configured schedule, by queue name. */
 export async function getScheduleStates(ctx: JobsContext): Promise<Map<string, ScheduleState>> {
   const options = getJobsOptions(ctx.config);
@@ -103,7 +116,11 @@ export async function getScheduleStates(ctx: JobsContext): Promise<Map<string, S
       const row = rows.find((candidate) => candidate.name === name);
       const since = row?.lastSucceededAt ?? row?.createdAt ?? null;
       const isLate = since === null || now - since.getTime() > 2 * everySeconds * 1000;
-      return [name, { everySeconds, lastEnqueuedAt: row?.lastSlotAt ?? null, lastSucceededAt: row?.lastSucceededAt ?? null, isLate }];
+      const lastSucceededAt = row?.lastSucceededAt ?? null;
+      const lastFailedAt = row?.lastFailedAt ?? null;
+      const isFailing = lastFailedAt !== null && (lastSucceededAt === null || lastFailedAt > lastSucceededAt);
+      const lastFailureCode = row?.lastFailureCode ?? null;
+      return [name, { everySeconds, lastEnqueuedAt: row?.lastSlotAt ?? null, lastSucceededAt, lastFailedAt, lastFailureCode, isLate, isFailing }];
     }),
   );
 }

package/src/worker/health.ts

@@ -0,0 +1,64 @@
+// The worker's heartbeat: a file the loop touches after each round whose claims reached the database, and the check
+// an image runs (`node worker.mjs --health`) to tell a running loop from a process that only stays alive.
+import { rmSync, statSync, writeFileSync } from "node:fs";
+import type { SoftureConfig } from "@softure-ai/core";
+import type { JobsLogger } from "../server/logger.js";
+import { getJobsOptions } from "../server/options.js";
+
+/** The age limit never drops below this, so a slow round (an hourly prune) does not read as a stuck loop. */
+const MIN_HEALTH_MAX_AGE_MS = 30_000;
+const POLL_INTERVALS_PER_MAX_AGE = 5;
+
+export type JobWorkerHealth = { ok: true; ageMs: number } | { ok: false; reason: "missing" } | { ok: false; reason: "stale"; ageMs: number };
+
+export interface CheckJobWorkerHealthInput {
+  readonly healthFile: string;
+  /** The oldest heartbeat that still counts; `getJobWorkerHealthMaxAgeMs(config)` by default in the process. */
+  readonly maxAgeMs: number;
+  /** Epoch ms to compare the file's modification time with; default `Date.now()`. */
+  readonly now?: number;
+}
+
+/** The default age limit: five poll intervals, at least 30 s. */
+export function getJobWorkerHealthMaxAgeMs(config: SoftureConfig): number {
+  return Math.max(MIN_HEALTH_MAX_AGE_MS, POLL_INTERVALS_PER_MAX_AGE * getJobsOptions(config).pollIntervalMs);
+}
+
+/** Whether the heartbeat file exists and is no older than `maxAgeMs`. Reads the file system only. */
+export function checkJobWorkerHealth(input: CheckJobWorkerHealthInput): JobWorkerHealth {
+  let mtimeMs: number;
+  try {
+    mtimeMs = statSync(input.healthFile).mtimeMs;
+  } catch {
+    return { ok: false, reason: "missing" };
+  }
+  const ageMs = Math.max(0, Math.round((input.now ?? Date.now()) - mtimeMs));
+  return ageMs > input.maxAgeMs ? { ok: false, reason: "stale", ageMs } : { ok: true, ageMs };
+}
+
+/** Writes and removes the heartbeat file; a failure is logged once until a write succeeds again, never thrown. */
+export function createHealthFile(healthFile: string, logger: JobsLogger): { touch: () => void; remove: () => void } {
+  let hasFailed = false;
+  const report = (operation: string, error: unknown) => {
+    if (hasFailed) return;
+    hasFailed = true;
+    logger.error(`could not ${operation} the worker's health file`, { healthFile, error });
+  };
+  return {
+    touch: () => {
+      try {
+        writeFileSync(healthFile, new Date().toISOString());
+        hasFailed = false;
+      } catch (error) {
+        report("write", error);
+      }
+    },
+    remove: () => {
+      try {
+        rmSync(healthFile, { force: true });
+      } catch (error) {
+        report("remove", error);
+      }
+    },
+  };
+}

package/src/worker/index.ts

@@ -1,4 +1,5 @@
-// The worker of @softure-ai/jobs: the loop (`runJobWorker`) and the process entry an app's `scripts/worker.ts` calls
-// (`runJobWorkerProcess`).
+// The worker of @softure-ai/jobs: the loop (`runJobWorker`), the process entry an app's `scripts/worker.ts` calls
+// (`runJobWorkerProcess`) and the check of its heartbeat (`checkJobWorkerHealth`).
+export { checkJobWorkerHealth, getJobWorkerHealthMaxAgeMs, type CheckJobWorkerHealthInput, type JobWorkerHealth } from "./health.js";
 export { runJobWorkerProcess, type RunJobWorkerProcessOptions } from "./process.js";
 export { runJobWorker, type RunJobWorkerOptions } from "./run-job-worker.js";

package/src/worker/process.ts

@@ -1,17 +1,21 @@
 // The worker as a process: what an app's `scripts/worker.ts` calls. It opens its own database pool, turns SIGTERM and
-// SIGINT into a graceful stop, closes the pool and reports an exit code.
+// SIGINT into a graceful stop, closes the pool and reports an exit code. With `--health` it checks the heartbeat file
+// of a running worker instead.
 import { errorLogLabel, type SoftureConfig } from "@softure-ai/core";
 import { openCommandDatabase, type Database } from "@softure-ai/db";
 import type { JobHandlers } from "../handlers.js";
 import { consoleJobsLogger, type JobsLogger } from "../server/logger.js";
 import { getJobsOptions } from "../server/options.js";
+import { checkJobWorkerHealth, getJobWorkerHealthMaxAgeMs } from "./health.js";
 import { runJobWorker, type RunJobWorkerOptions } from "./run-job-worker.js";
 
 const EXIT_OK = 0;
 const EXIT_FAILED = 1;
 const STOP_SIGNALS = ["SIGTERM", "SIGINT"] as const;
+const HEALTH_FLAG = "--health";
 
-export interface RunJobWorkerProcessOptions extends Partial<Pick<RunJobWorkerOptions, "queues" | "clock" | "workerId" | "random" | "heartbeatMs">> {
+export interface RunJobWorkerProcessOptions
+  extends Partial<Pick<RunJobWorkerOptions, "queues" | "clock" | "workerId" | "random" | "heartbeatMs" | "healthFile">> {
   readonly config: SoftureConfig;
   readonly handlers: JobHandlers;
   readonly logger?: JobsLogger;
@@ -19,11 +23,17 @@ export interface RunJobWorkerProcessOptions extends Partial<Pick<RunJobWorkerOpt
   readonly database?: Database;
   /** Stops the worker instead of SIGTERM and SIGINT (tests). */
   readonly signal?: AbortSignal;
+  /** The oldest heartbeat `--health` accepts; default `getJobWorkerHealthMaxAgeMs(config)` (five poll intervals, at least 30 s). */
+  readonly healthMaxAgeMs?: number;
+  /** The process arguments; default `process.argv.slice(2)`. `--health` checks `healthFile` instead of running. */
+  readonly argv?: readonly string[];
 }
 
 /**
  * Runs the worker until SIGTERM or SIGINT, then stops gracefully. Resolves to the exit code and sets
- * `process.exitCode`: 0 after a clean stop, 1 when it could not start or the database could not be opened.
+ * `process.exitCode`: 0 after a clean stop, 1 when it could not start or the database could not be opened. Started
+ * with `--health`, it opens nothing and checks the running worker's `healthFile` instead: 0 when its loop ran within
+ * `healthMaxAgeMs`, else 1.
  *
  *   // scripts/worker.ts
  *   import { runJobWorkerProcess } from "@softure-ai/jobs/worker";
@@ -32,11 +42,29 @@ export interface RunJobWorkerProcessOptions extends Partial<Pick<RunJobWorkerOpt
  *   await runJobWorkerProcess({ config, handlers });
  */
 export async function runJobWorkerProcess(options: RunJobWorkerProcessOptions): Promise<number> {
-  const exitCode = await runProcess(options);
+  const argv = options.argv ?? process.argv.slice(2);
+  const exitCode = argv.includes(HEALTH_FLAG) ? checkHealth(options) : await runProcess(options);
   process.exitCode = exitCode;
   return exitCode;
 }
 
+function checkHealth(options: RunJobWorkerProcessOptions): number {
+  const logger = options.logger ?? consoleJobsLogger;
+  if (options.healthFile === undefined) {
+    logger.error(`${HEALTH_FLAG} needs the worker's healthFile option; pass healthFile to runJobWorkerProcess`);
+    return EXIT_FAILED;
+  }
+  const maxAgeMs = options.healthMaxAgeMs ?? getJobWorkerHealthMaxAgeMs(options.config);
+  const health = checkJobWorkerHealth({ healthFile: options.healthFile, maxAgeMs });
+  if (health.ok) {
+    logger.info(`the worker's loop ran ${String(health.ageMs)} ms ago`);
+    return EXIT_OK;
+  }
+  if (health.reason === "missing") logger.error(`the worker's health file is missing: ${options.healthFile}`);
+  else logger.error(`the worker's loop last ran ${String(health.ageMs)} ms ago (limit ${String(maxAgeMs)} ms)`);
+  return EXIT_FAILED;
+}
+
 async function runProcess(options: RunJobWorkerProcessOptions): Promise<number> {
   const logger = options.logger ?? consoleJobsLogger;
   const database = options.config.database;

package/src/worker/run-job-worker.ts

@@ -13,6 +13,7 @@ import { pruneJobs } from "../server/maintenance.js";
 import { getJobsOptions, getQueue } from "../server/options.js";
 import { runClaimedJob } from "../server/process.js";
 import { enqueueScheduledRun, getSchedule, getScheduleNames, getScheduleSlot } from "../server/schedules.js";
+import { createHealthFile } from "./health.js";
 
 const PRUNE_INTERVAL_MS = 60 * 60 * 1000;
 /** A lease is extended three times per lease period, so one slow write does not lose it. */
@@ -35,6 +36,11 @@ export interface RunJobWorkerOptions {
   readonly random?: () => number;
   /** How often a running job's lease is extended; default a third of the queue's `leaseSeconds`. */
   readonly heartbeatMs?: number;
+  /**
+   * A file the loop touches after each round whose claims reached the database (removed at start and after a clean
+   * stop), for a health check of the loop: `checkJobWorkerHealth`. Default none.
+   */
+  readonly healthFile?: string;
 }
 
 interface RunningJob {
@@ -69,6 +75,9 @@ export async function runJobWorker(options: RunJobWorkerOptions): Promise<void>
   if (idle.length > 0 && options.queues === undefined) {
     logger.warn("configured queues without a handler are not run by this worker", { queues: idle.join(", ") });
   }
+  const healthFile = options.healthFile === undefined ? undefined : createHealthFile(options.healthFile, logger);
+  // A file left by an earlier run of the same container must not pass the check before this loop ran a round.
+  healthFile?.remove();
   logger.info("worker started", { workerId, queues: queues.join(", ") });
 
   const start = (claimed: ClaimedJob): RunningJob => {
@@ -113,6 +122,7 @@ export async function runJobWorker(options: RunJobWorkerOptions): Promise<void>
         logger.error("could not enqueue a scheduled run; trying again next round", { queue: name, error });
       }
     }
+    let isClaimFailed = false;
     for (const queue of queues) {
       const slots = running.get(queue) ?? new Set<RunningJob>();
       const free = getQueue(jobsOptions, queue).concurrency - slots.size;
@@ -121,6 +131,7 @@ export async function runJobWorker(options: RunJobWorkerOptions): Promise<void>
         const claimed = await claimJobs(ctx, { queue, limit: free, workerId });
         for (const job of claimed) slots.add(start(job));
       } catch (error) {
+        isClaimFailed = true;
         logger.error("could not claim jobs", { queue, error });
       }
     }
@@ -133,10 +144,12 @@ export async function runJobWorker(options: RunJobWorkerOptions): Promise<void>
         logger.error("could not prune finished jobs", { error });
       }
     }
+    if (!isClaimFailed) healthFile?.touch();
     await wakeUp.wait(jobsOptions.pollIntervalMs, options.signal);
   }
 
   await stop(running, { ctx, logger, graceMs: jobsOptions.shutdownGraceMs });
+  healthFile?.remove();
   logger.info("worker stopped", { workerId });
 }