@softure-ai/jobs 0.1.2 → 0.1.4

package/dist/worker/process.js

@@ -1,16 +1,21 @@
 // The worker as a process: what an app's `scripts/worker.ts` calls. It opens its own database pool, turns SIGTERM and
-// SIGINT into a graceful stop, closes the pool and reports an exit code.
+// SIGINT into a graceful stop, closes the pool and reports an exit code. With `--health` it checks the heartbeat file
+// of a running worker instead.
 import { errorLogLabel } from "@softure-ai/core";
 import { openCommandDatabase } from "@softure-ai/db";
 import { consoleJobsLogger } from "../server/logger.js";
 import { getJobsOptions } from "../server/options.js";
+import { checkJobWorkerHealth, getJobWorkerHealthMaxAgeMs } from "./health.js";
 import { runJobWorker } from "./run-job-worker.js";
 const EXIT_OK = 0;
 const EXIT_FAILED = 1;
 const STOP_SIGNALS = ["SIGTERM", "SIGINT"];
+const HEALTH_FLAG = "--health";
 /**
  * Runs the worker until SIGTERM or SIGINT, then stops gracefully. Resolves to the exit code and sets
- * `process.exitCode`: 0 after a clean stop, 1 when it could not start or the database could not be opened.
+ * `process.exitCode`: 0 after a clean stop, 1 when it could not start or the database could not be opened. Started
+ * with `--health`, it opens nothing and checks the running worker's `healthFile` instead: 0 when its loop ran within
+ * `healthMaxAgeMs`, else 1.
  *
  *   // scripts/worker.ts
  *   import { runJobWorkerProcess } from "@softure-ai/jobs/worker";
@@ -19,10 +24,29 @@ const STOP_SIGNALS = ["SIGTERM", "SIGINT"];
  *   await runJobWorkerProcess({ config, handlers });
  */
 export async function runJobWorkerProcess(options) {
-    const exitCode = await runProcess(options);
+    const argv = options.argv ?? process.argv.slice(2);
+    const exitCode = argv.includes(HEALTH_FLAG) ? checkHealth(options) : await runProcess(options);
     process.exitCode = exitCode;
     return exitCode;
 }
+function checkHealth(options) {
+    const logger = options.logger ?? consoleJobsLogger;
+    if (options.healthFile === undefined) {
+        logger.error(`${HEALTH_FLAG} needs the worker's healthFile option; pass healthFile to runJobWorkerProcess`);
+        return EXIT_FAILED;
+    }
+    const maxAgeMs = options.healthMaxAgeMs ?? getJobWorkerHealthMaxAgeMs(options.config);
+    const health = checkJobWorkerHealth({ healthFile: options.healthFile, maxAgeMs });
+    if (health.ok) {
+        logger.info(`the worker's loop ran ${String(health.ageMs)} ms ago`);
+        return EXIT_OK;
+    }
+    if (health.reason === "missing")
+        logger.error(`the worker's health file is missing: ${options.healthFile}`);
+    else
+        logger.error(`the worker's loop last ran ${String(health.ageMs)} ms ago (limit ${String(maxAgeMs)} ms)`);
+    return EXIT_FAILED;
+}
 async function runProcess(options) {
     const logger = options.logger ?? consoleJobsLogger;
     const database = options.config.database;

package/dist/worker/process.js.map

@@ -1 +1 @@
-{"version":3,"file":"process.js","sourceRoot":"","sources":["../../src/worker/process.ts"],"names":[],"mappings":"AAAA,sHAAsH;AACtH,yEAAyE;AACzE,OAAO,EAAE,aAAa,EAAsB,MAAM,kBAAkB,CAAC;AACrE,OAAO,EAAE,mBAAmB,EAAiB,MAAM,gBAAgB,CAAC;AAEpE,OAAO,EAAE,iBAAiB,EAAmB,MAAM,qBAAqB,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,YAAY,EAA4B,MAAM,qBAAqB,CAAC;AAE7E,MAAM,OAAO,GAAG,CAAC,CAAC;AAClB,MAAM,WAAW,GAAG,CAAC,CAAC;AACtB,MAAM,YAAY,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAU,CAAC;AAYpD;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,OAAmC;IAC3E,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAC;IAC3C,OAAO,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC5B,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,OAAmC;IAC3D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,iBAAiB,CAAC;IACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC;IACzC,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;QACxD,MAAM,CAAC,KAAK,CAAC,gEAAgE,CAAC,CAAC;QAC/E,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,IAAI,MAAoD,CAAC;IACzD,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IACjD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,6BAA6B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACvD,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,IAAI,GAAG,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IACtC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IAChC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,KAAK,MAAM,MAAM,IAAI,YAAY;YAAE,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAChE,CAAC;SAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;QAC5B,IAAI,EAAE,CAAC;IACT,CAAC;SAAM,CAAC;QACN,QAAQ,CAAC,gBAAgB,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3D,CAAC;IAED,IAAI,CAAC;QACH,MAAM,YAAY,CAAC,EAAE,GAAG,OAAO,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QACrF,OAAO,OAAO,CAAC;IACjB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,sBAAsB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/G,OAAO,WAAW,CAAC;IACrB,CAAC;YAAS,CAAC;QACT,KAAK,MAAM,MAAM,IAAI,YAAY;YAAE,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACxE,QAAQ,EAAE,mBAAmB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QAC7C,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;IAC1G,CAAC;AACH,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,OAAmC,EACnC,QAAmC;IAEnC,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS;QAAE,OAAO,EAAE,EAAE,EAAE,OAAO,CAAC,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;IACpG,IAAI,QAAQ,KAAK,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC;IACtD,8EAA8E;IAC9E,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IAC5H,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,WAAW,GAAG,CAAC,EAAE,CAAC,CAAC;IAC9E,OAAO,EAAE,EAAE,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;AACzD,CAAC"}
+{"version":3,"file":"process.js","sourceRoot":"","sources":["../../src/worker/process.ts"],"names":[],"mappings":"AAAA,sHAAsH;AACtH,sHAAsH;AACtH,+BAA+B;AAC/B,OAAO,EAAE,aAAa,EAAsB,MAAM,kBAAkB,CAAC;AACrE,OAAO,EAAE,mBAAmB,EAAiB,MAAM,gBAAgB,CAAC;AAEpE,OAAO,EAAE,iBAAiB,EAAmB,MAAM,qBAAqB,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,aAAa,CAAC;AAC/E,OAAO,EAAE,YAAY,EAA4B,MAAM,qBAAqB,CAAC;AAE7E,MAAM,OAAO,GAAG,CAAC,CAAC;AAClB,MAAM,WAAW,GAAG,CAAC,CAAC;AACtB,MAAM,YAAY,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAU,CAAC;AACpD,MAAM,WAAW,GAAG,UAAU,CAAC;AAiB/B;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,OAAmC;IAC3E,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,UAAU,CAAC,OAAO,CAAC,CAAC;IAC/F,OAAO,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC5B,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,WAAW,CAAC,OAAmC;IACtD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,iBAAiB,CAAC;IACnD,IAAI,OAAO,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QACrC,MAAM,CAAC,KAAK,CAAC,GAAG,WAAW,+EAA+E,CAAC,CAAC;QAC5G,OAAO,WAAW,CAAC;IACrB,CAAC;IACD,MAAM,QAAQ,GAAG,OAAO,CAAC,cAAc,IAAI,0BAA0B,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACtF,MAAM,MAAM,GAAG,oBAAoB,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,QAAQ,EAAE,CAAC,CAAC;IAClF,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;QACd,MAAM,CAAC,IAAI,CAAC,yBAAyB,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QACpE,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS;QAAE,MAAM,CAAC,KAAK,CAAC,wCAAwC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;;QACvG,MAAM,CAAC,KAAK,CAAC,8BAA8B,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC9G,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,OAAmC;IAC3D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,iBAAiB,CAAC;IACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC;IACzC,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;QACxD,MAAM,CAAC,KAAK,CAAC,gEAAgE,CAAC,CAAC;QAC/E,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,IAAI,MAAoD,CAAC;IACzD,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IACjD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,6BAA6B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACvD,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,IAAI,GAAG,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IACtC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IAChC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,KAAK,MAAM,MAAM,IAAI,YAAY;YAAE,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAChE,CAAC;SAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;QAC5B,IAAI,EAAE,CAAC;IACT,CAAC;SAAM,CAAC;QACN,QAAQ,CAAC,gBAAgB,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3D,CAAC;IAED,IAAI,CAAC;QACH,MAAM,YAAY,CAAC,EAAE,GAAG,OAAO,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QACrF,OAAO,OAAO,CAAC;IACjB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,sBAAsB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/G,OAAO,WAAW,CAAC;IACrB,CAAC;YAAS,CAAC;QACT,KAAK,MAAM,MAAM,IAAI,YAAY;YAAE,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACxE,QAAQ,EAAE,mBAAmB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QAC7C,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;IAC1G,CAAC;AACH,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,OAAmC,EACnC,QAAmC;IAEnC,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS;QAAE,OAAO,EAAE,EAAE,EAAE,OAAO,CAAC,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;IACpG,IAAI,QAAQ,KAAK,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC;IACtD,8EAA8E;IAC9E,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IAC5H,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,WAAW,GAAG,CAAC,EAAE,CAAC,CAAC;IAC9E,OAAO,EAAE,EAAE,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;AACzD,CAAC"}

package/dist/worker/run-job-worker.d.ts

@@ -19,6 +19,11 @@ export interface RunJobWorkerOptions {
     readonly random?: () => number;
     /** How often a running job's lease is extended; default a third of the queue's `leaseSeconds`. */
     readonly heartbeatMs?: number;
+    /**
+     * A file the loop touches after each round whose claims reached the database (removed at start and after a clean
+     * stop), for a health check of the loop: `checkJobWorkerHealth`. Default none.
+     */
+    readonly healthFile?: string;
 }
 /**
  * Runs until `signal` aborts, then resolves once every job finished or was released. Throws before starting on a

package/dist/worker/run-job-worker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run-job-worker.d.ts","sourceRoot":"","sources":["../../src/worker/run-job-worker.ts"],"names":[],"mappings":"AAIA,OAAO,EAAe,KAAK,KAAK,EAAE,KAAK,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAC/E,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAIlD,OAAO,EAAqB,KAAK,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAUzE,MAAM,WAAW,mBAAmB;IAClC,oHAAoH;IACpH,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC;IAC/B,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC;IAC/B,gGAAgG;IAChG,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,wEAAwE;IACxE,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACpC,QAAQ,CAAC,MAAM,CAAC,EAAE,UAAU,CAAC;IAC7B,QAAQ,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC;IACvB,yDAAyD;IACzD,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,6CAA6C;IAC7C,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,MAAM,CAAC;IAC/B,kGAAkG;IAClG,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CAC/B;AAQD;;;;GAIG;AACH,wBAAsB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC,CA0F9E"}
+{"version":3,"file":"run-job-worker.d.ts","sourceRoot":"","sources":["../../src/worker/run-job-worker.ts"],"names":[],"mappings":"AAIA,OAAO,EAAe,KAAK,KAAK,EAAE,KAAK,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAC/E,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAIlD,OAAO,EAAqB,KAAK,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAYzE,MAAM,WAAW,mBAAmB;IAClC,oHAAoH;IACpH,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC;IAC/B,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC;IAC/B,gGAAgG;IAChG,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,wEAAwE;IACxE,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACpC,QAAQ,CAAC,MAAM,CAAC,EAAE,UAAU,CAAC;IAC7B,QAAQ,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC;IACvB,yDAAyD;IACzD,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,6CAA6C;IAC7C,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,MAAM,CAAC;IAC/B,kGAAkG;IAClG,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B;;;OAGG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAC9B;AAQD;;;;GAIG;AACH,wBAAsB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC,CAuG9E"}

package/dist/worker/run-job-worker.js

@@ -7,9 +7,11 @@ import { claimJobs, extendJobLease, releaseJob } from "../server/claim.js";
 import { checkHandlers } from "../server/handler-check.js";
 import { consoleJobsLogger } from "../server/logger.js";
 import { pruneJobs } from "../server/maintenance.js";
+import { pruneSyncItems } from "../server/sync-items.js";
 import { getJobsOptions, getQueue } from "../server/options.js";
 import { runClaimedJob } from "../server/process.js";
 import { enqueueScheduledRun, getSchedule, getScheduleNames, getScheduleSlot } from "../server/schedules.js";
+import { createHealthFile } from "./health.js";
 const PRUNE_INTERVAL_MS = 60 * 60 * 1000;
 /** A lease is extended three times per lease period, so one slow write does not lose it. */
 const HEARTBEATS_PER_LEASE = 3;
@@ -38,6 +40,9 @@ export async function runJobWorker(options) {
     if (idle.length > 0 && options.queues === undefined) {
         logger.warn("configured queues without a handler are not run by this worker", { queues: idle.join(", ") });
     }
+    const healthFile = options.healthFile === undefined ? undefined : createHealthFile(options.healthFile, logger);
+    // A file left by an earlier run of the same container must not pass the check before this loop ran a round.
+    healthFile?.remove();
     logger.info("worker started", { workerId, queues: queues.join(", ") });
     const start = (claimed) => {
         const controller = new AbortController();
@@ -78,6 +83,7 @@ export async function runJobWorker(options) {
                 logger.error("could not enqueue a scheduled run; trying again next round", { queue: name, error });
             }
         }
+        let isClaimFailed = false;
         for (const queue of queues) {
             const slots = running.get(queue) ?? new Set();
             const free = getQueue(jobsOptions, queue).concurrency - slots.size;
@@ -89,6 +95,7 @@ export async function runJobWorker(options) {
                     slots.add(start(job));
             }
             catch (error) {
+                isClaimFailed = true;
                 logger.error("could not claim jobs", { queue, error });
             }
         }
@@ -102,10 +109,21 @@ export async function runJobWorker(options) {
             catch (error) {
                 logger.error("could not prune finished jobs", { error });
             }
+            try {
+                const deleted = await pruneSyncItems(ctx);
+                if (deleted > 0)
+                    logger.info("pruned sync ledger items", { deleted });
+            }
+            catch (error) {
+                logger.error("could not prune sync ledger items", { error });
+            }
         }
+        if (!isClaimFailed)
+            healthFile?.touch();
         await wakeUp.wait(jobsOptions.pollIntervalMs, options.signal);
     }
     await stop(running, { ctx, logger, graceMs: jobsOptions.shutdownGraceMs });
+    healthFile?.remove();
     logger.info("worker stopped", { workerId });
 }
 /**

package/dist/worker/run-job-worker.js.map

@@ -1 +1 @@
-{"version":3,"file":"run-job-worker.js","sourceRoot":"","sources":["../../src/worker/run-job-worker.ts"],"names":[],"mappings":"AAAA,oHAAoH;AACpH,sHAAsH;AACtH,sFAAsF;AACtF,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,WAAW,EAAkC,MAAM,kBAAkB,CAAC;AAG/E,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,UAAU,EAAmB,MAAM,oBAAoB,CAAC;AAE5F,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,iBAAiB,EAAmB,MAAM,qBAAqB,CAAC;AACzE,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChE,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,mBAAmB,EAAE,WAAW,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAE7G,MAAM,iBAAiB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AACzC,4FAA4F;AAC5F,MAAM,oBAAoB,GAAG,CAAC,CAAC;AA2B/B;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAA4B;IAC7D,MAAM,WAAW,GAAG,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,aAAa,CAAC,WAAW,EAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAC5E,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,oGAAoG,CAAC,CAAC;IACxH,CAAC;IACD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,iBAAiB,CAAC;IACnD,MAAM,GAAG,GAAgB,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;IACzG,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,GAAG,QAAQ,EAAE,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;IAC5E,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC;IAC7C,MAAM,OAAO,GAAG,IAAI,GAAG,CAA0B,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5F,MAAM,MAAM,GAAG,YAAY,EAAE,CAAC;IAC9B,IAAI,WAAW,GAAG,MAAM,CAAC,iBAAiB,CAAC;IAC3C,wHAAwH;IACxH,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC/C,MAAM,aAAa,GAAG,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IAE5D,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;IACxF,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QACpD,MAAM,CAAC,IAAI,CAAC,gEAAgE,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC7G,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEvE,MAAM,KAAK,GAAG,CAAC,OAAmB,EAAc,EAAE;QAChD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,oBAAoB,CAAC;QACjI,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE;YACjC,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,IAAI,CAC/B,CAAC,MAAM,EAAE,EAAE;gBACT,IAAI,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;oBAC1C,MAAM,CAAC,IAAI,CAAC,wCAAwC,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;oBAC3G,UAAU,CAAC,KAAK,EAAE,CAAC;gBACrB,CAAC;YACH,CAAC,EACD,CAAC,KAAc,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAC7H,CAAC;QACJ,CAAC,EAAE,WAAW,CAAC,CAAC;QAChB,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC;aAChH,IAAI,CACH,CAAC,OAAO,EAAE,EAAE;YACV,IAAI,OAAO,CAAC,IAAI,KAAK,OAAO;gBAAE,MAAM,CAAC,IAAI,CAAC,8DAA8D,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;QACjK,CAAC,EACD,CAAC,KAAc,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,mEAAmE,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAClK;aACA,OAAO,CAAC,GAAG,EAAE;YACZ,aAAa,CAAC,SAAS,CAAC,CAAC;YACzB,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;YAC9C,MAAM,CAAC,IAAI,EAAE,CAAC;QAChB,CAAC,CAAC,CAAC;QACL,MAAM,KAAK,GAAe,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;QACxD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;IAEF,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,eAAe,CAAC,WAAW,CAAC,WAAW,EAAE,IAAI,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;YAC9E,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,iBAAiB,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE;gBAAE,SAAS;YACrF,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,mBAAmB,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;gBAC9D,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;gBACvC,IAAI,UAAU;oBAAE,MAAM,CAAC,IAAI,CAAC,wBAAwB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;YACnG,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,4DAA4D,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YACrG,CAAC;QACH,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,GAAG,EAAc,CAAC;YAC1D,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC;YACnE,IAAI,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO;gBAAE,SAAS;YAClD,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;gBACvE,KAAK,MAAM,GAAG,IAAI,OAAO;oBAAE,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;YACnD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,sBAAsB,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;YACzD,CAAC;QACH,CAAC;QACD,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,GAAG,WAAW,IAAI,iBAAiB,EAAE,CAAC;YACjE,WAAW,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;YACxC,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,CAAC;gBACrC,IAAI,OAAO,GAAG,CAAC;oBAAE,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACpE,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC;QACD,MAAM,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,cAAc,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,IAAI,CAAC,OAAO,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,CAAC,eAAe,EAAE,CAAC,CAAC;IAC3E,MAAM,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;AAC9C,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,IAAI,CAAC,OAA6C,EAAE,OAAkE;IACnI,MAAM,IAAI,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;IAClE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAC9B,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,2CAA2C,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IACrH,IAAI,KAAiC,CAAC;IACtC,MAAM,KAAK,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAC1C,KAAK,GAAG,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;IACtE,YAAY,CAAC,KAAK,CAAC,CAAC;IAEpB,MAAM,UAAU,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;IACxE,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,GAAG,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;YAC9D,IAAI,UAAU;gBAAE,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,iCAAiC,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;QACtI,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,4DAA4D,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACzJ,CAAC;IACH,CAAC;AACH,CAAC;AAED,gFAAgF;AAChF,SAAS,YAAY;IACnB,IAAI,OAAiC,CAAC;IACtC,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,OAAO;QACL,IAAI,EAAE,GAAG,EAAE;YACT,OAAO,GAAG,IAAI,CAAC;YACf,OAAO,EAAE,EAAE,CAAC;QACd,CAAC;QACD,IAAI,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,CACnB,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAC5B,IAAI,OAAO,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBAC9B,OAAO,GAAG,KAAK,CAAC;gBAChB,OAAO,EAAE,CAAC;gBACV,OAAO;YACT,CAAC;YACD,MAAM,MAAM,GAAG,GAAG,EAAE;gBAClB,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;gBAC5C,OAAO,GAAG,SAAS,CAAC;gBACpB,OAAO,GAAG,KAAK,CAAC;gBAChB,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC;YACF,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACrC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YACzD,OAAO,GAAG,MAAM,CAAC;QACnB,CAAC,CAAC;KACL,CAAC;AACJ,CAAC"}
+{"version":3,"file":"run-job-worker.js","sourceRoot":"","sources":["../../src/worker/run-job-worker.ts"],"names":[],"mappings":"AAAA,oHAAoH;AACpH,sHAAsH;AACtH,sFAAsF;AACtF,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,WAAW,EAAkC,MAAM,kBAAkB,CAAC;AAG/E,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,UAAU,EAAmB,MAAM,oBAAoB,CAAC;AAE5F,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,iBAAiB,EAAmB,MAAM,qBAAqB,CAAC;AACzE,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChE,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,mBAAmB,EAAE,WAAW,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAC7G,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAE/C,MAAM,iBAAiB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AACzC,4FAA4F;AAC5F,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAgC/B;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAA4B;IAC7D,MAAM,WAAW,GAAG,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,aAAa,CAAC,WAAW,EAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAC5E,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,oGAAoG,CAAC,CAAC;IACxH,CAAC;IACD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,iBAAiB,CAAC;IACnD,MAAM,GAAG,GAAgB,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;IACzG,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,GAAG,QAAQ,EAAE,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;IAC5E,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC;IAC7C,MAAM,OAAO,GAAG,IAAI,GAAG,CAA0B,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5F,MAAM,MAAM,GAAG,YAAY,EAAE,CAAC;IAC9B,IAAI,WAAW,GAAG,MAAM,CAAC,iBAAiB,CAAC;IAC3C,wHAAwH;IACxH,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC/C,MAAM,aAAa,GAAG,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IAE5D,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;IACxF,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QACpD,MAAM,CAAC,IAAI,CAAC,gEAAgE,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC7G,CAAC;IACD,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,gBAAgB,CAAC,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IAC/G,4GAA4G;IAC5G,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,MAAM,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEvE,MAAM,KAAK,GAAG,CAAC,OAAmB,EAAc,EAAE;QAChD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,oBAAoB,CAAC;QACjI,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE;YACjC,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,IAAI,CAC/B,CAAC,MAAM,EAAE,EAAE;gBACT,IAAI,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;oBAC1C,MAAM,CAAC,IAAI,CAAC,wCAAwC,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;oBAC3G,UAAU,CAAC,KAAK,EAAE,CAAC;gBACrB,CAAC;YACH,CAAC,EACD,CAAC,KAAc,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAC7H,CAAC;QACJ,CAAC,EAAE,WAAW,CAAC,CAAC;QAChB,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC;aAChH,IAAI,CACH,CAAC,OAAO,EAAE,EAAE;YACV,IAAI,OAAO,CAAC,IAAI,KAAK,OAAO;gBAAE,MAAM,CAAC,IAAI,CAAC,8DAA8D,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;QACjK,CAAC,EACD,CAAC,KAAc,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,mEAAmE,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAClK;aACA,OAAO,CAAC,GAAG,EAAE;YACZ,aAAa,CAAC,SAAS,CAAC,CAAC;YACzB,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;YAC9C,MAAM,CAAC,IAAI,EAAE,CAAC;QAChB,CAAC,CAAC,CAAC;QACL,MAAM,KAAK,GAAe,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;QACxD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;IAEF,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,eAAe,CAAC,WAAW,CAAC,WAAW,EAAE,IAAI,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;YAC9E,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,iBAAiB,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE;gBAAE,SAAS;YACrF,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,mBAAmB,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;gBAC9D,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;gBACvC,IAAI,UAAU;oBAAE,MAAM,CAAC,IAAI,CAAC,wBAAwB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;YACnG,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,4DAA4D,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YACrG,CAAC;QACH,CAAC;QACD,IAAI,aAAa,GAAG,KAAK,CAAC;QAC1B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,GAAG,EAAc,CAAC;YAC1D,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC;YACnE,IAAI,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO;gBAAE,SAAS;YAClD,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;gBACvE,KAAK,MAAM,GAAG,IAAI,OAAO;oBAAE,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;YACnD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,aAAa,GAAG,IAAI,CAAC;gBACrB,MAAM,CAAC,KAAK,CAAC,sBAAsB,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;YACzD,CAAC;QACH,CAAC;QACD,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,GAAG,WAAW,IAAI,iBAAiB,EAAE,CAAC;YACjE,WAAW,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;YACxC,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,CAAC;gBACrC,IAAI,OAAO,GAAG,CAAC;oBAAE,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACpE,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YAC3D,CAAC;YACD,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,GAAG,CAAC,CAAC;gBAC1C,IAAI,OAAO,GAAG,CAAC;oBAAE,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACxE,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,mCAAmC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YAC/D,CAAC;QACH,CAAC;QACD,IAAI,CAAC,aAAa;YAAE,UAAU,EAAE,KAAK,EAAE,CAAC;QACxC,MAAM,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,cAAc,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,IAAI,CAAC,OAAO,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,CAAC,eAAe,EAAE,CAAC,CAAC;IAC3E,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,MAAM,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;AAC9C,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,IAAI,CAAC,OAA6C,EAAE,OAAkE;IACnI,MAAM,IAAI,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;IAClE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAC9B,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,2CAA2C,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IACrH,IAAI,KAAiC,CAAC;IACtC,MAAM,KAAK,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAC1C,KAAK,GAAG,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;IACtE,YAAY,CAAC,KAAK,CAAC,CAAC;IAEpB,MAAM,UAAU,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;IACxE,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,GAAG,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;YAC9D,IAAI,UAAU;gBAAE,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,iCAAiC,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;QACtI,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,4DAA4D,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACzJ,CAAC;IACH,CAAC;AACH,CAAC;AAED,gFAAgF;AAChF,SAAS,YAAY;IACnB,IAAI,OAAiC,CAAC;IACtC,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,OAAO;QACL,IAAI,EAAE,GAAG,EAAE;YACT,OAAO,GAAG,IAAI,CAAC;YACf,OAAO,EAAE,EAAE,CAAC;QACd,CAAC;QACD,IAAI,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,CACnB,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAC5B,IAAI,OAAO,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBAC9B,OAAO,GAAG,KAAK,CAAC;gBAChB,OAAO,EAAE,CAAC;gBACV,OAAO;YACT,CAAC;YACD,MAAM,MAAM,GAAG,GAAG,EAAE;gBAClB,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;gBAC5C,OAAO,GAAG,SAAS,CAAC;gBACpB,OAAO,GAAG,KAAK,CAAC;gBAChB,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC;YACF,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACrC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YACzD,OAAO,GAAG,MAAM,CAAC;QACnB,CAAC,CAAC;KACL,CAAC;AACJ,CAAC"}

package/migrations/0003_add_schedule_failures.sql

@@ -0,0 +1,8 @@
+-- The last terminal failure of a scheduled queue, so getQueueStats can tell whether it fails since its last success.
+-- Rollback: ALTER TABLE jobs.schedules DROP COLUMN last_failed_at, DROP COLUMN last_failure_code; then
+--   DELETE FROM softure.migrations WHERE module = 'jobs' AND version = 3;
+ALTER TABLE schedules
+  -- When a job of the schedule's queue last ended `failed`.
+  ADD COLUMN last_failed_at timestamptz,
+  -- That job's error code (`last_error`), never an error's text.
+  ADD COLUMN last_failure_code text;

package/migrations/0004_create_sync_state.sql

@@ -0,0 +1,41 @@
+-- Sync state of jobs that poll third-party sources: a cursor per source and a ledger of the items already handled.
+-- Rollback: DROP TABLE jobs.sync_items; DROP TABLE jobs.sync_cursors; then
+--   DELETE FROM softure.migrations WHERE module = 'jobs' AND version = 4;
+CREATE TABLE sync_cursors (
+  -- The source, named by the app: `gmail.inbox`, `todoist:project-42`.
+  source text PRIMARY KEY CHECK (source ~ '^[a-z][a-z0-9_.:-]{0,126}$'),
+  -- Where the next read starts: a timestamp, a history id, a page token. Moved only by a compare-and-set.
+  cursor text NOT NULL CHECK (char_length(cursor) BETWEEN 1 AND 2000),
+  created_at timestamptz NOT NULL,
+  updated_at timestamptz NOT NULL
+);
+
+CREATE TABLE sync_items (
+  source text NOT NULL CHECK (source ~ '^[a-z][a-z0-9_.:-]{0,126}$'),
+  -- The item's id at the source.
+  item_id text NOT NULL CHECK (char_length(item_id) BETWEEN 1 AND 500),
+  -- pending: seen, not handled yet (a failed attempt with attempts left); the rest are final.
+  status text NOT NULL CHECK (status IN ('pending', 'done', 'skipped', 'failed')),
+  -- The code of the last failed attempt, or why the item was skipped; never a message.
+  error_code text CHECK (char_length(error_code) BETWEEN 1 AND 200),
+  attempts integer NOT NULL CHECK (attempts BETWEEN 0 AND 1000),
+  -- What handling the item produced in the app (a task id, a mail id), for the overview.
+  result_ref text CHECK (char_length(result_ref) BETWEEN 1 AND 500),
+  first_seen_at timestamptz NOT NULL,
+  -- The last time a run met the item; pruning goes by it, so an item the source still returns is never forgotten.
+  last_seen_at timestamptz NOT NULL,
+  updated_at timestamptz NOT NULL,
+  finished_at timestamptz,
+  PRIMARY KEY (source, item_id),
+  CHECK ((status <> 'pending') = (finished_at IS NOT NULL)),
+  CHECK (status <> 'failed' OR (error_code IS NOT NULL AND attempts >= 1)),
+  CHECK (status <> 'done' OR attempts >= 1),
+  CHECK (result_ref IS NULL OR status = 'done'),
+  CHECK (last_seen_at >= first_seen_at)
+);
+
+-- The overview lists recent changes, per source or over all of them.
+CREATE INDEX sync_items_source_updated_idx ON sync_items (source, updated_at DESC);
+CREATE INDEX sync_items_updated_idx ON sync_items (updated_at DESC);
+-- Pruning deletes items no run has met for the retention period.
+CREATE INDEX sync_items_last_seen_idx ON sync_items (last_seen_at);

package/module.json

@@ -1,9 +1,9 @@
 {
   "id": "jobs",
-  "version": "0.1.2",
+  "version": "0.1.4",
   "dependsOn": {},
   "dbSchema": "jobs",
-  "tables": ["jobs", "schedules"],
+  "tables": ["jobs", "schedules", "sync_cursors", "sync_items"],
   "env": [],
   "switches": [],
   "routes": {},

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@softure-ai/jobs",
-  "version": "0.1.2",
+  "version": "0.1.4",
   "description": "SOFTURE AI jobs module: background jobs in Postgres. Enqueue inside the caller's transaction with an optional dedup key, claim with FOR UPDATE SKIP LOCKED under a lease, fenced completion, retry with exponential backoff and a terminal failed state, a worker process with graceful shutdown, and an in-process runner for tests.",
   "license": "MIT",
   "type": "module",

package/src/contract.ts

@@ -9,7 +9,9 @@ export type JobsErrorCode =
   | "jobs.not_found"
   | "jobs.not_cancellable"
   | "jobs.not_retryable"
-  | "jobs.dedup_key_taken";
+  | "jobs.dedup_key_taken"
+  | "jobs.sync_cursor_moved"
+  | "jobs.sync_item_threw";
 
 export const JOB_STATUSES = ["pending", "running", "succeeded", "failed", "cancelled"] as const;
 

package/src/index.ts

@@ -15,10 +15,10 @@ export const MODULE_ID = "jobs";
 export const jobs = defineModule({
   manifest: {
     id: MODULE_ID,
-    version: "0.1.2",
+    version: "0.1.4",
     dependsOn: {},
     dbSchema: "jobs",
-    tables: ["jobs", "schedules"],
+    tables: ["jobs", "schedules", "sync_cursors", "sync_items"],
     env: [],
     switches: [],
     routes: {},
@@ -51,4 +51,15 @@ export {
   type PeriodScheduleOptions,
 } from "./options.js";
 export { defineJobHandler, failJob, retryJobLater, type JobHandler, type JobHandlerContext, type JobHandlers } from "./handlers.js";
-export { jobs as jobsTable, schedules as jobSchedulesTable } from "./schema.js";
+export { jobs as jobsTable, schedules as jobSchedulesTable, syncCursors as jobSyncCursorsTable, syncItems as jobSyncItemsTable } from "./schema.js";
+export {
+  failSyncItem,
+  skipSyncItem,
+  SYNC_ITEM_STATUSES,
+  type SyncItem,
+  type SyncItemCompletion,
+  type SyncItemFailure,
+  type SyncItemOutcome,
+  type SyncItemRunResult,
+  type SyncItemStatus,
+} from "./sync.js";

package/src/messages/en.ts

@@ -7,5 +7,7 @@ export const en = {
     not_cancellable: "Only a job that has not started can be cancelled.",
     not_retryable: "Only a failed or cancelled job can be run again.",
     dedup_key_taken: "Another job for the same work is already waiting or running.",
+    sync_cursor_moved: "Another run moved the sync position first.",
+    sync_item_threw: "Handling the item failed unexpectedly.",
   },
 };

package/src/options.ts

@@ -104,6 +104,8 @@ const retentionSchema = z.strictObject({
   failedDays: z.number().int().min(1).max(3650).default(30),
   /** Days a cancelled job is kept. */
   cancelledDays: z.number().int().min(0).max(3650).default(7),
+  /** Days a sync ledger item is kept after the last run that met it (`processSyncItem`). */
+  syncItemsDays: z.number().int().min(1).max(3650).default(90),
 });
 
 export const jobsOptionsSchema = z
@@ -120,8 +122,8 @@ export const jobsOptionsSchema = z
     pollIntervalMs: z.number().int().min(50).max(60_000).default(1000),
     /** How long a stopping worker waits for running handlers before it releases their jobs. */
     shutdownGraceMs: z.number().int().min(0).max(10 * 60_000).default(20_000),
-    /** How long finished jobs are kept; the worker prunes once an hour. */
-    retention: retentionSchema.default({ succeededDays: 7, failedDays: 30, cancelledDays: 7 }),
+    /** How long finished jobs and unseen sync ledger items are kept; the worker prunes once an hour. */
+    retention: retentionSchema.default({ succeededDays: 7, failedDays: 30, cancelledDays: 7, syncItemsDays: 90 }),
   })
   .superRefine((options, context) => {
     if (Object.keys(options.queues).length === 0 && Object.keys(options.schedules).length === 0) {

package/src/schema.ts

@@ -1,6 +1,6 @@
 // Drizzle view of the module's tables (migrations/). The migrations are the source of truth; this
 // file only types the queries.
-import { bigint, integer, jsonb, pgSchema, text, timestamp, uuid } from "drizzle-orm/pg-core";
+import { bigint, integer, jsonb, pgSchema, primaryKey, text, timestamp, uuid } from "drizzle-orm/pg-core";
 
 export const jobsSchema = pgSchema("jobs");
 
@@ -28,6 +28,34 @@ export const schedules = jobsSchema.table("schedules", {
   name: text("name").primaryKey(),
   lastSlotAt: timestamp("last_slot_at", { withTimezone: true }).notNull(),
   lastSucceededAt: timestamp("last_succeeded_at", { withTimezone: true }),
+  lastFailedAt: timestamp("last_failed_at", { withTimezone: true }),
+  lastFailureCode: text("last_failure_code"),
   createdAt: timestamp("created_at", { withTimezone: true }).notNull(),
   updatedAt: timestamp("updated_at", { withTimezone: true }).notNull(),
 });
+
+export const syncCursors = jobsSchema.table("sync_cursors", {
+  source: text("source").primaryKey(),
+  cursor: text("cursor").notNull(),
+  createdAt: timestamp("created_at", { withTimezone: true }).notNull(),
+  updatedAt: timestamp("updated_at", { withTimezone: true }).notNull(),
+});
+
+export const syncItems = jobsSchema.table(
+  "sync_items",
+  {
+    source: text("source").notNull(),
+    itemId: text("item_id").notNull(),
+    status: text("status", { enum: ["pending", "done", "skipped", "failed"] }).notNull(),
+    errorCode: text("error_code"),
+    attempts: integer("attempts").notNull(),
+    resultRef: text("result_ref"),
+    firstSeenAt: timestamp("first_seen_at", { withTimezone: true }).notNull(),
+    lastSeenAt: timestamp("last_seen_at", { withTimezone: true }).notNull(),
+    updatedAt: timestamp("updated_at", { withTimezone: true }).notNull(),
+    finishedAt: timestamp("finished_at", { withTimezone: true }),
+  },
+  (table) => [primaryKey({ columns: [table.source, table.itemId] })],
+);
+
+export type SyncItemRow = typeof syncItems.$inferSelect;

package/src/server/claim.ts

@@ -6,7 +6,7 @@ import { jobs } from "../schema.js";
 import type { JobsContext } from "./context.js";
 import { toJob } from "./job-row.js";
 import { getJobsOptions, getQueue } from "./options.js";
-import { recordScheduleSuccess } from "./schedules.js";
+import { recordScheduleFailure, recordScheduleSuccess } from "./schedules.js";
 
 /** A job this worker holds: the job and the token its writes are fenced with. */
 export interface ClaimedJob {
@@ -89,7 +89,8 @@ export async function extendJobLease(ctx: JobsContext, claimed: ClaimedJob): Pro
 }
 
 /**
- * Records the end of an attempt; a success of a scheduled queue also updates its schedule, in the same transaction.
+ * Records the end of an attempt; a success or a terminal failure of a scheduled queue also updates its schedule, in
+ * the same transaction.
  * False when the job is no longer this claim's: nothing was written.
  */
 export async function completeJob(ctx: JobsContext, claimed: ClaimedJob, completion: Completion): Promise<boolean> {
@@ -105,6 +106,7 @@ export async function completeJob(ctx: JobsContext, claimed: ClaimedJob, complet
     const rows = await tx.update(jobs).set(set).where(isHeld(claimed)).returning();
     if (rows.length === 0) return false;
     if (completion.kind === "succeeded") await recordScheduleSuccess({ ...ctx, db: tx }, claimed.job.queue);
+    if (completion.kind === "failed") await recordScheduleFailure({ ...ctx, db: tx }, claimed.job.queue, completion.error);
     return true;
   });
 }
@@ -132,18 +134,22 @@ export async function releaseJob(ctx: JobsContext, claimed: ClaimedJob): Promise
 }
 
 async function failExhaustedLeases(ctx: JobsContext, queue: string, now: Date): Promise<void> {
-  await ctx.db
-    .update(jobs)
-    .set({
-      status: "failed",
-      lastError: LEASE_EXPIRED,
-      finishedAt: now,
-      lockedUntil: null,
-      lockedBy: null,
-      claimToken: null,
-      updatedAt: now,
-    })
-    .where(and(eq(jobs.queue, queue), eq(jobs.status, "running"), lte(jobs.lockedUntil, now), gte(jobs.attempts, jobs.maxAttempts)));
+  await ctx.db.transaction(async (tx) => {
+    const failed = await tx
+      .update(jobs)
+      .set({
+        status: "failed",
+        lastError: LEASE_EXPIRED,
+        finishedAt: now,
+        lockedUntil: null,
+        lockedBy: null,
+        claimToken: null,
+        updatedAt: now,
+      })
+      .where(and(eq(jobs.queue, queue), eq(jobs.status, "running"), lte(jobs.lockedUntil, now), gte(jobs.attempts, jobs.maxAttempts)))
+      .returning({ id: jobs.id });
+    if (failed.length > 0) await recordScheduleFailure({ ...ctx, db: tx }, queue, LEASE_EXPIRED);
+  });
 }
 
 function isHeld(claimed: ClaimedJob) {

package/src/server/index.ts

@@ -23,4 +23,13 @@ export {
   type EnqueueDueSchedulesInput,
   type ScheduleState,
 } from "./schedules.js";
+export { advanceSyncCursor, getSyncCursor, SYNC_SOURCE, type AdvanceSyncCursorInput } from "./sync-cursors.js";
+export {
+  listSyncItems,
+  processSyncItem,
+  pruneSyncItems,
+  type ListSyncItemsInput,
+  type ProcessSyncItemInput,
+  type SyncItemContext,
+} from "./sync-items.js";
 export { runDueJobs, type RunDueJobsOptions, type RunDueJobsReport } from "./run-due.js";

package/src/server/schedules.ts

@@ -23,11 +23,17 @@ export interface ScheduleState {
   readonly lastEnqueuedAt: Date | null;
   /** When a job of the schedule's queue last succeeded; null when none has yet. */
   readonly lastSucceededAt: Date | null;
+  /** When a job of the schedule's queue last ended `failed` (attempts used up, or failed for good); null when none has. */
+  readonly lastFailedAt: Date | null;
+  /** That job's error code (the handler's code, `jobs.handler_threw`, `jobs.lease_expired`…); null when none failed. */
+  readonly lastFailureCode: string | null;
   /**
    * True when no run succeeded within two periods (counted from when a worker first saw the schedule), or no worker
    * has seen it at all: no worker runs its queue, or its runs keep failing.
    */
   readonly isLate: boolean;
+  /** True when the latest outcome is a failure: `lastFailedAt` is later than `lastSucceededAt`, or nothing succeeded. */
+  readonly isFailing: boolean;
 }
 
 export interface EnqueueDueSchedulesInput {
@@ -90,6 +96,13 @@ export async function recordScheduleSuccess(ctx: JobsContext, queue: string): Pr
   await ctx.db.update(schedules).set({ lastSucceededAt: now, updatedAt: now }).where(eq(schedules.name, queue));
 }
 
+/** Notes a failed job of a scheduled queue (terminal, not a retried attempt) with its error code. */
+export async function recordScheduleFailure(ctx: JobsContext, queue: string, code: string): Promise<void> {
+  if (!Object.hasOwn(getJobsOptions(ctx.config).schedules, queue)) return;
+  const now = ctx.clock.now();
+  await ctx.db.update(schedules).set({ lastFailedAt: now, lastFailureCode: code, updatedAt: now }).where(eq(schedules.name, queue));
+}
+
 /** The state of every configured schedule, by queue name. */
 export async function getScheduleStates(ctx: JobsContext): Promise<Map<string, ScheduleState>> {
   const options = getJobsOptions(ctx.config);
@@ -103,7 +116,11 @@ export async function getScheduleStates(ctx: JobsContext): Promise<Map<string, S
       const row = rows.find((candidate) => candidate.name === name);
       const since = row?.lastSucceededAt ?? row?.createdAt ?? null;
       const isLate = since === null || now - since.getTime() > 2 * everySeconds * 1000;
-      return [name, { everySeconds, lastEnqueuedAt: row?.lastSlotAt ?? null, lastSucceededAt: row?.lastSucceededAt ?? null, isLate }];
+      const lastSucceededAt = row?.lastSucceededAt ?? null;
+      const lastFailedAt = row?.lastFailedAt ?? null;
+      const isFailing = lastFailedAt !== null && (lastSucceededAt === null || lastFailedAt > lastSucceededAt);
+      const lastFailureCode = row?.lastFailureCode ?? null;
+      return [name, { everySeconds, lastEnqueuedAt: row?.lastSlotAt ?? null, lastSucceededAt, lastFailedAt, lastFailureCode, isLate, isFailing }];
     }),
   );
 }

package/src/server/sync-cursors.ts

@@ -0,0 +1,67 @@
+// The cursor of a polled source: where its next read starts. It moves only by compare-and-set, so a worker that read
+// an older cursor cannot move it back over a newer one; a crash before the move re-reads the batch instead of skipping it.
+import { err, ok, type Result } from "@softure-ai/core";
+import { and, eq } from "drizzle-orm";
+import { syncCursors } from "../schema.js";
+import type { JobsContext } from "./context.js";
+
+/** A source name: lowercase letters, digits, `_ . : -`, starting with a letter, up to 127 characters. */
+export const SYNC_SOURCE = /^[a-z][a-z0-9_.:-]{0,126}$/;
+const MAX_CURSOR_LENGTH = 2000;
+
+export interface AdvanceSyncCursorInput {
+  readonly source: string;
+  /** The cursor the batch was read from, as `getSyncCursor` returned it; null for the first batch. */
+  readonly from: string | null;
+  /** Where the next batch starts. */
+  readonly to: string;
+}
+
+/** The source's cursor, or null before its first `advanceSyncCursor`. Throws on an invalid source name. */
+export async function getSyncCursor(ctx: JobsContext, source: string): Promise<string | null> {
+  checkSyncSource(source);
+  const [row] = await ctx.db.select({ cursor: syncCursors.cursor }).from(syncCursors).where(eq(syncCursors.source, source));
+  return row?.cursor ?? null;
+}
+
+/**
+ * Moves the source's cursor from `from` to `to`, after the batch read from `from` was handled. Refused with
+ * `jobs.sync_cursor_moved` when the stored cursor is no longer `from` (another worker moved it first): re-read from
+ * `getSyncCursor`. `ctx.db` may be a transaction, so the move commits with the batch's last writes. Throws on an
+ * invalid source name or cursor; database failures propagate.
+ */
+export async function advanceSyncCursor(ctx: JobsContext, input: AdvanceSyncCursorInput): Promise<Result<undefined, "jobs.sync_cursor_moved">> {
+  checkSyncSource(input.source);
+  checkCursor(input.to);
+  if (input.from !== null) checkCursor(input.from);
+  const now = ctx.clock.now();
+
+  if (input.from === null) {
+    const inserted = await ctx.db
+      .insert(syncCursors)
+      .values({ source: input.source, cursor: input.to, createdAt: now, updatedAt: now })
+      .onConflictDoNothing({ target: syncCursors.source })
+      .returning({ source: syncCursors.source });
+    return inserted.length === 1 ? ok() : err("jobs.sync_cursor_moved");
+  }
+
+  const updated = await ctx.db
+    .update(syncCursors)
+    .set({ cursor: input.to, updatedAt: now })
+    .where(and(eq(syncCursors.source, input.source), eq(syncCursors.cursor, input.from)))
+    .returning({ source: syncCursors.source });
+  return updated.length === 1 ? ok() : err("jobs.sync_cursor_moved");
+}
+
+/** Throws on a name the tables refuse: a typo in code, not a user error. */
+export function checkSyncSource(source: string): void {
+  if (!SYNC_SOURCE.test(source)) {
+    throw new RangeError(`@softure-ai/jobs: "${source}" is not a sync source name: lowercase letters, digits, _ . : and -, starting with a letter, up to 127 characters`);
+  }
+}
+
+function checkCursor(cursor: string): void {
+  if (cursor.length === 0 || cursor.length > MAX_CURSOR_LENGTH) {
+    throw new RangeError(`@softure-ai/jobs: a sync cursor must have 1 to ${String(MAX_CURSOR_LENGTH)} characters`);
+  }
+}