@structure-ai/jobs 0.0.13 → 0.0.15

package/README.md

@@ -52,7 +52,7 @@ Payloads are `Schema<P, string>` (usually `Schema.parseJson(...)`) — stored as
 
 ## Worker lifecycle and roles
 
-`workerLayer({ role })` follows the platform's `SERVICE_ROLE` convention: `api` processes only schedule (the layer logs and does nothing), `worker`/`all` fork the dispatch loop. The loop registers a `Shutdown` finalizer: on shutdown it stops claiming, waits for in-flight handlers (bounded by the coordinator's finalizer timeout), then exits — no in-flight job is killed. `jobsSettings` (`@structure-ai/config`) maps `SERVICE_ROLE`, `JOBS_POLL_INTERVAL`, `JOBS_BATCH_SIZE`, `JOBS_LEASE`, `JOBS_TABLE_PREFIX`.
+`workerLayer({ role })` follows the platform's `SERVICE_ROLE` convention: `api` processes only schedule (the layer logs and does nothing), `worker`/`all` fork the dispatch loop. The loop runs at most `concurrency` handlers at once (default `batchSize`; a semaphore enforces it) and claims only as many rows as it has free slots, waking when a slot frees rather than polling, so a backlog never turns into an unbounded fan-out. Every claim mints a `lease_owner` token, and every completion, retry, dead-letter and heartbeat is fenced on it (`WHERE id = ? AND status = 'running' AND lease_owner = ?`): a worker whose lease another worker reclaimed affects zero rows and logs `job lease lost; write skipped` instead of destroying the row the other worker is running (the fence is the token, not the expiry, so a late completion nobody reclaimed still lands). It registers a `Shutdown` finalizer: on shutdown it stops claiming and waits for in-flight handlers, up to `drainTimeout` when set (keep it below the coordinator's finalizer timeout) or until the coordinator cuts the finalizer. A drain that gives up either way is loud and reversible: it logs `jobs worker abandoned drain` at error level with the job ids and returns every in-flight job to the queue (status `queued`, lease released, claimable at once), and the abandoned handlers' late writes are fenced out; no handler is interrupted. `jobsSettings` (`@structure-ai/config`) maps `SERVICE_ROLE`, `JOBS_POLL_INTERVAL`, `JOBS_BATCH_SIZE`, `JOBS_CONCURRENCY`, `JOBS_LEASE`, `JOBS_DRAIN_TIMEOUT`, `JOBS_TABLE_PREFIX`.
 
 ## Observability
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@structure-ai/jobs",
-  "version": "0.0.13",
+  "version": "0.0.15",
   "description": "Delayed and recurring jobs on PostgreSQL: named schema-typed handlers, SKIP LOCKED dispatch with heartbeat leases, cron scheduling with timezones, at-least-once delivery, dead letters, graceful drain, per-job metrics.",
   "license": "MIT",
   "type": "module",
@@ -24,9 +24,9 @@
     "test": "bun test"
   },
   "dependencies": {
-    "@structure-ai/config": "0.0.13",
-    "@structure-ai/observability": "0.0.13",
-    "@structure-ai/runtime": "0.0.13",
+    "@structure-ai/config": "0.0.15",
+    "@structure-ai/observability": "0.0.15",
+    "@structure-ai/runtime": "0.0.15",
     "@effect/sql": "^0.52.1",
     "@effect/sql-pg": "^0.53.0",
     "effect": "^3.22.1"

package/src/scheduler.ts

@@ -1,14 +1,17 @@
 import * as SqlClient from "@effect/sql/SqlClient";
+import type * as Statement from "@effect/sql/Statement";
 import { Correlation, Metrics } from "@structure-ai/observability";
 import { Shutdown } from "@structure-ai/runtime";
 import {
   Context,
   Data,
+  Deferred,
   Duration,
   Effect,
   Fiber,
   Layer,
   Metric,
+  Option,
   Schema as S,
   Schedule,
 } from "effect";
@@ -123,10 +126,25 @@ export interface RecurOptions {
 export interface WorkerOptions {
   /** Idle poll interval. Default 1s. */
   readonly pollInterval?: Duration.DurationInput;
-  /** Claims per poll. Default 10. */
+  /** Rows claimed per poll, never more than the free concurrency. Default 10. */
   readonly batchSize?: number;
+  /**
+   * Ceiling on handlers executing at once in this worker, enforced by a
+   * semaphore; the claim loop never takes more rows than it has free slots
+   * and waits for a slot instead of polling when full. Default `batchSize`.
+   */
+  readonly concurrency?: number;
   /** Lease held while a handler runs; expiry makes the row reclaimable. Default 60s. */
   readonly lease?: Duration.DurationInput;
+  /**
+   * How long the shutdown drain waits for in-flight handlers before giving
+   * up. Keep it below the coordinator's finalizer timeout so the worker,
+   * not the coordinator, decides. On overrun (or when the coordinator cuts
+   * the drain first) every job still in flight is returned to the queue:
+   * lease released, claimable at once, logged by id at error level; its
+   * late completion is fenced out. Default: wait until the coordinator cuts.
+   */
+  readonly drainTimeout?: Duration.DurationInput;
   /**
    * Role-aware boot: `api` runs no worker (scheduling only), `worker` and
    * `all` do. Default `all`.
@@ -144,6 +162,8 @@ interface QueueRow {
   readonly cron_timezone: string | null;
   readonly correlation_id: string | null;
   readonly run_at: Date | string;
+  /** The token minted by the claim that produced this row; the fence for every write to it. */
+  readonly lease_owner: string;
 }
 
 export interface SchedulerService {
@@ -168,7 +188,8 @@ export interface SchedulerService {
   readonly registeredJobs: () => ReadonlyArray<string>;
   /**
    * Runs the worker loop until the Shutdown coordinator triggers, then
-   * drains: no in-flight job is interrupted. Claims with
+   * drains: no in-flight job is interrupted, and a drain that outlives its
+   * budget returns the in-flight jobs to the queue. Claims with
    * `FOR UPDATE SKIP LOCKED`, heartbeats its lease, retries transient
    * failures with jittered backoff, and dead-letters permanent failures and
    * exhausted attempts.
@@ -232,16 +253,58 @@ export const makeScheduler = (
 
     const maxAttemptsFor = (jobName: string): number => handlers.get(jobName)?.maxAttempts ?? 5;
 
-    const deadLetter = (row: QueueRow, reason: string): Effect.Effect<void, JobQueueError> =>
+    /**
+     * Runs a terminal write scoped to the row AND the lease this worker
+     * holds on it. A worker whose lease was reclaimed by another affects
+     * zero rows and logs that it lost the lease instead of destroying the
+     * row the other worker is running. The fence is the owner token, not
+     * the lease expiry: a late completion whose lease expired but was not
+     * reclaimed still lands, which is a job done once rather than twice.
+     */
+    const fenced = (
+      row: QueueRow,
+      operation: string,
+      statement: Statement.Fragment,
+    ): Effect.Effect<boolean, unknown> =>
       Effect.gen(function* () {
-        yield* sql`
-          INSERT INTO ${sql(tables.deadLetters)}
-            (id, job_name, payload, attempts, last_error, correlation_id, dead_at)
-          VALUES
-            (${crypto.randomUUID()}, ${row.job_name}, ${row.payload}, ${row.attempt},
-             ${reason.slice(0, 2_048)}, ${row.correlation_id}, ${now().toISOString()})
+        const affected = yield* sql<{ readonly id: string }>`
+          ${statement}
+          WHERE id = ${row.id} AND status = 'running' AND lease_owner = ${row.lease_owner}
+          RETURNING id
         `;
-        yield* sql`DELETE FROM ${sql(tables.queue)} WHERE id = ${row.id}`;
+        if (affected.length > 0) return true;
+        yield* Effect.logWarning("job lease lost; write skipped").pipe(
+          Effect.annotateLogs({
+            jobId: row.id,
+            jobName: row.job_name,
+            jobAttempt: row.attempt,
+            jobOperation: operation,
+          }),
+        );
+        return false;
+      });
+
+    const deadLetter = (row: QueueRow, reason: string): Effect.Effect<void, JobQueueError> =>
+      Effect.gen(function* () {
+        // The fence comes first, in one transaction with the dead-letter
+        // insert: a worker that lost its lease must not file a dead letter
+        // for a job another worker is running, and a job whose queue row is
+        // gone must always have its dead letter.
+        const owned = yield* sql.withTransaction(
+          Effect.gen(function* () {
+            const owned = yield* fenced(row, "dead-letter", sql`DELETE FROM ${sql(tables.queue)}`);
+            if (!owned) return false;
+            yield* sql`
+              INSERT INTO ${sql(tables.deadLetters)}
+                (id, job_name, payload, attempts, last_error, correlation_id, dead_at)
+              VALUES
+                (${crypto.randomUUID()}, ${row.job_name}, ${row.payload}, ${row.attempt},
+                 ${reason.slice(0, 2_048)}, ${row.correlation_id}, ${now().toISOString()})
+            `;
+            return true;
+          }),
+        );
+        if (!owned) return;
         yield* Metric.increment(deadLettered);
         yield* Effect.logError("job dead-lettered").pipe(
           Effect.annotateLogs({
@@ -256,7 +319,7 @@ export const makeScheduler = (
     const completeSuccess = (row: QueueRow): Effect.Effect<void, JobQueueError> =>
       Effect.gen(function* () {
         if (row.cron_expr === null) {
-          yield* sql`DELETE FROM ${sql(tables.queue)} WHERE id = ${row.id}`;
+          yield* fenced(row, "complete-success", sql`DELETE FROM ${sql(tables.queue)}`);
           return;
         }
         const fields = yield* parseCron(row.cron_expr);
@@ -269,27 +332,34 @@ export const makeScheduler = (
           timezone,
         );
         if (next === undefined) {
-          yield* sql`DELETE FROM ${sql(tables.queue)} WHERE id = ${row.id}`;
+          yield* fenced(row, "complete-success", sql`DELETE FROM ${sql(tables.queue)}`);
           return;
         }
-        yield* sql`
-          UPDATE ${sql(tables.queue)}
-          SET status = 'queued', run_at = ${next.toISOString()}, attempt = 0,
-              lease_expires_at = NULL, updated_at = ${now().toISOString()}
-          WHERE id = ${row.id}
-        `;
+        yield* fenced(
+          row,
+          "complete-success",
+          sql`
+            UPDATE ${sql(tables.queue)}
+            SET status = 'queued', run_at = ${next.toISOString()}, attempt = 0,
+                lease_expires_at = NULL, lease_owner = NULL, updated_at = ${now().toISOString()}
+          `,
+        );
       }).pipe(Effect.mapError((cause) => queueError("complete-success", cause)));
 
     const rescheduleRetry = (row: QueueRow, reason: string): Effect.Effect<void, JobQueueError> =>
       Effect.gen(function* () {
         const backoff = backoffMillis(row.attempt, random);
-        yield* sql`
-          UPDATE ${sql(tables.queue)}
-          SET status = 'queued', run_at = ${new Date(now().getTime() + backoff).toISOString()},
-              lease_expires_at = NULL, last_error = ${reason.slice(0, 2_048)},
-              updated_at = ${now().toISOString()}
-          WHERE id = ${row.id}
-        `;
+        const owned = yield* fenced(
+          row,
+          "reschedule-retry",
+          sql`
+            UPDATE ${sql(tables.queue)}
+            SET status = 'queued', run_at = ${new Date(now().getTime() + backoff).toISOString()},
+                lease_expires_at = NULL, lease_owner = NULL, last_error = ${reason.slice(0, 2_048)},
+                updated_at = ${now().toISOString()}
+          `,
+        );
+        if (!owned) return;
         yield* Metric.increment(retried);
         yield* Effect.logWarning("job attempt failed, retrying with backoff").pipe(
           Effect.annotateLogs({
@@ -308,6 +378,7 @@ export const makeScheduler = (
     ): Effect.Effect<ReadonlyArray<QueueRow>, JobQueueError> => {
       const timestamp = now();
       const leaseUntil = new Date(timestamp.getTime() + leaseMillis);
+      const owner = crypto.randomUUID();
       return sql<QueueRow>`
         WITH picked AS (
           SELECT id FROM ${sql(tables.queue)}
@@ -320,12 +391,13 @@ export const makeScheduler = (
         UPDATE ${sql(tables.queue)} queue
         SET status = 'running', attempt = queue.attempt + 1,
             lease_expires_at = ${leaseUntil.toISOString()},
+            lease_owner = ${owner},
             updated_at = ${timestamp.toISOString()}
         FROM picked
         WHERE queue.id = picked.id
         RETURNING queue.id, queue.job_name, queue.payload, queue.attempt,
                   queue.max_attempts, queue.cron_expr, queue.cron_timezone,
-                  queue.correlation_id, queue.run_at
+                  queue.correlation_id, queue.run_at, queue.lease_owner
       `.pipe(
         Effect.mapError((cause) => queueError("claim", cause)),
         Effect.tap((rows) =>
@@ -349,10 +421,12 @@ export const makeScheduler = (
 
       const heartbeat: Effect.Effect<void> = Effect.gen(function* () {
         const until = new Date(now().getTime() + leaseMillis);
+        // Fenced like every other write: an evicted worker must not keep
+        // extending the lease another worker now owns.
         yield* sql`
           UPDATE ${sql(tables.queue)}
           SET lease_expires_at = ${until.toISOString()}
-          WHERE id = ${row.id} AND status = 'running'
+          WHERE id = ${row.id} AND status = 'running' AND lease_owner = ${row.lease_owner}
         `.pipe(Effect.asVoid);
       }).pipe(
         Effect.asVoid,
@@ -427,15 +501,61 @@ export const makeScheduler = (
           workerOptions.pollInterval === undefined
             ? 1_000
             : Duration.toMillis(Duration.decode(workerOptions.pollInterval));
-        const batchSize = workerOptions.batchSize ?? 10;
+        const batchSize = Math.max(1, workerOptions.batchSize ?? 10);
+        const concurrency = Math.max(1, workerOptions.concurrency ?? batchSize);
         const leaseMillis =
           workerOptions.lease === undefined
             ? 60_000
             : Duration.toMillis(Duration.decode(workerOptions.lease));
 
-        const inflight = new Set<Fiber.RuntimeFiber<void, unknown>>();
+        const drainTimeout =
+          workerOptions.drainTimeout === undefined
+            ? undefined
+            : Duration.decode(workerOptions.drainTimeout);
+
+        const inflight = new Map<Fiber.RuntimeFiber<void, unknown>, QueueRow>();
+        // The bound itself: a handler runs only while holding one permit, so
+        // even a miscounted claim can never exceed `concurrency` executions.
+        const permits = yield* Effect.makeSemaphore(concurrency);
+        // Signalled whenever an in-flight fiber ends, so a full worker wakes
+        // as soon as a slot frees instead of sleeping a whole poll interval.
+        let slotFreed = yield* Deferred.make<void>();
         let stopRequested = false;
 
+        const awaitInflight: Effect.Effect<void> = Effect.whileLoop({
+          while: () => inflight.size > 0,
+          body: () => Effect.sleep("10 millis"),
+          step: () => undefined,
+        });
+
+        // The drain gave up (its own budget, or the coordinator's cut): return
+        // every in-flight job to the queue so another worker picks it up at
+        // once instead of after a full lease, and say which ones, loudly.
+        // The handlers keep running until the process ends; their writes are
+        // fenced out by the released lease.
+        const abandonDrain: Effect.Effect<void> = Effect.gen(function* () {
+          const rows = [...inflight.values()];
+          if (rows.length === 0) return;
+          yield* Effect.logError("jobs worker abandoned drain").pipe(
+            Effect.annotateLogs({
+              jobIds: rows.map((row) => row.id).join(","),
+              jobCount: rows.length,
+            }),
+          );
+          const timestamp = now().toISOString();
+          yield* Effect.forEach(
+            rows,
+            (row) =>
+              sql`
+                UPDATE ${sql(tables.queue)}
+                SET status = 'queued', run_at = ${timestamp}, lease_expires_at = NULL,
+                    lease_owner = NULL, updated_at = ${timestamp}
+                WHERE id = ${row.id} AND status = 'running' AND lease_owner = ${row.lease_owner}
+              `.pipe(Effect.asVoid),
+            { discard: true },
+          );
+        }).pipe(Effect.catchAllCause(() => Effect.void));
+
         yield* shutdown.onShutdown(
           "jobs-worker",
           Effect.sync(() => {
@@ -443,12 +563,16 @@ export const makeScheduler = (
           }).pipe(
             Effect.zipRight(Effect.logInfo("jobs worker draining")),
             Effect.zipRight(
-              Effect.whileLoop({
-                while: () => inflight.size > 0,
-                body: () => Effect.sleep("10 millis"),
-                step: () => undefined,
-              }),
+              drainTimeout === undefined
+                ? awaitInflight
+                : awaitInflight.pipe(
+                    Effect.timeoutOption(drainTimeout),
+                    Effect.flatMap((completed) =>
+                      Option.isNone(completed) ? abandonDrain : Effect.void,
+                    ),
+                  ),
             ),
+            Effect.onInterrupt(() => abandonDrain),
           ),
         );
 
@@ -459,26 +583,36 @@ export const makeScheduler = (
               const shuttingDown = yield* shutdown.isShuttingDown;
               if (shuttingDown) stopRequested = true;
               if (stopRequested) return;
-              const rows = yield* Effect.orDie(claim(batchSize, leaseMillis));
+              // Claim at most as many rows as there are free slots: a claimed
+              // row that could not start would sit on a ticking lease.
+              const waiter = slotFreed;
+              const capacity = Math.min(batchSize, concurrency - inflight.size);
+              if (capacity <= 0) {
+                yield* Deferred.await(waiter).pipe(Effect.timeout(pollMillis), Effect.ignore);
+                return;
+              }
+              const rows = yield* Effect.orDie(claim(capacity, leaseMillis));
               if (rows.length === 0) {
                 yield* Effect.sleep(pollMillis);
                 return;
               }
               for (const row of rows) {
-                const fiber = yield* Effect.fork(execute(row, leaseMillis));
-                inflight.add(fiber);
-                void fiber.addObserver(() => inflight.delete(fiber));
+                const fiber = yield* Effect.fork(permits.withPermits(1)(execute(row, leaseMillis)));
+                inflight.set(fiber, row);
+                void fiber.addObserver(() => {
+                  inflight.delete(fiber);
+                  Deferred.unsafeDone(slotFreed, Effect.void);
+                });
+              }
+              if (inflight.size >= concurrency) {
+                slotFreed = yield* Deferred.make<void>();
               }
             }),
           step: () => undefined,
         });
         yield* loop;
         // Graceful drain: wait for in-flight handlers to finish.
-        yield* Effect.whileLoop({
-          while: () => inflight.size > 0,
-          body: () => Effect.sleep("10 millis"),
-          step: () => undefined,
-        });
+        yield* awaitInflight;
         yield* Effect.logInfo("jobs worker drained").pipe(
           Effect.annotateLogs({ drained: inflight.size }),
         );

package/src/schema.ts

@@ -23,7 +23,8 @@ export const tableNames = (options: AdapterOptions = {}): TableNames => {
  * Creates the jobs schema in one transaction, `@structure-ai/migrations`
  * style (idempotent DDL a designated migrator can run at boot). The queue
  * index covers the dispatch predicate (`status = queued AND run_at <= now`
- * plus lease-expiry reclaims).
+ * plus lease-expiry reclaims). `lease_owner` is the per-claim token every
+ * completion, retry, dead-letter and heartbeat is fenced on.
  */
 export const migrate = (
   options: AdapterOptions = {},
@@ -43,6 +44,7 @@ export const migrate = (
         attempt INTEGER NOT NULL DEFAULT 0,
         max_attempts INTEGER NOT NULL DEFAULT 5,
         lease_expires_at TIMESTAMPTZ,
+        lease_owner TEXT,
         last_error TEXT,
         correlation_id TEXT,
         causation_id TEXT,
@@ -50,6 +52,10 @@ export const migrate = (
         updated_at TIMESTAMPTZ NOT NULL
       )
     `;
+    // Instances created before the lease fence existed gain the owner column.
+    yield* sql`
+      ALTER TABLE ${sql(tables.queue)} ADD COLUMN IF NOT EXISTS lease_owner TEXT
+    `;
     yield* sql`
       CREATE INDEX IF NOT EXISTS ${sql(`${tables.queue}_dispatch_idx`)}
       ON ${sql(tables.queue)} (status, run_at)

package/src/settings.ts

@@ -16,13 +16,24 @@ export const jobsSettings = Settings.struct({
     default: Duration.seconds(1),
   }),
   batchSize: Settings.int("JOBS_BATCH_SIZE", {
-    description: "jobs claimed per poll",
+    description: "jobs claimed per poll (never more than the free concurrency)",
     default: 10,
   }),
+  concurrency: Settings.optional(
+    Settings.int("JOBS_CONCURRENCY", {
+      description: "ceiling on jobs executing at once per worker (default: JOBS_BATCH_SIZE)",
+    }),
+  ),
   lease: Settings.duration("JOBS_LEASE", {
     description: "dispatch lease duration before a running job becomes reclaimable",
     default: Duration.seconds(60),
   }),
+  drainTimeout: Settings.optional(
+    Settings.duration("JOBS_DRAIN_TIMEOUT", {
+      description:
+        "how long the shutdown drain waits for in-flight jobs before returning them to the queue (keep below the coordinator's finalizer timeout; default: until the coordinator cuts)",
+    }),
+  ),
   tablePrefix: Settings.string("JOBS_TABLE_PREFIX", {
     description: "jobs table name prefix",
     default: "jobs_",