@structure-ai/jobs 0.1.1 → 0.2.1

package/README.md

@@ -60,8 +60,12 @@ Per-job metrics under bounded, handler-derived names: `job_<name>_calls_total` /
 
 ## Errors
 
-`UnknownJob`, `InvalidJobPayload` (permanent), `JobQueueError` (transient), `InvalidCronExpression` (permanent, lists every problem). All classified per the framework taxonomy.
+`UnknownJob`, `InvalidJobPayload` (permanent), `JobQueueError` (transient), `InvalidCronExpression` (permanent, lists every problem). All classified per the framework taxonomy. `migrate` exposes `SqlError`. `runWorker` exposes `WorkerError`: queue errors, plus handler failures retained in compound causes.
 
 ## Schema
 
 Two tables created by the idempotent `migrate` (own prefix, `@structure-ai/migrations`-compatible DDL): `jobs_queue` (status `queued|running`, `run_at`, `cron_expr`, `cron_timezone`, `attempt`, `max_attempts`, `lease_expires_at`, `last_error`, correlation fields) with dispatch and lease indexes, and `jobs_dead_letters`.
+
+Only a single expected handler failure participates in job retry/dead-letter policy. A handler defect or cancellation, including one combined with an expected failure, stops the worker with its cause intact. Claim, completion and heartbeat failures also stop the worker. A failed heartbeat interrupts its handler because the lease can no longer be maintained. Rows remain reclaimable after lease expiry.
+
+`workerLayer` logs unexpected worker termination and registers a `jobs-worker` readiness check when a `Readiness` service is available. A stopped worker reports not ready. Failed lease release during shutdown is logged; rows remain reclaimable after expiry.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@structure-ai/jobs",
-  "version": "0.1.1",
+  "version": "0.2.1",
   "description": "Delayed and recurring jobs on PostgreSQL: named schema-typed handlers, SKIP LOCKED dispatch with heartbeat leases, cron scheduling with timezones, at-least-once delivery, dead letters, graceful drain, per-job metrics.",
   "license": "MIT",
   "type": "module",
@@ -24,15 +24,16 @@
     "test": "bun test"
   },
   "dependencies": {
-    "@structure-ai/config": "0.1.1",
-    "@structure-ai/observability": "0.1.1",
-    "@structure-ai/runtime": "0.1.1",
+    "@structure-ai/config": "0.2.1",
+    "@structure-ai/observability": "0.2.1",
+    "@structure-ai/runtime": "0.2.1",
     "@effect/sql": "^0.52.1",
     "@effect/sql-pg": "^0.53.0",
     "effect": "^3.22.1"
   },
   "devDependencies": {
     "@types/bun": "^1.3.14",
-    "typescript": "^5.9.2"
+    "typescript": "^5.9.2",
+    "@effect/experimental": "^0.61.0"
   }
 }

package/src/index.ts

@@ -34,6 +34,7 @@ export {
   type SchedulerService,
   schedulerLayer,
   UnknownJob,
+  type WorkerError,
   type WorkerOptions,
 } from "./scheduler.js";
 export { type AdapterOptions, migrate, type TableNames, tableNames } from "./schema.js";

package/src/layer.ts

@@ -1,8 +1,8 @@
 import type * as SqlClient from "@effect/sql/SqlClient";
 import type { SqlError } from "@effect/sql/SqlError";
 import { PgClient } from "@effect/sql-pg";
-import type { Shutdown } from "@structure-ai/runtime";
-import { Effect, Layer, Redacted } from "effect";
+import { Readiness, type Shutdown } from "@structure-ai/runtime";
+import { Cause, Effect, Layer, Option, Redacted, Ref } from "effect";
 import {
   Scheduler,
   type SchedulerOptions,
@@ -64,6 +64,20 @@ export const workerLayer = (
         );
         return;
       }
-      yield* Effect.forkScoped(scheduler.runWorker(options));
+      const readiness = yield* Effect.serviceOption(Readiness);
+      const running = yield* Ref.make(true);
+      if (Option.isSome(readiness)) {
+        yield* readiness.value.register({ name: "jobs-worker", run: Ref.get(running) });
+      }
+      yield* Effect.forkScoped(
+        scheduler.runWorker(options).pipe(
+          Effect.tapErrorCause((cause) =>
+            Cause.isInterruptedOnly(cause)
+              ? Effect.void
+              : Effect.logError("jobs worker stopped after failure", cause),
+          ),
+          Effect.ensuring(Ref.set(running, false)),
+        ),
+      );
     }),
   );

package/src/scheduler.ts

@@ -3,12 +3,14 @@ import type * as Statement from "@effect/sql/Statement";
 import { Correlation, Metrics } from "@structure-ai/observability";
 import { Shutdown } from "@structure-ai/runtime";
 import {
+  Cause,
   Context,
   Data,
   Deferred,
   Duration,
   Effect,
-  Fiber,
+  Exit,
+  type Fiber,
   Layer,
   Metric,
   Option,
@@ -58,6 +60,9 @@ export interface JobFailure {
   readonly classification: "transient" | "permanent";
 }
 
+/** Worker failures include handler failures only when part of an unhandled compound cause. */
+export type WorkerError = JobQueueError | JobFailure;
+
 // --- definitions ------------------------------------------------------------------
 
 /** Minimal reference for scheduling: a name plus its payload codec schema. */
@@ -194,7 +199,7 @@ export interface SchedulerService {
    * failures with jittered backoff, and dead-letters permanent failures and
    * exhausted attempts.
    */
-  readonly runWorker: (options?: WorkerOptions) => Effect.Effect<void, never, Shutdown>;
+  readonly runWorker: (options?: WorkerOptions) => Effect.Effect<void, WorkerError, Shutdown>;
 }
 
 export class Scheduler extends Context.Tag("@structure-ai/jobs/Scheduler")<
@@ -406,7 +411,7 @@ export const makeScheduler = (
       );
     };
 
-    const execute = (row: QueueRow, leaseMillis: number): Effect.Effect<void> => {
+    const execute = (row: QueueRow, leaseMillis: number): Effect.Effect<void, WorkerError> => {
       const handler: StoredJobHandler | undefined = handlers.get(row.job_name);
       const context: JobContext = {
         jobId: row.id,
@@ -419,7 +424,7 @@ export const makeScheduler = (
         causationId: row.id,
       });
 
-      const heartbeat: Effect.Effect<void> = Effect.gen(function* () {
+      const heartbeat: Effect.Effect<void, JobQueueError> = Effect.gen(function* () {
         const until = new Date(now().getTime() + leaseMillis);
         // Fenced like every other write: an evicted worker must not keep
         // extending the lease another worker now owns.
@@ -432,35 +437,34 @@ export const makeScheduler = (
         Effect.asVoid,
         Effect.repeat(Schedule.spaced(`${Math.max(1, Math.floor(leaseMillis / 3))} millis`)),
         Effect.asVoid,
-        Effect.catchAllCause(() => Effect.void),
+        Effect.mapError((cause) => queueError("heartbeat", cause)),
       );
 
-      const runOutcome: Effect.Effect<void> = Effect.gen(function* () {
+      const runOutcome: Effect.Effect<void, WorkerError> = Effect.gen(function* () {
         if (handler === undefined) {
           yield* Effect.logError("job dispatched with no registered handler").pipe(
             Effect.annotateLogs({ jobId: row.id, jobName: row.job_name }),
           );
-          yield* deadLetter(row, "unknown-job").pipe(Effect.orDie);
+          yield* deadLetter(row, "unknown-job");
           return;
         }
         const decoded = yield* S.decodeUnknown(handler.payloadSchema)(row.payload).pipe(
           Effect.either,
         );
         if (decoded._tag === "Left") {
-          yield* deadLetter(row, `invalid-payload: ${String(decoded.left).slice(0, 128)}`).pipe(
-            Effect.orDie,
-          );
+          yield* deadLetter(row, `invalid-payload: ${String(decoded.left).slice(0, 128)}`);
           return;
         }
         const failure = yield* handler
           .handle(decoded.right, context)
-          .pipe(Metrics.track(`job_${row.job_name}`, boundaryFor(row.job_name)), Effect.either);
-        if (failure._tag === "Right") {
+          .pipe(Metrics.track(`job_${row.job_name}`, boundaryFor(row.job_name)), Effect.exit);
+        if (Exit.isSuccess(failure)) {
           yield* Metric.increment(succeeded);
-          yield* completeSuccess(row).pipe(Effect.orDie);
+          yield* completeSuccess(row);
           return;
         }
-        const error = failure.left;
+        if (!Cause.isFailType(failure.cause)) return yield* Effect.failCause(failure.cause);
+        const error = failure.cause.error;
         yield* Effect.logWarning("job attempt failed").pipe(
           Effect.annotateLogs({
             jobId: row.id,
@@ -472,29 +476,20 @@ export const makeScheduler = (
         );
         const exhausted = row.attempt >= (row.max_attempts || maxAttemptsFor(row.job_name));
         if (error.classification === "permanent" || exhausted) {
-          yield* deadLetter(row, error.reason).pipe(Effect.orDie);
+          yield* deadLetter(row, error.reason);
           return;
         }
-        yield* rescheduleRetry(row, error.reason).pipe(Effect.orDie);
+        yield* rescheduleRetry(row, error.reason);
       });
 
-      return Effect.gen(function* () {
-        const heartbeatFiber = yield* Effect.fork(
-          heartbeat.pipe(Effect.catchAllCause(() => Effect.void)),
-        );
-        yield* runOutcome.pipe(
-          Effect.ensuring(
-            Fiber.interrupt(heartbeatFiber).pipe(
-              Effect.catchAllCause(() => Effect.void),
-              Effect.asVoid,
-            ),
-          ),
-          correlation,
-        );
-      });
+      // A failed heartbeat invalidates this execution's lease. Stop the
+      // handler and propagate the failure to the worker's supervisor.
+      return runOutcome.pipe(Effect.raceFirst(heartbeat), correlation);
     };
 
-    const runWorker = (workerOptions: WorkerOptions = {}): Effect.Effect<void, never, Shutdown> =>
+    const runWorker = (
+      workerOptions: WorkerOptions = {},
+    ): Effect.Effect<void, WorkerError, Shutdown> =>
       Effect.gen(function* () {
         const shutdown = yield* Shutdown;
         const pollMillis =
@@ -513,7 +508,8 @@ export const makeScheduler = (
             ? undefined
             : Duration.decode(workerOptions.drainTimeout);
 
-        const inflight = new Map<Fiber.RuntimeFiber<void, unknown>, QueueRow>();
+        const inflight = new Map<Fiber.RuntimeFiber<void, WorkerError>, QueueRow>();
+        const workerFailed = yield* Deferred.make<never, WorkerError>();
         // The bound itself: a handler runs only while holding one permit, so
         // even a miscounted claim can never exceed `concurrency` executions.
         const permits = yield* Effect.makeSemaphore(concurrency);
@@ -554,7 +550,13 @@ export const makeScheduler = (
               `.pipe(Effect.asVoid),
             { discard: true },
           );
-        }).pipe(Effect.catchAllCause(() => Effect.void));
+        }).pipe(
+          Effect.catchTag("SqlError", () =>
+            Effect.logError(
+              "jobs worker could not release abandoned leases; rows remain reclaimable after lease expiry",
+            ),
+          ),
+        );
 
         yield* shutdown.onShutdown(
           "jobs-worker",
@@ -576,7 +578,7 @@ export const makeScheduler = (
           ),
         );
 
-        const loop: Effect.Effect<void> = Effect.whileLoop({
+        const loop: Effect.Effect<void, JobQueueError> = Effect.whileLoop({
           while: () => !stopRequested,
           body: () =>
             Effect.gen(function* () {
@@ -591,13 +593,17 @@ export const makeScheduler = (
                 yield* Deferred.await(waiter).pipe(Effect.timeout(pollMillis), Effect.ignore);
                 return;
               }
-              const rows = yield* Effect.orDie(claim(capacity, leaseMillis));
+              const rows = yield* claim(capacity, leaseMillis);
               if (rows.length === 0) {
                 yield* Effect.sleep(pollMillis);
                 return;
               }
               for (const row of rows) {
-                const fiber = yield* Effect.fork(permits.withPermits(1)(execute(row, leaseMillis)));
+                const fiber = yield* Effect.fork(
+                  permits
+                    .withPermits(1)(execute(row, leaseMillis))
+                    .pipe(Effect.tapErrorCause((cause) => Deferred.failCause(workerFailed, cause))),
+                );
                 inflight.set(fiber, row);
                 void fiber.addObserver(() => {
                   inflight.delete(fiber);
@@ -610,9 +616,12 @@ export const makeScheduler = (
             }),
           step: () => undefined,
         });
-        yield* loop;
-        // Graceful drain: wait for in-flight handlers to finish.
-        yield* awaitInflight;
+        // Observe child failures as well as claims. Otherwise a broken
+        // completion/heartbeat or handler defect silently frees a slot.
+        yield* loop.pipe(
+          Effect.zipRight(awaitInflight),
+          Effect.raceFirst(Deferred.await(workerFailed)),
+        );
         yield* Effect.logInfo("jobs worker drained").pipe(
           Effect.annotateLogs({ drained: inflight.size }),
         );

package/src/schema.ts

@@ -1,4 +1,5 @@
 import * as SqlClient from "@effect/sql/SqlClient";
+import type { SqlError } from "@effect/sql/SqlError";
 import { Effect } from "effect";
 
 export interface AdapterOptions {
@@ -28,7 +29,7 @@ export const tableNames = (options: AdapterOptions = {}): TableNames => {
  */
 export const migrate = (
   options: AdapterOptions = {},
-): Effect.Effect<void, never, SqlClient.SqlClient> =>
+): Effect.Effect<void, SqlError, SqlClient.SqlClient> =>
   Effect.gen(function* () {
     const sql = yield* SqlClient.SqlClient;
     const tables = tableNames(options);
@@ -75,4 +76,4 @@ export const migrate = (
         dead_at TIMESTAMPTZ NOT NULL
       )
     `;
-  }).pipe(Effect.orDie);
+  });