cairnq 0.5.0 → 0.6.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cairnq",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "description": "SQLite-first, cross-language, storage-centered durable task runtime",
   "license": "MIT",
   "author": "Jannchie <jannchie@gmail.com>",

package/src/backoff.ts

@@ -0,0 +1,53 @@
+/**
+ * Retry backoff, in its own module because two callers need it.
+ *
+ * The worker computes it when a handler's failure ends an attempt; TaskContext
+ * computes it when a handler fails one task of a batch itself. Keeping it in
+ * worker.ts would make context.ts import the module that imports it.
+ */
+
+export const DEFAULT_RETRY_BACKOFF_MS = 1_000;
+export const DEFAULT_RETRY_BACKOFF_MAX_MS = 30_000;
+
+/**
+ * Exponential backoff with equal jitter: the window doubles per attempt up to
+ * `maxMs`, and the delay lands uniformly in its upper half, `[w/2, w)`.
+ *
+ * The jitter is what keeps a fleet from retrying in lockstep. Failures align
+ * when the downstream fails fast enough that a whole concurrency batch raises
+ * at once (connection refused, DNS gone), and capped exponential backoff then
+ * *preserves* that alignment — once every task sits at `maxMs`, they all retry
+ * on the same beat forever. Spreading over half the window breaks it; keeping
+ * the lower half as a floor means jitter never shortens the wait to less than
+ * half of what plain exponential backoff would have asked for.
+ *
+ * `rand` is injected so tests can pin an exact delay.
+ */
+export function retryDelayMs(
+  attempt: number,
+  baseMs: number,
+  maxMs: number,
+  rand: () => number = Math.random,
+): number {
+  if (baseMs <= 0) return 0;
+  const exponent = Math.max(0, attempt - 1);
+  const window = Math.min(maxMs, baseMs * 2 ** exponent);
+  const floor = Math.floor(window / 2);
+  return floor + Math.floor(rand() * (window - floor));
+}
+
+/**
+ * The delay a `fail` write should carry. Not just the backoff: a permanent
+ * failure is never re-run, so it always delays 0. Both settlement paths — the
+ * worker's and a handler's `ctx.fail` — go through this, so they cannot end up
+ * backing off differently.
+ */
+export function failDelayMs(
+  attempt: number,
+  retryable: boolean,
+  baseMs: number,
+  maxMs: number,
+  rand: () => number = Math.random,
+): number {
+  return retryable ? retryDelayMs(attempt, baseMs, maxMs, rand) : 0;
+}

package/src/context.ts

@@ -1,24 +1,48 @@
-import { LostLease } from "./errors.js";
+import { DEFAULT_RETRY_BACKOFF_MAX_MS, DEFAULT_RETRY_BACKOFF_MS, failDelayMs } from "./backoff.js";
+import { asEnvelope, type FailReason, LostLease } from "./errors.js";
 import { cancelRequested, type Task } from "./models.js";
 import type { SubmitOptions } from "./client.js";
 import type { TaskStore } from "./store/base.js";
 import { type TaskDef, taskName } from "./task.js";
 import { pollWait } from "./wait.js";
 
-/** Handed to a task handler. Worker-side capabilities mirror the Python SDK. */
+export interface TaskContextOptions {
+  retryBackoffMs?: number;
+  retryBackoffMaxMs?: number;
+}
+
+/**
+ * Handed to a task handler. Worker-side capabilities mirror the Python SDK.
+ *
+ * One of these per task, whether a handler is delivered one task or a batch: a
+ * batch handler receives a `TaskContext[]`, so a single-task handler's `ctx` is
+ * literally the batch-of-one element. Lease, cancellation and settlement are per
+ * task, which is why they live here rather than on anything batch-shaped.
+ */
 export class TaskContext {
   private readonly abort = new AbortController();
   private leaseLost = false;
   // Cancellation is monotonic: once the DB has told us a cancel was requested it
   // can't be taken back, so canceled() can answer from this without a re-read.
   private cancelSeen = false;
+  // Set once this task reached a terminal state through succeed()/fail(). The
+  // worker reads it to know which tasks a batch handler already decided, so it
+  // neither settles them twice nor keeps renewing their leases — the bookkeeping
+  // every ack/nack-style handler otherwise has to carry itself.
+  private isSettled = false;
+  private readonly backoffMs: number;
+  private readonly backoffMaxMs: number;
 
   constructor(
     private readonly store: TaskStore,
     private readonly task: Task,
     public readonly workerId: string,
     private readonly leaseMs: number,
-  ) {}
+    opts: TaskContextOptions = {},
+  ) {
+    this.backoffMs = opts.retryBackoffMs ?? DEFAULT_RETRY_BACKOFF_MS;
+    this.backoffMaxMs = opts.retryBackoffMaxMs ?? DEFAULT_RETRY_BACKOFF_MAX_MS;
+  }
 
   get taskId(): string {
     return this.task.id;
@@ -44,6 +68,19 @@ export class TaskContext {
   get payload(): any {
     return this.task.payload;
   }
+  /**
+   * True once this task reached a terminal state — whether the handler settled
+   * it with succeed()/fail() or the worker settled it on the handler's behalf.
+   * The heartbeat and the settlement paths both read it.
+   */
+  get settled(): boolean {
+    return this.isSettled;
+  }
+
+  /** @internal Called by the worker when it finalizes this task itself. */
+  markSettled(): void {
+    this.isSettled = true;
+  }
 
   /**
    * True once this worker has lost the task's lease — it expired and another
@@ -70,17 +107,36 @@ export class TaskContext {
   // Every owned write returns the current row, so cancellation and lease loss
   // ride along on writes the handler was making anyway.
   private observe(task: Task): Task {
-    if (cancelRequested(task)) this.cancelSeen = true;
+    this.observeCancel(cancelRequested(task));
     return task;
   }
 
+  /**
+   * @internal The same observation from just the flag, for a caller that read it
+   * without materializing a Task — the shared heartbeat, whose statement returns
+   * only the id and the cancel column precisely so it does not have to drag
+   * every payload back on every beat.
+   */
+  observeCancel(cancelRequested: boolean): void {
+    if (cancelRequested) this.cancelSeen = true;
+  }
+
   private async owned(write: () => Promise<Task>): Promise<Task> {
-    // Short-circuit once the lease is known lost: nothing this context writes
-    // may be recorded any more. Locally, not just via the store's ownership
-    // check — after an abandoned (timed-out) attempt the same worker may
-    // re-claim this task under the same workerId, and a zombie handler's write
-    // would then pass ownership against the NEW attempt.
+    // One gate for every write through this context, so "may I still write?" is
+    // answered in one place rather than at each call site.
+    //
+    // Lease lost: nothing this context writes may be recorded any more. Checked
+    // locally, not just via the store's ownership check — after an abandoned
+    // (timed-out) attempt the same worker may re-claim this task under the same
+    // workerId, and a zombie handler's write would then pass ownership against
+    // the NEW attempt.
     if (this.leaseLost) throw new LostLease(this.task.id);
+    // Settled: the task is terminal, so the statement would match no row and come
+    // back as a lost lease — telling the handler "another worker took this" when
+    // the truth is "you already finished it", and flipping lostLease on the way.
+    // Refuse here instead, without the round trip and without corrupting the
+    // lease state.
+    if (this.isSettled) throw new LostLease(this.task.id);
     try {
       return this.observe(await write());
     } catch (err) {
@@ -119,6 +175,57 @@ export class TaskContext {
     return this.cancelSeen || t.status === "canceled";
   }
 
+  // ------------------------------------------------------------- settlement
+  // Finalizing a task is normally the worker's job, decided by whether the
+  // handler returned or threw. These two let a handler decide one task itself,
+  // which is what a batch needs: four of 256 tasks failing for four different
+  // reasons is the ordinary case, not the edge one, and it cannot be expressed
+  // by a single return value or a single throw.
+  //
+  // Settling twice is a no-op rather than an error. Handlers built on ack/nack
+  // queues all end up carrying a `finalizedIds` set to guarantee exactly that;
+  // holding it here instead is the point.
+
+  /**
+   * Finalize this task as succeeded, now, without waiting for the handler to
+   * return. `complete` semantics: a cancel requested while it ran wins and the
+   * task finalizes as canceled instead, its result discarded. Returns null if
+   * this task was already settled.
+   */
+  async succeed(result: unknown = null): Promise<Task | null> {
+    if (this.isSettled) return null;
+    const task = await this.owned(() =>
+      this.store.complete({ taskId: this.task.id, workerId: this.workerId, result }),
+    );
+    this.markSettled();
+    return task;
+  }
+
+  /**
+   * Finalize this task as failed, now. `error` may be a string reason, an Error,
+   * a TaskError (which carries its own retryability), or a ready envelope.
+   * Retryable failures get the worker's backoff and are re-queued while attempts
+   * remain, exactly as a thrown error would be. Returns null if already settled.
+   */
+  async fail(
+    error: FailReason = "task failed",
+    opts: { retryable?: boolean } = {},
+  ): Promise<Task | null> {
+    if (this.isSettled) return null;
+    const [envelope, retryable] = asEnvelope(error, opts.retryable ?? true);
+    const task = await this.owned(() =>
+      this.store.fail({
+        taskId: this.task.id,
+        workerId: this.workerId,
+        error: envelope,
+        retryable,
+        delayMs: failDelayMs(this.task.attempt, retryable, this.backoffMs, this.backoffMaxMs),
+      }),
+    );
+    this.markSettled();
+    return task;
+  }
+
   /** Submit a child task; parent/root/correlation are wired automatically. */
   submit(name: string, payload?: unknown, opts?: SubmitOptions): Promise<Task>;
   submit<P, R>(task: TaskDef<P, R>, payload?: P, opts?: SubmitOptions): Promise<Task>;

package/src/errors.ts

@@ -20,6 +20,24 @@ export function errorEnvelope(e: {
   };
 }
 
+/**
+ * How an arbitrary thrown value becomes an envelope. Split out from `asEnvelope`
+ * below because the worker also reaches it directly, for a thrown plain object —
+ * which `asEnvelope` reads as a ready envelope, the right call for `ctx.fail` and
+ * the wrong one for something that was thrown. Both must agree on `code` and on
+ * deriving `type` from the error's name, or the same error reads differently
+ * depending on which way it was recorded.
+ */
+export function exceptionEnvelope(err: unknown, retryable = true): Record<string, unknown> {
+  const e = err as { name?: string; message?: string };
+  return errorEnvelope({
+    type: e?.name ?? "Error",
+    code: "handler_error",
+    message: String(e?.message ?? err),
+    retryable,
+  });
+}
+
 export class CairnQError extends Error {
   constructor(message?: string) {
     super(message);
@@ -183,3 +201,33 @@ export class TaskError extends CairnQError {
     });
   }
 }
+
+/** What a handler may pass to `ctx.fail`. */
+export type FailReason = string | Error | TaskError | Record<string, unknown>;
+
+/**
+ * Normalize anything that can end a task into [envelope, retryable].
+ *
+ * Shared by both ways a failure is recorded — a handler passing a reason to
+ * `ctx.fail`, and the worker classifying an error that ended an attempt — so the
+ * two cannot disagree about what a given error means. It lives here, beside the
+ * envelope constructors it dispatches to, rather than in the module that happens
+ * to expose it to handlers.
+ *
+ * A handler failing one task of a batch has a reason, not an exception object:
+ * `item.fail("no source records", { retryable: false })` is the shape the real
+ * code wants. A TaskError carries its own retryability and wins over the option;
+ * everything else takes the caller's. A ready envelope passes through, which is
+ * how the worker hands in the ones it composes itself.
+ */
+export function asEnvelope(
+  error: FailReason,
+  retryable: boolean,
+): [Record<string, unknown>, boolean] {
+  if (error instanceof TaskError) return [error.envelope(), error.retryable];
+  if (error instanceof Error) return [exceptionEnvelope(error, retryable), retryable];
+  if (typeof error === "object" && error !== null) return [error, retryable];
+  // A bare reason is a TaskError in everything but the throwing, so let
+  // TaskError own its own type/code defaults rather than restating them.
+  return [new TaskError(String(error), { retryable }).envelope(), retryable];
+}

package/src/index.ts

@@ -3,8 +3,9 @@ export type { CallOptions, ClientOptions, SubmitOptions } from "./client.js";
 export { QueueDepthGate } from "./backpressure.js";
 export type { BackpressureOptions, QueueDepthLimit } from "./backpressure.js";
 export { Worker } from "./worker.js";
-export type { Handler, TypedHandler, WorkerOptions } from "./worker.js";
+export type { BatchHandler, Handler, TypedHandler, WorkerOptions } from "./worker.js";
 export { TaskContext } from "./context.js";
+export type { TaskContextOptions } from "./context.js";
 export { defineTask } from "./task.js";
 export type { TaskDef } from "./task.js";
 export { SQLiteStore } from "./store/sqlite.js";
@@ -34,3 +35,4 @@ export {
   ProtocolVersionMismatch,
   SerializationError,
 } from "./errors.js";
+export type { FailReason } from "./errors.js";

package/src/store/base.ts

@@ -425,26 +425,82 @@ export abstract class TaskStore {
     limit?: number;
     names?: string[];
   }): Promise<Task[]> {
-    // One queue is the common case and gets its own statement: a list-valued queue
-    // filter cannot be read in claim order, so the planner sorts every claimable
-    // row to take LIMIT of them, and claim's cost grows with the queued backlog
-    // while it holds the claim transaction. See claim_one_queue.sql.
+    const names = input.names ?? null;
+    const claimed = await this.claimSession(
+      { queues: input.queues, workerId: input.workerId, leaseMs: input.leaseMs, names },
+      (claim) => claim(names, input.limit ?? 1),
+    );
+    return claimed ?? [];
+  }
+
+  /**
+   * Open one claim transaction and let the caller draw from it repeatedly.
+   *
+   * The transaction is what has to live here: the read-only probe that keeps an
+   * idle worker off SQLite's single write lock, the `recover_leases` whose
+   * reclaimed leases must be visible to the claims that follow and to nobody in
+   * between, and the write lock itself. *What* gets claimed under it is the
+   * caller's business — a worker drawing a separate quota per task name is
+   * scheduling policy, and this layer has no vocabulary for the "handler call"
+   * that policy is denominated in. It knows queues, names, limits and rows.
+   *
+   * `plan` is handed a `claim(names, limit)` it may call any number of times,
+   * each a separate statement under the same lock and the same recovery, and
+   * each free to size itself from what the previous one returned. That feedback
+   * is the reason this is a callback rather than a list of quotas: a caller
+   * dividing a budget up front has to guess, and every share handed to a name
+   * with nothing queued is a slot left idle until the next poll.
+   *
+   * `plan` runs with the write lock held, so it must await nothing but that
+   * callback.
+   *
+   * `names` is the union `plan` might ask for — the probe and the recovery are
+   * filtered by it. Returns undefined when the probe finds nothing claimable, in
+   * which case `plan` never runs and no transaction is opened.
+   */
+  async claimSession<T>(
+    input: { queues: string[]; workerId: string; leaseMs?: number; names: string[] | null },
+    plan: (claim: (names: string[] | null, limit: number) => Promise<Task[]>) => Promise<T>,
+  ): Promise<T | undefined> {
+    // A list-valued filter cannot be read in claim order, so the planner sorts
+    // every claimable row to take LIMIT of them and the claim's cost grows with
+    // the backlog while it holds the transaction. Both filters therefore have an
+    // equality form, picked per draw: one queue is the common deployment, and one
+    // name is every per-name quota. See claim_one_queue.sql and claim_one_name.sql.
     const oneQueue = input.queues.length === 1;
-    const params: Params = {
+    const base: Params = {
       queues: input.queues,
       queue: oneQueue ? input.queues[0] : null,
-      names: input.names ?? null,
+      names: input.names,
+      name: null,
       worker_id: input.workerId,
       lease_ms: input.leaseMs ?? 30_000,
-      limit: input.limit ?? 1,
+      limit: 1,
       lease_expired_error: LEASE_EXPIRED_ERROR_JSON,
     };
-    if (!(await this.hasClaimableWork(params))) return [];
-    // Recovery must share the claim's transaction: a lease reclaimed here has to
-    // be visible to the claim that follows, and to nobody in between.
+    if (!(await this.hasClaimableWork(base))) return undefined;
     return this.tx(async (fetch) => {
-      await fetch("recover_leases", params);
-      return (await fetch(oneQueue ? "claim_one_queue" : "claim", params)).map(rowToTask);
+      await fetch("recover_leases", base);
+      return plan(async (names, limit) => {
+        // A draw asking for nothing, or filtered to no names, claims nothing —
+        // answer it here rather than spending a statement to learn that.
+        if (limit <= 0 || names?.length === 0) return [];
+        const oneName = names?.length === 1;
+        const statement = oneName
+          ? oneQueue
+            ? "claim_one_queue_one_name"
+            : "claim_one_name"
+          : oneQueue
+            ? "claim_one_queue"
+            : "claim";
+        const rows = await fetch(statement, {
+          ...base,
+          names,
+          name: oneName ? names![0] : null,
+          limit,
+        });
+        return rows.map(rowToTask);
+      });
     });
   }
 
@@ -456,6 +512,33 @@ export abstract class TaskStore {
     });
   }
 
+  /**
+   * Renew several leases in one statement. Returns `taskId -> cancel requested`
+   * for the tasks this worker still holds.
+   *
+   * Deliberately not an ownedWrite: ownership is per task here, so there is no
+   * single answer to "did it work". A task **absent** from the result lost its
+   * lease, and the caller decides what that means for that one task rather than
+   * failing the whole beat.
+   *
+   * It returns flags rather than Tasks because nothing downstream needs a task:
+   * the caller renews leases and observes cancellation, and whole rows would drag
+   * every payload back on every beat for the life of the call.
+   */
+  async heartbeatBatch(input: {
+    taskIds: string[];
+    workerId: string;
+    leaseMs?: number;
+  }): Promise<Map<string, boolean>> {
+    if (!input.taskIds.length) return new Map();
+    const rows = await this.fetch("heartbeat_batch", {
+      ids: input.taskIds,
+      worker_id: input.workerId,
+      lease_ms: input.leaseMs ?? 30_000,
+    });
+    return new Map(rows.map((r) => [r.id as string, r.cancel_requested_at_ms != null]));
+  }
+
   async progress(input: {
     taskId: string;
     workerId: string;

package/src/store/sqlite.ts

@@ -357,7 +357,10 @@ export class SQLiteStore extends TaskStore {
           bound[name] = now - (params.older_than_ms as number);
           break;
         case "queues":
-          bound[name] = JSON.stringify(params.queues);
+        case "ids":
+          // json_each needs a JSON array. Postgres binds the array itself as
+          // text[], so only this dialect encodes.
+          bound[name] = JSON.stringify(params[name]);
           break;
         case "names":
           // json_each needs a JSON array; null stays null so the SQL's