cairnq 0.4.0 → 0.6.0

package/dist/context.d.ts

@@ -1,8 +1,20 @@
+import { type FailReason } from "./errors.js";
 import { type Task } from "./models.js";
 import type { SubmitOptions } from "./client.js";
 import type { TaskStore } from "./store/base.js";
 import { type TaskDef } from "./task.js";
-/** Handed to a task handler. Worker-side capabilities mirror the Python SDK. */
+export interface TaskContextOptions {
+    retryBackoffMs?: number;
+    retryBackoffMaxMs?: number;
+}
+/**
+ * Handed to a task handler. Worker-side capabilities mirror the Python SDK.
+ *
+ * One of these per task, whether a handler is delivered one task or a batch: a
+ * batch handler receives a `TaskContext[]`, so a single-task handler's `ctx` is
+ * literally the batch-of-one element. Lease, cancellation and settlement are per
+ * task, which is why they live here rather than on anything batch-shaped.
+ */
 export declare class TaskContext {
     private readonly store;
     private readonly task;
@@ -11,7 +23,10 @@ export declare class TaskContext {
     private readonly abort;
     private leaseLost;
     private cancelSeen;
-    constructor(store: TaskStore, task: Task, workerId: string, leaseMs: number);
+    private isSettled;
+    private readonly backoffMs;
+    private readonly backoffMaxMs;
+    constructor(store: TaskStore, task: Task, workerId: string, leaseMs: number, opts?: TaskContextOptions);
     get taskId(): string;
     get name(): string;
     get queue(): string;
@@ -20,6 +35,14 @@ export declare class TaskContext {
     get rootId(): string | null;
     get correlationId(): string | null;
     get payload(): any;
+    /**
+     * True once this task reached a terminal state — whether the handler settled
+     * it with succeed()/fail() or the worker settled it on the handler's behalf.
+     * The heartbeat and the settlement paths both read it.
+     */
+    get settled(): boolean;
+    /** @internal Called by the worker when it finalizes this task itself. */
+    markSettled(): void;
     /**
      * True once this worker has lost the task's lease — it expired and another
      * worker reclaimed it. Nothing this handler writes will be recorded any more
@@ -32,11 +55,34 @@ export declare class TaskContext {
     /** @internal Called by the worker when an owned write reports a lost lease. */
     markLeaseLost(): void;
     private observe;
+    /**
+     * @internal The same observation from just the flag, for a caller that read it
+     * without materializing a Task — the shared heartbeat, whose statement returns
+     * only the id and the cancel column precisely so it does not have to drag
+     * every payload back on every beat.
+     */
+    observeCancel(cancelRequested: boolean): void;
     private owned;
     progress(value: number | null, message?: string | null): Promise<Task>;
     heartbeat(): Promise<Task>;
     /** Cooperative cancel check. Free once a heartbeat has already seen the flag. */
     canceled(): Promise<boolean>;
+    /**
+     * Finalize this task as succeeded, now, without waiting for the handler to
+     * return. `complete` semantics: a cancel requested while it ran wins and the
+     * task finalizes as canceled instead, its result discarded. Returns null if
+     * this task was already settled.
+     */
+    succeed(result?: unknown): Promise<Task | null>;
+    /**
+     * Finalize this task as failed, now. `error` may be a string reason, an Error,
+     * a TaskError (which carries its own retryability), or a ready envelope.
+     * Retryable failures get the worker's backoff and are re-queued while attempts
+     * remain, exactly as a thrown error would be. Returns null if already settled.
+     */
+    fail(error?: FailReason, opts?: {
+        retryable?: boolean;
+    }): Promise<Task | null>;
     /** Submit a child task; parent/root/correlation are wired automatically. */
     submit(name: string, payload?: unknown, opts?: SubmitOptions): Promise<Task>;
     submit<P, R>(task: TaskDef<P, R>, payload?: P, opts?: SubmitOptions): Promise<Task>;

package/dist/context.js

@@ -1,8 +1,16 @@
-import { LostLease } from "./errors.js";
+import { DEFAULT_RETRY_BACKOFF_MAX_MS, DEFAULT_RETRY_BACKOFF_MS, failDelayMs } from "./backoff.js";
+import { asEnvelope, LostLease } from "./errors.js";
 import { cancelRequested } from "./models.js";
 import { taskName } from "./task.js";
 import { pollWait } from "./wait.js";
-/** Handed to a task handler. Worker-side capabilities mirror the Python SDK. */
+/**
+ * Handed to a task handler. Worker-side capabilities mirror the Python SDK.
+ *
+ * One of these per task, whether a handler is delivered one task or a batch: a
+ * batch handler receives a `TaskContext[]`, so a single-task handler's `ctx` is
+ * literally the batch-of-one element. Lease, cancellation and settlement are per
+ * task, which is why they live here rather than on anything batch-shaped.
+ */
 export class TaskContext {
     store;
     task;
@@ -13,11 +21,20 @@ export class TaskContext {
     // Cancellation is monotonic: once the DB has told us a cancel was requested it
     // can't be taken back, so canceled() can answer from this without a re-read.
     cancelSeen = false;
-    constructor(store, task, workerId, leaseMs) {
+    // Set once this task reached a terminal state through succeed()/fail(). The
+    // worker reads it to know which tasks a batch handler already decided, so it
+    // neither settles them twice nor keeps renewing their leases — the bookkeeping
+    // every ack/nack-style handler otherwise has to carry itself.
+    isSettled = false;
+    backoffMs;
+    backoffMaxMs;
+    constructor(store, task, workerId, leaseMs, opts = {}) {
         this.store = store;
         this.task = task;
         this.workerId = workerId;
         this.leaseMs = leaseMs;
+        this.backoffMs = opts.retryBackoffMs ?? DEFAULT_RETRY_BACKOFF_MS;
+        this.backoffMaxMs = opts.retryBackoffMaxMs ?? DEFAULT_RETRY_BACKOFF_MAX_MS;
     }
     get taskId() {
         return this.task.id;
@@ -43,6 +60,18 @@ export class TaskContext {
     get payload() {
         return this.task.payload;
     }
+    /**
+     * True once this task reached a terminal state — whether the handler settled
+     * it with succeed()/fail() or the worker settled it on the handler's behalf.
+     * The heartbeat and the settlement paths both read it.
+     */
+    get settled() {
+        return this.isSettled;
+    }
+    /** @internal Called by the worker when it finalizes this task itself. */
+    markSettled() {
+        this.isSettled = true;
+    }
     /**
      * True once this worker has lost the task's lease — it expired and another
      * worker reclaimed it. Nothing this handler writes will be recorded any more
@@ -66,18 +95,37 @@ export class TaskContext {
     // Every owned write returns the current row, so cancellation and lease loss
     // ride along on writes the handler was making anyway.
     observe(task) {
-        if (cancelRequested(task))
-            this.cancelSeen = true;
+        this.observeCancel(cancelRequested(task));
         return task;
     }
+    /**
+     * @internal The same observation from just the flag, for a caller that read it
+     * without materializing a Task — the shared heartbeat, whose statement returns
+     * only the id and the cancel column precisely so it does not have to drag
+     * every payload back on every beat.
+     */
+    observeCancel(cancelRequested) {
+        if (cancelRequested)
+            this.cancelSeen = true;
+    }
     async owned(write) {
-        // Short-circuit once the lease is known lost: nothing this context writes
-        // may be recorded any more. Locally, not just via the store's ownership
-        // check — after an abandoned (timed-out) attempt the same worker may
-        // re-claim this task under the same workerId, and a zombie handler's write
-        // would then pass ownership against the NEW attempt.
+        // One gate for every write through this context, so "may I still write?" is
+        // answered in one place rather than at each call site.
+        //
+        // Lease lost: nothing this context writes may be recorded any more. Checked
+        // locally, not just via the store's ownership check — after an abandoned
+        // (timed-out) attempt the same worker may re-claim this task under the same
+        // workerId, and a zombie handler's write would then pass ownership against
+        // the NEW attempt.
         if (this.leaseLost)
             throw new LostLease(this.task.id);
+        // Settled: the task is terminal, so the statement would match no row and come
+        // back as a lost lease — telling the handler "another worker took this" when
+        // the truth is "you already finished it", and flipping lostLease on the way.
+        // Refuse here instead, without the round trip and without corrupting the
+        // lease state.
+        if (this.isSettled)
+            throw new LostLease(this.task.id);
         try {
             return this.observe(await write());
         }
@@ -113,6 +161,49 @@ export class TaskContext {
             this.cancelSeen = true;
         return this.cancelSeen || t.status === "canceled";
     }
+    // ------------------------------------------------------------- settlement
+    // Finalizing a task is normally the worker's job, decided by whether the
+    // handler returned or threw. These two let a handler decide one task itself,
+    // which is what a batch needs: four of 256 tasks failing for four different
+    // reasons is the ordinary case, not the edge one, and it cannot be expressed
+    // by a single return value or a single throw.
+    //
+    // Settling twice is a no-op rather than an error. Handlers built on ack/nack
+    // queues all end up carrying a `finalizedIds` set to guarantee exactly that;
+    // holding it here instead is the point.
+    /**
+     * Finalize this task as succeeded, now, without waiting for the handler to
+     * return. `complete` semantics: a cancel requested while it ran wins and the
+     * task finalizes as canceled instead, its result discarded. Returns null if
+     * this task was already settled.
+     */
+    async succeed(result = null) {
+        if (this.isSettled)
+            return null;
+        const task = await this.owned(() => this.store.complete({ taskId: this.task.id, workerId: this.workerId, result }));
+        this.markSettled();
+        return task;
+    }
+    /**
+     * Finalize this task as failed, now. `error` may be a string reason, an Error,
+     * a TaskError (which carries its own retryability), or a ready envelope.
+     * Retryable failures get the worker's backoff and are re-queued while attempts
+     * remain, exactly as a thrown error would be. Returns null if already settled.
+     */
+    async fail(error = "task failed", opts = {}) {
+        if (this.isSettled)
+            return null;
+        const [envelope, retryable] = asEnvelope(error, opts.retryable ?? true);
+        const task = await this.owned(() => this.store.fail({
+            taskId: this.task.id,
+            workerId: this.workerId,
+            error: envelope,
+            retryable,
+            delayMs: failDelayMs(this.task.attempt, retryable, this.backoffMs, this.backoffMaxMs),
+        }));
+        this.markSettled();
+        return task;
+    }
     async submit(task, payload, opts = {}) {
         return this.store.submit({
             name: taskName(task),

package/dist/errors.d.ts

@@ -9,6 +9,15 @@ export declare function errorEnvelope(e: {
     retryable: boolean;
     details?: Record<string, unknown>;
 }): Record<string, unknown>;
+/**
+ * How an arbitrary thrown value becomes an envelope. Split out from `asEnvelope`
+ * below because the worker also reaches it directly, for a thrown plain object —
+ * which `asEnvelope` reads as a ready envelope, the right call for `ctx.fail` and
+ * the wrong one for something that was thrown. Both must agree on `code` and on
+ * deriving `type` from the error's name, or the same error reads differently
+ * depending on which way it was recorded.
+ */
+export declare function exceptionEnvelope(err: unknown, retryable?: boolean): Record<string, unknown>;
 export declare class CairnQError extends Error {
     constructor(message?: string);
 }
@@ -16,6 +25,16 @@ export declare class AlreadyExists extends CairnQError {
     key: string;
     constructor(key: string);
 }
+/** A gated submit waited out `maxWaitMs` without the queue draining below its
+ * depth limit. Nothing was enqueued. Distinct from a slow submit on purpose: a
+ * queue this far behind is a capacity problem, and a caller that silently
+ * retries forever converts it into an invisible one. */
+export declare class QueueFull extends CairnQError {
+    queue: string;
+    maxDepth: number;
+    waitedMs: number;
+    constructor(queue: string, maxDepth: number, waitedMs: number);
+}
 /** wait/call did not reach a terminal status in time. The task keeps running.
  * `task` is the last snapshot wait() observed (null if get() found nothing), and
  * the message says what state it was stuck in — a queued-never-claimed task is
@@ -74,3 +93,21 @@ export declare class TaskError extends CairnQError {
     });
     envelope(): Record<string, unknown>;
 }
+/** What a handler may pass to `ctx.fail`. */
+export type FailReason = string | Error | TaskError | Record<string, unknown>;
+/**
+ * Normalize anything that can end a task into [envelope, retryable].
+ *
+ * Shared by both ways a failure is recorded — a handler passing a reason to
+ * `ctx.fail`, and the worker classifying an error that ended an attempt — so the
+ * two cannot disagree about what a given error means. It lives here, beside the
+ * envelope constructors it dispatches to, rather than in the module that happens
+ * to expose it to handlers.
+ *
+ * A handler failing one task of a batch has a reason, not an exception object:
+ * `item.fail("no source records", { retryable: false })` is the shape the real
+ * code wants. A TaskError carries its own retryability and wins over the option;
+ * everything else takes the caller's. A ready envelope passes through, which is
+ * how the worker hands in the ones it composes itself.
+ */
+export declare function asEnvelope(error: FailReason, retryable: boolean): [Record<string, unknown>, boolean];

package/dist/errors.js

@@ -12,6 +12,23 @@ export function errorEnvelope(e) {
         details: e.details ?? {},
     };
 }
+/**
+ * How an arbitrary thrown value becomes an envelope. Split out from `asEnvelope`
+ * below because the worker also reaches it directly, for a thrown plain object —
+ * which `asEnvelope` reads as a ready envelope, the right call for `ctx.fail` and
+ * the wrong one for something that was thrown. Both must agree on `code` and on
+ * deriving `type` from the error's name, or the same error reads differently
+ * depending on which way it was recorded.
+ */
+export function exceptionEnvelope(err, retryable = true) {
+    const e = err;
+    return errorEnvelope({
+        type: e?.name ?? "Error",
+        code: "handler_error",
+        message: String(e?.message ?? err),
+        retryable,
+    });
+}
 export class CairnQError extends Error {
     constructor(message) {
         super(message);
@@ -29,6 +46,23 @@ export class AlreadyExists extends CairnQError {
         this.name = "AlreadyExists";
     }
 }
+/** A gated submit waited out `maxWaitMs` without the queue draining below its
+ * depth limit. Nothing was enqueued. Distinct from a slow submit on purpose: a
+ * queue this far behind is a capacity problem, and a caller that silently
+ * retries forever converts it into an invisible one. */
+export class QueueFull extends CairnQError {
+    queue;
+    maxDepth;
+    waitedMs;
+    constructor(queue, maxDepth, waitedMs) {
+        super(`queue ${queue} still holds ${maxDepth} or more queued tasks after ` +
+            `${waitedMs}ms; refusing to enqueue more`);
+        this.queue = queue;
+        this.maxDepth = maxDepth;
+        this.waitedMs = waitedMs;
+        this.name = "QueueFull";
+    }
+}
 /** One line of "why hasn't this finished" from the last snapshot wait()
  * observed. No worker running, no handler for the name, wrong queue, and two
  * processes on different database files all look identical from the API side —
@@ -144,3 +178,29 @@ export class TaskError extends CairnQError {
         });
     }
 }
+/**
+ * Normalize anything that can end a task into [envelope, retryable].
+ *
+ * Shared by both ways a failure is recorded — a handler passing a reason to
+ * `ctx.fail`, and the worker classifying an error that ended an attempt — so the
+ * two cannot disagree about what a given error means. It lives here, beside the
+ * envelope constructors it dispatches to, rather than in the module that happens
+ * to expose it to handlers.
+ *
+ * A handler failing one task of a batch has a reason, not an exception object:
+ * `item.fail("no source records", { retryable: false })` is the shape the real
+ * code wants. A TaskError carries its own retryability and wins over the option;
+ * everything else takes the caller's. A ready envelope passes through, which is
+ * how the worker hands in the ones it composes itself.
+ */
+export function asEnvelope(error, retryable) {
+    if (error instanceof TaskError)
+        return [error.envelope(), error.retryable];
+    if (error instanceof Error)
+        return [exceptionEnvelope(error, retryable), retryable];
+    if (typeof error === "object" && error !== null)
+        return [error, retryable];
+    // A bare reason is a TaskError in everything but the throwing, so let
+    // TaskError own its own type/code defaults rather than restating them.
+    return [new TaskError(String(error), { retryable }).envelope(), retryable];
+}

package/dist/index.d.ts

@@ -1,8 +1,11 @@
 export { CairnQ } from "./client.js";
-export type { CallOptions, SubmitOptions } from "./client.js";
+export type { CallOptions, ClientOptions, SubmitOptions } from "./client.js";
+export { QueueDepthGate } from "./backpressure.js";
+export type { BackpressureOptions, QueueDepthLimit } from "./backpressure.js";
 export { Worker } from "./worker.js";
-export type { Handler, TypedHandler, WorkerOptions } from "./worker.js";
+export type { BatchHandler, Handler, TypedHandler, WorkerOptions } from "./worker.js";
 export { TaskContext } from "./context.js";
+export type { TaskContextOptions } from "./context.js";
 export { defineTask } from "./task.js";
 export type { TaskDef } from "./task.js";
 export { SQLiteStore } from "./store/sqlite.js";
@@ -11,4 +14,5 @@ export { TaskStore } from "./store/base.js";
 export type { ListInput, PurgeInput, SubmitInput, Conflict } from "./store/base.js";
 export type { Task, TaskStatus } from "./models.js";
 export { STATUSES, isTerminal, cancelRequested, isQueued, isRunning, isSucceeded, isFailed, isCanceled, } from "./models.js";
-export { CairnQError, AlreadyExists, TaskTimeout, TaskFailed, TaskCanceled, TaskError, LostLease, ProtocolVersionMismatch, SerializationError, } from "./errors.js";
+export { CairnQError, AlreadyExists, QueueFull, TaskTimeout, TaskFailed, TaskCanceled, TaskError, LostLease, ProtocolVersionMismatch, SerializationError, } from "./errors.js";
+export type { FailReason } from "./errors.js";

package/dist/index.js

@@ -1,4 +1,5 @@
 export { CairnQ } from "./client.js";
+export { QueueDepthGate } from "./backpressure.js";
 export { Worker } from "./worker.js";
 export { TaskContext } from "./context.js";
 export { defineTask } from "./task.js";
@@ -6,4 +7,4 @@ export { SQLiteStore } from "./store/sqlite.js";
 export { PostgresStore } from "./store/postgres.js";
 export { TaskStore } from "./store/base.js";
 export { STATUSES, isTerminal, cancelRequested, isQueued, isRunning, isSucceeded, isFailed, isCanceled, } from "./models.js";
-export { CairnQError, AlreadyExists, TaskTimeout, TaskFailed, TaskCanceled, TaskError, LostLease, ProtocolVersionMismatch, SerializationError, } from "./errors.js";
+export { CairnQError, AlreadyExists, QueueFull, TaskTimeout, TaskFailed, TaskCanceled, TaskError, LostLease, ProtocolVersionMismatch, SerializationError, } from "./errors.js";

package/dist/store/base.d.ts

@@ -1,4 +1,5 @@
 import { type Task, type TaskStatus } from "../models.js";
+import { type BackpressureOptions } from "../backpressure.js";
 /** Encode a value for a protocol JSON column, raising SerializationError on
  * anything JSON cannot represent. Refuses what JSON.stringify would silently
  * mangle into `null`: NaN/Infinity anywhere, undefined/function/symbol inside an
@@ -12,6 +13,9 @@ export declare function dumpJson(value: unknown): string;
 export declare function checkProtocolVersion(version: number): void;
 declare const CONFLICTS: readonly ["reuse", "reject", "replace"];
 export type Conflict = (typeof CONFLICTS)[number];
+/** The queue a submit lands on when it names none. Owned here, where the
+ * default is applied, so nothing above has to re-derive it. */
+export declare const DEFAULT_QUEUE = "default";
 export interface SubmitInput {
     name: string;
     payload: unknown;
@@ -69,6 +73,8 @@ export declare function statementParams(sql: string): readonly string[];
  * behavior; the shared SQL already stops them from drifting in wording.
  */
 export declare abstract class TaskStore {
+    /** Set by useBackpressure; null means submit is ungated. */
+    private gate;
     abstract connect(): Promise<void>;
     abstract close(): Promise<void>;
     abstract protocolVersion(): Promise<number>;
@@ -103,6 +109,15 @@ export declare abstract class TaskStore {
      */
     private ownedWrite;
     private static one;
+    /**
+     * Bound how deep a queue may get before `submit` blocks. Off unless set.
+     *
+     * It hangs here rather than on `CairnQ` because the store is the one choke
+     * point every submit passes through — a handler spawning children via
+     * `TaskContext.submit` is the shape most likely to outrun its workers, and
+     * gating only the client would leave exactly that path unbounded.
+     */
+    useBackpressure(opts: BackpressureOptions): void;
     submit(input: SubmitInput): Promise<Task>;
     get(taskId: string): Promise<Task | null>;
     getByKey(key: string): Promise<Task | null>;
@@ -136,6 +151,15 @@ export declare abstract class TaskStore {
      * them.
      */
     stats(): Promise<Record<string, Record<TaskStatus, number>>>;
+    /**
+     * How many more tasks fit on `queue` under `maxDepth` — 0 once it is full.
+     *
+     * The cheap half of backpressure: bounded at `maxDepth` index entries, unlike
+     * `stats()`, which aggregates the whole table (terminal rows included) and so
+     * costs more the longer a database has been running. Use it directly to shed
+     * load or shape a producer; `QueueDepthGate` builds the blocking form on top.
+     */
+    queueDepth(queue: string, maxDepth: number): Promise<number>;
     /**
      * Take up to `limit` claimable tasks. `names` restricts the claim to task names
      * this caller can actually run — a worker passes its registered handlers.
@@ -150,11 +174,60 @@ export declare abstract class TaskStore {
         limit?: number;
         names?: string[];
     }): Promise<Task[]>;
+    /**
+     * Open one claim transaction and let the caller draw from it repeatedly.
+     *
+     * The transaction is what has to live here: the read-only probe that keeps an
+     * idle worker off SQLite's single write lock, the `recover_leases` whose
+     * reclaimed leases must be visible to the claims that follow and to nobody in
+     * between, and the write lock itself. *What* gets claimed under it is the
+     * caller's business — a worker drawing a separate quota per task name is
+     * scheduling policy, and this layer has no vocabulary for the "handler call"
+     * that policy is denominated in. It knows queues, names, limits and rows.
+     *
+     * `plan` is handed a `claim(names, limit)` it may call any number of times,
+     * each a separate statement under the same lock and the same recovery, and
+     * each free to size itself from what the previous one returned. That feedback
+     * is the reason this is a callback rather than a list of quotas: a caller
+     * dividing a budget up front has to guess, and every share handed to a name
+     * with nothing queued is a slot left idle until the next poll.
+     *
+     * `plan` runs with the write lock held, so it must await nothing but that
+     * callback.
+     *
+     * `names` is the union `plan` might ask for — the probe and the recovery are
+     * filtered by it. Returns undefined when the probe finds nothing claimable, in
+     * which case `plan` never runs and no transaction is opened.
+     */
+    claimSession<T>(input: {
+        queues: string[];
+        workerId: string;
+        leaseMs?: number;
+        names: string[] | null;
+    }, plan: (claim: (names: string[] | null, limit: number) => Promise<Task[]>) => Promise<T>): Promise<T | undefined>;
     heartbeat(input: {
         taskId: string;
         workerId: string;
         leaseMs?: number;
     }): Promise<Task>;
+    /**
+     * Renew several leases in one statement. Returns `taskId -> cancel requested`
+     * for the tasks this worker still holds.
+     *
+     * Deliberately not an ownedWrite: ownership is per task here, so there is no
+     * single answer to "did it work". A task **absent** from the result lost its
+     * lease, and the caller decides what that means for that one task rather than
+     * failing the whole beat.
+     *
+     * It returns flags rather than Tasks because nothing downstream needs a task:
+     * the caller renews leases and observes cancellation, and whole rows would drag
+     * every payload back on every beat for the life of the call.
+     */
+    heartbeatBatch(input: {
+        taskIds: string[];
+        workerId: string;
+        leaseMs?: number;
+    }): Promise<Map<string, boolean>>;
     progress(input: {
         taskId: string;
         workerId: string;