@chidchanun/bcp 0.2.9 → 0.2.11

package/packages/server/src/jobs.ts

@@ -23,6 +23,17 @@ export interface JobRecord<
     startedAt?: number;
     completedAt?: number;
     error?: string;
+    leaseOwner?: string;
+    leaseUntil?: number;
+    heartbeatAt?: number;
+    recoveredAt?: number;
+}
+
+export interface DeadLetterJobRecord<
+    TPayload = unknown
+> extends JobRecord<TPayload> {
+    state: "failed";
+    deadLetteredAt: number;
 }
 
 export interface EnqueueJobOptions {
@@ -31,18 +42,59 @@ export interface EnqueueJobOptions {
     maxAttempts?: number;
 }
 
+export interface ReserveJobOptions {
+    ownerId: string;
+    visibilityTimeoutMs: number;
+}
+
+export interface HeartbeatJobOptions {
+    ownerId: string;
+    heartbeatAt: number;
+    visibilityTimeoutMs: number;
+}
+
+export interface RecoverStaleJobsOptions {
+    limit?: number;
+}
+
+export interface RequeueDeadLetterOptions {
+    delayMs?: number;
+    resetAttempts?: boolean;
+}
+
+export interface CleanupJobsOptions {
+    before: number;
+    states?: Array<
+        "succeeded" |
+        "failed" |
+        "cancelled"
+    >;
+}
+
+export interface JobQueueStats {
+    total: number;
+    queued: number;
+    running: number;
+    succeeded: number;
+    failed: number;
+    cancelled: number;
+    deadLetters: number;
+}
+
 export interface JobQueueAdapter {
     enqueue<TPayload = unknown>(
         job: JobRecord<TPayload>
     ): Promise<void>;
 
     reserve(
-        now: number
+        now: number,
+        options?: ReserveJobOptions
     ): Promise<JobRecord | null>;
 
     complete(
         id: string,
-        completedAt: number
+        completedAt: number,
+        ownerId?: string
     ): Promise<void>;
 
     fail(
@@ -51,6 +103,7 @@ export interface JobQueueAdapter {
             error: string;
             failedAt: number;
             retryAt?: number;
+            ownerId?: string;
         }
     ): Promise<void>;
 
@@ -65,6 +118,30 @@ export interface JobQueueAdapter {
 
     list(): Promise<JobRecord[]>;
 
+    heartbeat?(
+        id: string,
+        options: HeartbeatJobOptions
+    ): Promise<boolean>;
+
+    recoverStale?(
+        now: number,
+        options?: RecoverStaleJobsOptions
+    ): Promise<number>;
+
+    listDeadLetters?(): Promise<DeadLetterJobRecord[]>;
+
+    requeueDeadLetter?(
+        id: string,
+        now: number,
+        options?: RequeueDeadLetterOptions
+    ): Promise<boolean>;
+
+    cleanup?(
+        options: CleanupJobsOptions
+    ): Promise<number>;
+
+    stats?(): Promise<JobQueueStats>;
+
     close?(): Promise<void>;
 }
 
@@ -99,13 +176,26 @@ export interface JobQueueOptions {
     idFactory?: () => string;
 }
 
-export interface StartJobWorkerOptions {
+export interface ProcessNextJobOptions {
+    ownerId?: string;
+    visibilityTimeoutMs?: number;
+    heartbeatIntervalMs?: number;
+    recoveryLimit?: number;
+}
+
+export interface StartJobWorkerOptions
+extends ProcessNextJobOptions {
     concurrency?: number;
     pollIntervalMs?: number;
+    workerId?: string;
+    onError?: (
+        error: unknown
+    ) => void | Promise<void>;
 }
 
 export interface JobWorker {
     readonly running: boolean;
+    readonly workerId?: string;
     stop(): Promise<void>;
 }
 
@@ -132,13 +222,31 @@ export interface BackgroundJobQueue {
     cancel(id: string): Promise<boolean>;
 
     processNext(
-        signal?: AbortSignal
+        signal?: AbortSignal,
+        options?: ProcessNextJobOptions
     ): Promise<boolean>;
 
     startWorker(
         options?: StartJobWorkerOptions
     ): JobWorker;
 
+    recoverStale(
+        options?: RecoverStaleJobsOptions
+    ): Promise<number>;
+
+    deadLetters(): Promise<DeadLetterJobRecord[]>;
+
+    requeueDeadLetter(
+        id: string,
+        options?: RequeueDeadLetterOptions
+    ): Promise<boolean>;
+
+    cleanup(
+        options: CleanupJobsOptions
+    ): Promise<number>;
+
+    stats(): Promise<JobQueueStats>;
+
     close(): Promise<void>;
 }
 
@@ -146,6 +254,54 @@ export function createMemoryJobQueueAdapter():
 MemoryJobQueueAdapter {
     const jobs =
         new Map<string, JobRecord>();
+    const deadLetters =
+        new Map<
+            string,
+            DeadLetterJobRecord
+        >();
+
+    const recoverStale = (
+        now: number,
+        options: RecoverStaleJobsOptions = {}
+    ): number => {
+        const limit =
+            normalizePositiveInteger(
+                options.limit ?? 100,
+                "recovery limit"
+            );
+        const stale =
+            Array.from(
+                jobs.values()
+            )
+                .filter(
+                    job =>
+                        job.state === "running" &&
+                        job.leaseUntil !== undefined &&
+                        job.leaseUntil <= now
+                )
+                .sort(
+                    (left, right) =>
+                        (left.leaseUntil ?? 0) -
+                            (right.leaseUntil ?? 0) ||
+                        left.id.localeCompare(
+                            right.id
+                        )
+                )
+                .slice(
+                    0,
+                    limit
+                );
+
+        for (const job of stale) {
+            job.state = "queued";
+            job.availableAt = now;
+            job.startedAt = undefined;
+            clearLease(job);
+            job.recoveredAt = now;
+        }
+
+        return stale.length;
+    };
 
     return {
         async enqueue(job) {
@@ -161,7 +317,17 @@ MemoryJobQueueAdapter {
             );
         },
 
-        async reserve(now) {
+        async reserve(
+            now,
+            options
+        ) {
+            recoverStale(
+                now,
+                {
+                    limit: 100,
+                }
+            );
+
             const candidate =
                 Array.from(
                     jobs.values()
@@ -182,39 +348,53 @@ MemoryJobQueueAdapter {
                 return null;
             }
 
-            candidate.state =
-                "running";
+            candidate.state = "running";
             candidate.attempts += 1;
-            candidate.startedAt =
-                now;
-            candidate.error =
-                undefined;
+            candidate.startedAt = now;
+            candidate.error = undefined;
+            candidate.recoveredAt = undefined;
+
+            if (options) {
+                candidate.leaseOwner =
+                    normalizeWorkerId(
+                        options.ownerId
+                    );
+                candidate.leaseUntil =
+                    now +
+                    normalizePositiveInteger(
+                        options.visibilityTimeoutMs,
+                        "visibilityTimeoutMs"
+                    );
+                candidate.heartbeatAt = now;
+            }
 
-            return cloneJob(
-                candidate
-            );
+            return cloneJob(candidate);
         },
 
         async complete(
             id,
-            completedAt
+            completedAt,
+            ownerId
         ) {
             const job =
                 jobs.get(id);
 
             if (
                 !job ||
-                job.state === "cancelled"
+                job.state === "cancelled" ||
+                !leaseOwnerMatches(
+                    job,
+                    ownerId
+                )
             ) {
                 return;
             }
 
-            job.state =
-                "succeeded";
-            job.completedAt =
-                completedAt;
-            job.error =
-                undefined;
+            job.state = "succeeded";
+            job.completedAt = completedAt;
+            job.error = undefined;
+            clearLease(job);
+            deadLetters.delete(id);
         },
 
         async fail(
@@ -226,31 +406,41 @@ MemoryJobQueueAdapter {
 
             if (
                 !job ||
-                job.state === "cancelled"
+                job.state === "cancelled" ||
+                !leaseOwnerMatches(
+                    job,
+                    options.ownerId
+                )
             ) {
                 return;
             }
 
-            job.error =
-                options.error;
+            job.error = options.error;
+            clearLease(job);
 
             if (
                 options.retryAt !== undefined &&
                 job.attempts < job.maxAttempts
             ) {
-                job.state =
-                    "queued";
+                job.state = "queued";
                 job.availableAt =
                     options.retryAt;
-                job.startedAt =
-                    undefined;
+                job.startedAt = undefined;
                 return;
             }
 
-            job.state =
-                "failed";
+            job.state = "failed";
             job.completedAt =
                 options.failedAt;
+            deadLetters.set(
+                id,
+                {
+                    ...cloneJob(job),
+                    state: "failed",
+                    deadLetteredAt:
+                        options.failedAt,
+                }
+            );
         },
 
         async cancel(
@@ -269,10 +459,9 @@ MemoryJobQueueAdapter {
                 return false;
             }
 
-            job.state =
-                "cancelled";
-            job.completedAt =
-                cancelledAt;
+            job.state = "cancelled";
+            job.completedAt = cancelledAt;
+            clearLease(job);
             return true;
         },
 
@@ -297,8 +486,153 @@ MemoryJobQueueAdapter {
                 );
         },
 
+        async heartbeat(
+            id,
+            options
+        ) {
+            const job =
+                jobs.get(id);
+
+            if (
+                !job ||
+                job.state !== "running" ||
+                job.leaseOwner !==
+                    options.ownerId
+            ) {
+                return false;
+            }
+
+            const visibilityTimeoutMs =
+                normalizePositiveInteger(
+                    options.visibilityTimeoutMs,
+                    "visibilityTimeoutMs"
+                );
+
+            job.heartbeatAt =
+                options.heartbeatAt;
+            job.leaseUntil =
+                options.heartbeatAt +
+                visibilityTimeoutMs;
+            return true;
+        },
+
+        async recoverStale(
+            now,
+            options
+        ) {
+            return recoverStale(
+                now,
+                options
+            );
+        },
+
+        async listDeadLetters() {
+            return Array.from(
+                deadLetters.values()
+            )
+                .map(cloneDeadLetter)
+                .sort(
+                    (left, right) =>
+                        left.deadLetteredAt -
+                            right.deadLetteredAt ||
+                        left.id.localeCompare(
+                            right.id
+                        )
+                );
+        },
+
+        async requeueDeadLetter(
+            id,
+            now,
+            options = {}
+        ) {
+            const job =
+                jobs.get(id);
+
+            if (
+                !job ||
+                job.state !== "failed" ||
+                !deadLetters.has(id)
+            ) {
+                return false;
+            }
+
+            const delayMs =
+                normalizeNonNegativeNumber(
+                    options.delayMs ?? 0,
+                    "delayMs"
+                );
+
+            job.state = "queued";
+            job.availableAt =
+                now + delayMs;
+            job.startedAt = undefined;
+            job.completedAt = undefined;
+            job.error = undefined;
+            job.recoveredAt = undefined;
+            clearLease(job);
+
+            if (
+                options.resetAttempts !==
+                false
+            ) {
+                job.attempts = 0;
+            }
+
+            deadLetters.delete(id);
+            return true;
+        },
+
+        async cleanup(options) {
+            const states =
+                new Set(
+                    options.states ?? [
+                        "succeeded",
+                        "failed",
+                        "cancelled",
+                    ]
+                );
+            let removed = 0;
+
+            for (
+                const [
+                    id,
+                    job,
+                ] of jobs
+            ) {
+                if (
+                    !states.has(
+                        job.state as
+                            "succeeded" |
+                            "failed" |
+                            "cancelled"
+                    ) ||
+                    job.completedAt === undefined ||
+                    job.completedAt >= options.before
+                ) {
+                    continue;
+                }
+
+                jobs.delete(id);
+                deadLetters.delete(id);
+                removed += 1;
+            }
+
+            return removed;
+        },
+
+        async stats() {
+            return calculateJobStats(
+                Array.from(
+                    jobs.values()
+                ),
+                deadLetters.size
+            );
+        },
+
         clear() {
             jobs.clear();
+            deadLetters.clear();
         },
     };
 }
@@ -314,11 +648,9 @@ export function createJobQueue(
     const workers =
         new Set<InternalJobWorker>();
     const now =
-        options.now ??
-        Date.now;
+        options.now ?? Date.now;
     const idFactory =
-        options.idFactory ??
-        randomUUID;
+        options.idFactory ?? randomUUID;
     const defaultMaxAttempts =
         normalizePositiveInteger(
             options.defaultMaxAttempts ?? 3,
@@ -389,8 +721,7 @@ export function createJobQueue(
             payload,
             enqueueOptions = {}
         ) {
-            const createdAt =
-                now();
+            const createdAt = now();
             const delayMs =
                 normalizeNonNegativeNumber(
                     enqueueOptions.delayMs ?? 0,
@@ -411,23 +742,17 @@ export function createJobQueue(
                 JobRecord<typeof payload> = {
                 id,
                 name:
-                    normalizeJobName(
-                        name
-                    ),
+                    normalizeJobName(name),
                 payload,
                 state: "queued",
                 attempts: 0,
                 maxAttempts,
                 createdAt,
                 availableAt:
-                    createdAt +
-                    delayMs,
+                    createdAt + delayMs,
             };
 
-            await adapter.enqueue(
-                job
-            );
-
+            await adapter.enqueue(job);
             return cloneJob(job);
         },
 
@@ -451,81 +776,141 @@ export function createJobQueue(
         async processNext(
             signal =
                 new AbortController()
-                    .signal
+                    .signal,
+            processOptions = {}
         ) {
             if (signal.aborted) {
                 return false;
             }
 
+            const visibilityTimeoutMs =
+                normalizePositiveInteger(
+                    processOptions
+                        .visibilityTimeoutMs ??
+                    30_000,
+                    "visibilityTimeoutMs"
+                );
+            const heartbeatIntervalMs =
+                normalizePositiveInteger(
+                    processOptions
+                        .heartbeatIntervalMs ??
+                    Math.max(
+                        1,
+                        Math.floor(
+                            visibilityTimeoutMs /
+                            3
+                        )
+                    ),
+                    "heartbeatIntervalMs"
+                );
+            const ownerId =
+                normalizeWorkerId(
+                    processOptions.ownerId ??
+                    `process-${randomUUID()}`
+                );
+
+            if (adapter.recoverStale) {
+                await adapter.recoverStale(
+                    now(),
+                    {
+                        limit:
+                            normalizePositiveInteger(
+                                processOptions
+                                    .recoveryLimit ??
+                                100,
+                                "recoveryLimit"
+                            ),
+                    }
+                );
+            }
+
             const job =
                 await adapter.reserve(
-                    now()
+                    now(),
+                    {
+                        ownerId,
+                        visibilityTimeoutMs,
+                    }
                 );
 
             if (!job) {
                 return false;
             }
 
-            const handler =
-                handlers.get(
-                    job.name
-                );
-
-            if (!handler) {
-                await adapter.fail(
+            const stopHeartbeat =
+                startJobHeartbeat(
+                    adapter,
                     job.id,
-                    {
-                        error:
-                            `No handler registered for job "${job.name}".`,
-                        failedAt:
-                            now(),
-                    }
+                    ownerId,
+                    visibilityTimeoutMs,
+                    heartbeatIntervalMs,
+                    now,
+                    signal
                 );
-                return true;
-            }
+            const handler =
+                handlers.get(job.name);
 
             try {
-                await handler({
-                    job:
-                        cloneJob(job),
-                    payload:
-                        job.payload,
-                    signal,
-                });
+                if (!handler) {
+                    await adapter.fail(
+                        job.id,
+                        {
+                            error:
+                                `No handler registered for job "${job.name}".`,
+                            failedAt:
+                                now(),
+                            ownerId,
+                        }
+                    );
+                    return true;
+                }
 
-                await adapter.complete(
-                    job.id,
-                    now()
-                );
-            } catch (error) {
-                const failedAt =
-                    now();
-                const shouldRetry =
-                    job.attempts <
-                    job.maxAttempts;
-                const retryAt =
-                    shouldRetry
-                        ? failedAt +
-                            resolveRetryDelay(
-                                retryDelay,
-                                job.attempts
-                            )
-                        : undefined;
-
-                await adapter.fail(
-                    job.id,
-                    {
-                        error:
-                            formatJobError(
-                                error
-                            ),
-                        failedAt,
-                        retryAt,
-                    }
-                );
-            }
+                try {
+                    await handler({
+                        job:
+                            cloneJob(job),
+                        payload:
+                            job.payload,
+                        signal,
+                    });
+
+                    await adapter.complete(
+                        job.id,
+                        now(),
+                        ownerId
+                    );
+                } catch (error) {
+                    const failedAt = now();
+                    const shouldRetry =
+                        job.attempts <
+                        job.maxAttempts;
+                    const retryAt =
+                        shouldRetry
+                            ? failedAt +
+                                resolveRetryDelay(
+                                    retryDelay,
+                                    job.attempts
+                                )
+                            : undefined;
+
+                    await adapter.fail(
+                        job.id,
+                        {
+                            error:
+                                formatJobError(
+                                    error
+                                ),
+                            failedAt,
+                            retryAt,
+                            ownerId,
+                        }
+                    );
+                }
 
-            return true;
+                return true;
+            } finally {
+                stopHeartbeat();
+            }
         },
 
         startWorker(
@@ -545,6 +930,76 @@ export function createJobQueue(
             return worker;
         },
 
+        async recoverStale(
+            recoverOptions = {}
+        ) {
+            if (!adapter.recoverStale) {
+                return 0;
+            }
+
+            return adapter.recoverStale(
+                now(),
+                recoverOptions
+            );
+        },
+
+        async deadLetters() {
+            return adapter.listDeadLetters
+                ? adapter.listDeadLetters()
+                : [];
+        },
+
+        async requeueDeadLetter(
+            id,
+            requeueOptions = {}
+        ) {
+            if (!adapter.requeueDeadLetter) {
+                return false;
+            }
+
+            return adapter.requeueDeadLetter(
+                normalizeJobId(id),
+                now(),
+                requeueOptions
+            );
+        },
+
+        async cleanup(cleanupOptions) {
+            if (!adapter.cleanup) {
+                return 0;
+            }
+
+            const before =
+                normalizeNonNegativeNumber(
+                    cleanupOptions.before,
+                    "cleanup before"
+                );
+
+            return adapter.cleanup({
+                ...cleanupOptions,
+                before,
+            });
+        },
+
+        async stats() {
+            if (adapter.stats) {
+                return adapter.stats();
+            }
+
+            const jobs =
+                await adapter.list();
+
+            return calculateJobStats(
+                jobs,
+                adapter.listDeadLetters
+                    ? (
+                        await adapter
+                            .listDeadLetters()
+                    ).length
+                    : 0
+            );
+        },
+
         async close() {
             await Promise.all(
                 Array.from(
@@ -581,10 +1036,37 @@ function createWorker(
             options.pollIntervalMs ?? 250,
             "pollIntervalMs"
         );
+    const visibilityTimeoutMs =
+        normalizePositiveInteger(
+            options.visibilityTimeoutMs ??
+            30_000,
+            "visibilityTimeoutMs"
+        );
+    const heartbeatIntervalMs =
+        normalizePositiveInteger(
+            options.heartbeatIntervalMs ??
+            Math.max(
+                1,
+                Math.floor(
+                    visibilityTimeoutMs /
+                    3
+                )
+            ),
+            "heartbeatIntervalMs"
+        );
+    const recoveryLimit =
+        normalizePositiveInteger(
+            options.recoveryLimit ?? 100,
+            "recoveryLimit"
+        );
+    const workerId =
+        normalizeWorkerId(
+            options.workerId ??
+            `worker-${randomUUID()}`
+        );
     const controller =
         new AbortController();
-    let running =
-        true;
+    let running = true;
     let stopPromise:
         Promise<void> | null =
         null;
@@ -595,11 +1077,19 @@ function createWorker(
                 length:
                     concurrency,
             },
-            () =>
+            (_, index) =>
                 runWorkerLoop(
                     queue,
                     controller.signal,
-                    pollIntervalMs
+                    pollIntervalMs,
+                    {
+                        ownerId:
+                            `${workerId}:${index + 1}`,
+                        visibilityTimeoutMs,
+                        heartbeatIntervalMs,
+                        recoveryLimit,
+                    },
+                    options.onError
                 )
         );
 
@@ -608,13 +1098,14 @@ function createWorker(
             return running;
         },
 
+        workerId,
+
         stop() {
             if (stopPromise) {
                 return stopPromise;
             }
 
-            running =
-                false;
+            running = false;
             controller.abort();
             stopPromise =
                 Promise.allSettled(
@@ -634,13 +1125,32 @@ function createWorker(
 async function runWorkerLoop(
     queue: BackgroundJobQueue,
     signal: AbortSignal,
-    pollIntervalMs: number
+    pollIntervalMs: number,
+    options: ProcessNextJobOptions,
+    onError?: (
+        error: unknown
+    ) => void | Promise<void>
 ): Promise<void> {
     while (!signal.aborted) {
-        const processed =
-            await queue.processNext(
-                signal
-            );
+        let processed = false;
+
+        try {
+            processed =
+                await queue.processNext(
+                    signal,
+                    options
+                );
+        } catch (error) {
+            if (signal.aborted) {
+                break;
+            }
+
+            if (onError) {
+                await onError(error);
+            } else {
+                throw error;
+            }
+        }
 
         if (
             !processed &&
@@ -654,14 +1164,82 @@ async function runWorkerLoop(
     }
 }
 
+function startJobHeartbeat(
+    adapter: JobQueueAdapter,
+    jobId: string,
+    ownerId: string,
+    visibilityTimeoutMs: number,
+    heartbeatIntervalMs: number,
+    now: () => number,
+    signal: AbortSignal
+): () => void {
+    if (!adapter.heartbeat) {
+        return () => undefined;
+    }
+
+    let stopped = false;
+    let timeout:
+        ReturnType<typeof setTimeout> |
+        undefined;
+
+    const schedule = () => {
+        if (
+            stopped ||
+            signal.aborted
+        ) {
+            return;
+        }
+
+        timeout =
+            setTimeout(
+                async () => {
+                    if (
+                        stopped ||
+                        signal.aborted
+                    ) {
+                        return;
+                    }
+
+                    try {
+                        const active =
+                            await adapter
+                                .heartbeat?.(
+                                    jobId,
+                                    {
+                                        ownerId,
+                                        heartbeatAt:
+                                            now(),
+                                        visibilityTimeoutMs,
+                                    }
+                                );
+
+                        if (active !== false) {
+                            schedule();
+                        }
+                    } catch {
+                        schedule();
+                    }
+                },
+                heartbeatIntervalMs
+            );
+    };
+
+    schedule();
+
+    return () => {
+        stopped = true;
+
+        if (timeout) {
+            clearTimeout(timeout);
+        }
+    };
+}
+
 function sleep(
     durationMs: number,
     signal: AbortSignal
 ): Promise<void> {
-    if (
-        durationMs === 0 ||
-        signal.aborted
-    ) {
+    if (signal.aborted) {
         return Promise.resolve();
     }
 
@@ -672,10 +1250,8 @@ function sleep(
                     finish,
                     durationMs
                 );
-
             const onAbort =
-                () =>
-                    finish();
+                () => finish();
 
             signal.addEventListener(
                 "abort",
@@ -697,6 +1273,28 @@ function sleep(
     );
 }
 
+function calculateJobStats(
+    jobs: JobRecord[],
+    deadLetters: number
+): JobQueueStats {
+    const stats:
+        JobQueueStats = {
+        total: jobs.length,
+        queued: 0,
+        running: 0,
+        succeeded: 0,
+        failed: 0,
+        cancelled: 0,
+        deadLetters,
+    };
+
+    for (const job of jobs) {
+        stats[job.state] += 1;
+    }
+
+    return stats;
+}
+
 function resolveRetryDelay(
     value: JobRetryDelay,
     attempt: number
@@ -716,8 +1314,7 @@ function normalizeJobName(
     value: string
 ): string {
     const name =
-        String(value)
-            .trim();
+        String(value).trim();
 
     if (!name) {
         throw new TypeError(
@@ -738,8 +1335,7 @@ function normalizeJobId(
     value: string
 ): string {
     const id =
-        String(value)
-            .trim();
+        String(value).trim();
 
     if (!id) {
         throw new TypeError(
@@ -756,6 +1352,21 @@ function normalizeJobId(
     return id;
 }
 
+function normalizeWorkerId(
+    value: string
+): string {
+    const id =
+        String(value).trim();
+
+    if (!id) {
+        throw new TypeError(
+            "BCP Jobs: worker id must be a non-empty string."
+        );
+    }
+
+    return id;
+}
+
 function normalizePositiveInteger(
     value: number,
     field: string
@@ -811,6 +1422,25 @@ function formatJobError(
     }
 }
 
+function leaseOwnerMatches(
+    job: JobRecord,
+    ownerId?: string
+): boolean {
+    if (ownerId === undefined) {
+        return true;
+    }
+
+    return job.leaseOwner === ownerId;
+}
+
+function clearLease(
+    job: JobRecord
+): void {
+    job.leaseOwner = undefined;
+    job.leaseUntil = undefined;
+    job.heartbeatAt = undefined;
+}
+
 function cloneJob<
     TPayload
 >(
@@ -820,3 +1450,13 @@ function cloneJob<
         ...job,
     };
 }
+
+function cloneDeadLetter<
+    TPayload
+>(
+    job: DeadLetterJobRecord<TPayload>
+): DeadLetterJobRecord<TPayload> {
+    return {
+        ...job,
+    };
+}