@chidchanun/bcp 0.2.9 → 0.2.10

package/packages/server/src/jobs-redis.ts

@@ -0,0 +1,1226 @@
+import type {
+    CleanupJobsOptions,
+    HeartbeatJobOptions,
+    JobQueueAdapter,
+    JobQueueStats,
+    JobRecord,
+    RecoverStaleJobsOptions,
+    RequeueDeadLetterOptions,
+} from "./jobs.js";
+import type {
+    JobScheduleRecord,
+    JobScheduleStore,
+} from "./job-scheduler.js";
+
+export interface RedisCommandClient {
+    sendCommand(
+        command: string[]
+    ): Promise<unknown>;
+}
+
+export interface RedisJobsAdapterOptions {
+    client: RedisCommandClient;
+    namespace?: string;
+    close?: () => Promise<void>;
+}
+
+export interface RedisJobQueueAdapter
+extends JobQueueAdapter {
+    readonly namespace: string;
+}
+
+export interface RedisJobScheduleStore
+extends JobScheduleStore {
+    readonly namespace: string;
+}
+
+export function createRedisJobQueueAdapter(
+    options: RedisJobsAdapterOptions
+): RedisJobQueueAdapter {
+    const client =
+        requireRedisClient(
+            options.client
+        );
+    const namespace =
+        normalizeNamespace(
+            options.namespace
+        );
+    const keys =
+        createQueueKeys(
+            namespace
+        );
+
+    return {
+        namespace,
+
+        async enqueue(job) {
+            const result =
+                await evalRedis(
+                    client,
+                    ENQUEUE_SCRIPT,
+                    [
+                        jobKey(
+                            keys.jobPrefix,
+                            job.id
+                        ),
+                        keys.all,
+                        keys.available,
+                    ],
+                    [
+                        job.id,
+                        JSON.stringify(job),
+                        String(
+                            job.availableAt
+                        ),
+                    ]
+                );
+
+            if (Number(result) !== 1) {
+                throw new Error(
+                    `BCP Jobs Redis: job id "${job.id}" already exists.`
+                );
+            }
+        },
+
+        async reserve(
+            now,
+            reserveOptions
+        ) {
+            const options =
+                reserveOptions ?? {
+                    ownerId:
+                        "legacy-worker",
+                    visibilityTimeoutMs:
+                        30_000,
+                };
+            const raw =
+                await evalRedis(
+                    client,
+                    RESERVE_SCRIPT,
+                    [
+                        keys.available,
+                        keys.running,
+                    ],
+                    [
+                        keys.jobPrefix,
+                        String(now),
+                        options.ownerId,
+                        String(
+                            normalizePositiveInteger(
+                                options.visibilityTimeoutMs,
+                                "visibilityTimeoutMs"
+                            )
+                        ),
+                    ]
+                );
+
+            return parseJsonReply<
+                JobRecord
+            >(raw);
+        },
+
+        async complete(
+            id,
+            completedAt,
+            ownerId
+        ) {
+            await evalRedis(
+                client,
+                COMPLETE_SCRIPT,
+                [
+                    jobKey(
+                        keys.jobPrefix,
+                        id
+                    ),
+                    keys.available,
+                    keys.running,
+                    keys.terminal,
+                    keys.deadLetters,
+                ],
+                [
+                    id,
+                    String(completedAt),
+                    ownerId ?? "",
+                ]
+            );
+        },
+
+        async fail(
+            id,
+            failOptions
+        ) {
+            await evalRedis(
+                client,
+                FAIL_SCRIPT,
+                [
+                    jobKey(
+                        keys.jobPrefix,
+                        id
+                    ),
+                    keys.available,
+                    keys.running,
+                    keys.terminal,
+                    keys.deadLetters,
+                ],
+                [
+                    id,
+                    failOptions.error,
+                    String(
+                        failOptions.failedAt
+                    ),
+                    failOptions.retryAt ===
+                    undefined
+                        ? ""
+                        : String(
+                            failOptions.retryAt
+                        ),
+                    failOptions.ownerId ??
+                        "",
+                ]
+            );
+        },
+
+        async cancel(
+            id,
+            cancelledAt
+        ) {
+            const result =
+                await evalRedis(
+                    client,
+                    CANCEL_SCRIPT,
+                    [
+                        jobKey(
+                            keys.jobPrefix,
+                            id
+                        ),
+                        keys.available,
+                        keys.running,
+                        keys.terminal,
+                    ],
+                    [
+                        id,
+                        String(cancelledAt),
+                    ]
+                );
+
+            return Number(result) === 1;
+        },
+
+        async get(id) {
+            const raw =
+                await client.sendCommand([
+                    "GET",
+                    jobKey(
+                        keys.jobPrefix,
+                        id
+                    ),
+                ]);
+
+            return parseJsonReply<
+                JobRecord<any>
+            >(raw);
+        },
+
+        async list() {
+            return readJobsByIds(
+                client,
+                keys.jobPrefix,
+                await readStringList(
+                    client,
+                    [
+                        "SMEMBERS",
+                        keys.all,
+                    ]
+                )
+            );
+        },
+
+        async heartbeat(
+            id,
+            heartbeatOptions
+        ) {
+            const result =
+                await evalRedis(
+                    client,
+                    HEARTBEAT_SCRIPT,
+                    [
+                        jobKey(
+                            keys.jobPrefix,
+                            id
+                        ),
+                        keys.running,
+                    ],
+                    [
+                        id,
+                        heartbeatOptions.ownerId,
+                        String(
+                            heartbeatOptions
+                                .heartbeatAt
+                        ),
+                        String(
+                            normalizePositiveInteger(
+                                heartbeatOptions
+                                    .visibilityTimeoutMs,
+                                "visibilityTimeoutMs"
+                            )
+                        ),
+                    ]
+                );
+
+            return Number(result) === 1;
+        },
+
+        async recoverStale(
+            now,
+            recoverOptions = {}
+        ) {
+            const result =
+                await evalRedis(
+                    client,
+                    RECOVER_STALE_SCRIPT,
+                    [
+                        keys.running,
+                        keys.available,
+                    ],
+                    [
+                        keys.jobPrefix,
+                        String(now),
+                        String(
+                            normalizePositiveInteger(
+                                recoverOptions.limit ??
+                                100,
+                                "recovery limit"
+                            )
+                        ),
+                    ]
+                );
+
+            return Number(result) || 0;
+        },
+
+        async listDeadLetters() {
+            const ids =
+                await readStringList(
+                    client,
+                    [
+                        "ZRANGE",
+                        keys.deadLetters,
+                        "0",
+                        "-1",
+                    ]
+                );
+            const jobs =
+                await readJobsByIds(
+                    client,
+                    keys.jobPrefix,
+                    ids
+                );
+
+            return jobs
+                .filter(
+                    (
+                        job
+                    ): job is JobRecord & {
+                        state: "failed";
+                        completedAt: number;
+                    } =>
+                        job.state ===
+                            "failed" &&
+                        job.completedAt !==
+                            undefined
+                )
+                .map(
+                    job => ({
+                        ...job,
+                        state:
+                            "failed" as const,
+                        deadLetteredAt:
+                            job.completedAt,
+                    })
+                );
+        },
+
+        async requeueDeadLetter(
+            id,
+            now,
+            requeueOptions = {}
+        ) {
+            const result =
+                await evalRedis(
+                    client,
+                    REQUEUE_DEAD_LETTER_SCRIPT,
+                    [
+                        jobKey(
+                            keys.jobPrefix,
+                            id
+                        ),
+                        keys.available,
+                        keys.running,
+                        keys.terminal,
+                        keys.deadLetters,
+                    ],
+                    [
+                        id,
+                        String(now),
+                        String(
+                            normalizeNonNegativeNumber(
+                                requeueOptions.delayMs ??
+                                0,
+                                "delayMs"
+                            )
+                        ),
+                        requeueOptions
+                            .resetAttempts ===
+                        false
+                            ? "0"
+                            : "1",
+                    ]
+                );
+
+            return Number(result) === 1;
+        },
+
+        async cleanup(cleanupOptions) {
+            const states =
+                cleanupOptions.states ?? [
+                    "succeeded",
+                    "failed",
+                    "cancelled",
+                ];
+            const result =
+                await evalRedis(
+                    client,
+                    CLEANUP_SCRIPT,
+                    [
+                        keys.terminal,
+                        keys.all,
+                        keys.available,
+                        keys.running,
+                        keys.deadLetters,
+                    ],
+                    [
+                        keys.jobPrefix,
+                        String(
+                            cleanupOptions.before
+                        ),
+                        JSON.stringify(states),
+                    ]
+                );
+
+            return Number(result) || 0;
+        },
+
+        async stats() {
+            const jobs =
+                await readJobsByIds(
+                    client,
+                    keys.jobPrefix,
+                    await readStringList(
+                        client,
+                        [
+                            "SMEMBERS",
+                            keys.all,
+                        ]
+                    )
+                );
+            const deadLetters =
+                Number(
+                    await client
+                        .sendCommand([
+                            "ZCARD",
+                            keys.deadLetters,
+                        ])
+                ) || 0;
+
+            return calculateStats(
+                jobs,
+                deadLetters
+            );
+        },
+
+        async close() {
+            if (options.close) {
+                await options.close();
+            }
+        },
+    };
+}
+
+export function createRedisJobScheduleStore(
+    options: RedisJobsAdapterOptions
+): RedisJobScheduleStore {
+    const client =
+        requireRedisClient(
+            options.client
+        );
+    const namespace =
+        normalizeNamespace(
+            options.namespace
+        );
+    const keys =
+        createScheduleKeys(
+            namespace
+        );
+
+    return {
+        namespace,
+
+        async upsert(schedule) {
+            await evalRedis(
+                client,
+                UPSERT_SCHEDULE_SCRIPT,
+                [
+                    scheduleKey(
+                        keys.schedulePrefix,
+                        schedule.id
+                    ),
+                    keys.all,
+                    keys.due,
+                ],
+                [
+                    schedule.id,
+                    JSON.stringify(schedule),
+                    String(
+                        schedule.nextRunAt
+                    ),
+                ]
+            );
+        },
+
+        async get(id) {
+            return parseJsonReply<
+                JobScheduleRecord<any>
+            >(
+                await client
+                    .sendCommand([
+                        "GET",
+                        scheduleKey(
+                            keys.schedulePrefix,
+                            id
+                        ),
+                    ])
+            );
+        },
+
+        async list() {
+            const ids =
+                await readStringList(
+                    client,
+                    [
+                        "SMEMBERS",
+                        keys.all,
+                    ]
+                );
+            const schedules =
+                await readSchedulesByIds(
+                    client,
+                    keys.schedulePrefix,
+                    ids
+                );
+
+            return schedules.sort(
+                (left, right) =>
+                    left.nextRunAt -
+                        right.nextRunAt ||
+                    left.id.localeCompare(
+                        right.id
+                    )
+            );
+        },
+
+        async remove(id) {
+            const result =
+                await evalRedis(
+                    client,
+                    REMOVE_SCHEDULE_SCRIPT,
+                    [
+                        scheduleKey(
+                            keys.schedulePrefix,
+                            id
+                        ),
+                        keys.all,
+                        keys.due,
+                    ],
+                    [
+                        id,
+                    ]
+                );
+
+            return Number(result) === 1;
+        },
+
+        async acquireDue(
+            now,
+            acquireOptions
+        ) {
+            const raw =
+                await evalRedis(
+                    client,
+                    ACQUIRE_DUE_SCHEDULES_SCRIPT,
+                    [
+                        keys.due,
+                    ],
+                    [
+                        keys.schedulePrefix,
+                        String(now),
+                        acquireOptions.ownerId,
+                        String(
+                            normalizePositiveInteger(
+                                acquireOptions.leaseMs,
+                                "schedule leaseMs"
+                            )
+                        ),
+                        String(
+                            normalizePositiveInteger(
+                                acquireOptions.limit,
+                                "schedule limit"
+                            )
+                        ),
+                    ]
+                );
+
+            if (!Array.isArray(raw)) {
+                return [];
+            }
+
+            return raw
+                .map(
+                    value =>
+                        parseJsonReply<
+                            JobScheduleRecord
+                        >(value)
+                )
+                .filter(
+                    (
+                        value
+                    ): value is JobScheduleRecord =>
+                        value !== null
+                );
+        },
+
+        async complete(
+            id,
+            completeOptions
+        ) {
+            await evalRedis(
+                client,
+                COMPLETE_SCHEDULE_SCRIPT,
+                [
+                    scheduleKey(
+                        keys.schedulePrefix,
+                        id
+                    ),
+                    keys.due,
+                ],
+                [
+                    id,
+                    completeOptions.ownerId,
+                    String(
+                        completeOptions
+                            .lastRunAt
+                    ),
+                    String(
+                        completeOptions
+                            .nextRunAt
+                    ),
+                    String(
+                        completeOptions
+                            .updatedAt
+                    ),
+                ]
+            );
+        },
+
+        async release(
+            id,
+            ownerId
+        ) {
+            await evalRedis(
+                client,
+                RELEASE_SCHEDULE_SCRIPT,
+                [
+                    scheduleKey(
+                        keys.schedulePrefix,
+                        id
+                    ),
+                ],
+                [
+                    ownerId,
+                ]
+            );
+        },
+
+        async close() {
+            if (options.close) {
+                await options.close();
+            }
+        },
+    };
+}
+
+function createQueueKeys(
+    namespace: string
+) {
+    return {
+        all:
+            `${namespace}:jobs:all`,
+        available:
+            `${namespace}:jobs:available`,
+        running:
+            `${namespace}:jobs:running`,
+        terminal:
+            `${namespace}:jobs:terminal`,
+        deadLetters:
+            `${namespace}:jobs:dlq`,
+        jobPrefix:
+            `${namespace}:job:`,
+    };
+}
+
+function createScheduleKeys(
+    namespace: string
+) {
+    return {
+        all:
+            `${namespace}:schedules:all`,
+        due:
+            `${namespace}:schedules:due`,
+        schedulePrefix:
+            `${namespace}:schedule:`,
+    };
+}
+
+function jobKey(
+    prefix: string,
+    id: string
+): string {
+    return `${prefix}${id}`;
+}
+
+function scheduleKey(
+    prefix: string,
+    id: string
+): string {
+    return `${prefix}${id}`;
+}
+
+async function evalRedis(
+    client: RedisCommandClient,
+    script: string,
+    keys: string[],
+    args: string[]
+): Promise<unknown> {
+    return client.sendCommand([
+        "EVAL",
+        script,
+        String(keys.length),
+        ...keys,
+        ...args,
+    ]);
+}
+
+async function readStringList(
+    client: RedisCommandClient,
+    command: string[]
+): Promise<string[]> {
+    const result =
+        await client.sendCommand(
+            command
+        );
+
+    if (!Array.isArray(result)) {
+        return [];
+    }
+
+    return result
+        .map(
+            value =>
+                redisString(value)
+        )
+        .filter(
+            (
+                value
+            ): value is string =>
+                value !== null
+        );
+}
+
+async function readJobsByIds(
+    client: RedisCommandClient,
+    prefix: string,
+    ids: string[]
+): Promise<JobRecord[]> {
+    if (ids.length === 0) {
+        return [];
+    }
+
+    const raw =
+        await client.sendCommand([
+            "MGET",
+            ...ids.map(
+                id =>
+                    jobKey(
+                        prefix,
+                        id
+                    )
+            ),
+        ]);
+
+    if (!Array.isArray(raw)) {
+        return [];
+    }
+
+    return raw
+        .map(
+            value =>
+                parseJsonReply<
+                    JobRecord
+                >(value)
+        )
+        .filter(
+            (
+                value
+            ): value is JobRecord =>
+                value !== null
+        )
+        .sort(
+            (left, right) =>
+                left.createdAt -
+                    right.createdAt ||
+                left.id.localeCompare(
+                    right.id
+                )
+        );
+}
+
+async function readSchedulesByIds(
+    client: RedisCommandClient,
+    prefix: string,
+    ids: string[]
+): Promise<JobScheduleRecord[]> {
+    if (ids.length === 0) {
+        return [];
+    }
+
+    const raw =
+        await client.sendCommand([
+            "MGET",
+            ...ids.map(
+                id =>
+                    scheduleKey(
+                        prefix,
+                        id
+                    )
+            ),
+        ]);
+
+    if (!Array.isArray(raw)) {
+        return [];
+    }
+
+    return raw
+        .map(
+            value =>
+                parseJsonReply<
+                    JobScheduleRecord
+                >(value)
+        )
+        .filter(
+            (
+                value
+            ): value is JobScheduleRecord =>
+                value !== null
+        );
+}
+
+function parseJsonReply<T>(
+    value: unknown
+): T | null {
+    const text =
+        redisString(value);
+
+    if (text === null) {
+        return null;
+    }
+
+    return JSON.parse(text) as T;
+}
+
+function redisString(
+    value: unknown
+): string | null {
+    if (typeof value === "string") {
+        return value;
+    }
+
+    if (
+        value instanceof Uint8Array
+    ) {
+        return Buffer.from(
+            value
+        ).toString("utf8");
+    }
+
+    if (
+        typeof value === "number"
+    ) {
+        return String(value);
+    }
+
+    return null;
+}
+
+function calculateStats(
+    jobs: JobRecord[],
+    deadLetters: number
+): JobQueueStats {
+    const stats:
+        JobQueueStats = {
+        total: jobs.length,
+        queued: 0,
+        running: 0,
+        succeeded: 0,
+        failed: 0,
+        cancelled: 0,
+        deadLetters,
+    };
+
+    for (const job of jobs) {
+        stats[job.state] += 1;
+    }
+
+    return stats;
+}
+
+function requireRedisClient(
+    client: RedisCommandClient
+): RedisCommandClient {
+    if (
+        !client ||
+        typeof client.sendCommand !==
+            "function"
+    ) {
+        throw new TypeError(
+            "BCP Jobs Redis: client.sendCommand(command) is required."
+        );
+    }
+
+    return client;
+}
+
+function normalizeNamespace(
+    value = "bcp:{jobs}"
+): string {
+    const namespace =
+        String(value).trim();
+
+    if (!namespace) {
+        throw new TypeError(
+            "BCP Jobs Redis: namespace must be non-empty."
+        );
+    }
+
+    return namespace.replace(
+        /:+$/,
+        ""
+    );
+}
+
+function normalizePositiveInteger(
+    value: number,
+    field: string
+): number {
+    if (
+        !Number.isInteger(value) ||
+        value <= 0
+    ) {
+        throw new TypeError(
+            `BCP Jobs Redis: ${field} must be a positive integer.`
+        );
+    }
+
+    return value;
+}
+
+function normalizeNonNegativeNumber(
+    value: number,
+    field: string
+): number {
+    if (
+        !Number.isFinite(value) ||
+        value < 0
+    ) {
+        throw new TypeError(
+            `BCP Jobs Redis: ${field} must be a non-negative finite number.`
+        );
+    }
+
+    return Math.floor(value);
+}
+
+const ENQUEUE_SCRIPT = `
+if redis.call('EXISTS', KEYS[1]) == 1 then
+    return 0
+end
+redis.call('SET', KEYS[1], ARGV[2])
+redis.call('SADD', KEYS[2], ARGV[1])
+redis.call('ZADD', KEYS[3], ARGV[3], ARGV[1])
+return 1
+`;
+
+const RESERVE_SCRIPT = `
+local ids = redis.call('ZRANGEBYSCORE', KEYS[1], '-inf', ARGV[2], 'LIMIT', 0, 20)
+for _, id in ipairs(ids) do
+    local key = ARGV[1] .. id
+    local raw = redis.call('GET', key)
+    if not raw then
+        redis.call('ZREM', KEYS[1], id)
+    else
+        local job = cjson.decode(raw)
+        if job.state == 'queued' then
+            job.state = 'running'
+            job.attempts = (job.attempts or 0) + 1
+            job.startedAt = tonumber(ARGV[2])
+            job.error = nil
+            job.recoveredAt = nil
+            job.leaseOwner = ARGV[3]
+            job.leaseUntil = tonumber(ARGV[2]) + tonumber(ARGV[4])
+            job.heartbeatAt = tonumber(ARGV[2])
+            local encoded = cjson.encode(job)
+            redis.call('SET', key, encoded)
+            redis.call('ZREM', KEYS[1], id)
+            redis.call('ZADD', KEYS[2], job.leaseUntil, id)
+            return encoded
+        end
+        redis.call('ZREM', KEYS[1], id)
+    end
+end
+return nil
+`;
+
+const COMPLETE_SCRIPT = `
+local raw = redis.call('GET', KEYS[1])
+if not raw then return 0 end
+local job = cjson.decode(raw)
+if job.state == 'cancelled' then return 0 end
+if ARGV[3] ~= '' and job.leaseOwner ~= ARGV[3] then return 0 end
+job.state = 'succeeded'
+job.completedAt = tonumber(ARGV[2])
+job.error = nil
+job.leaseOwner = nil
+job.leaseUntil = nil
+job.heartbeatAt = nil
+redis.call('SET', KEYS[1], cjson.encode(job))
+redis.call('ZREM', KEYS[2], ARGV[1])
+redis.call('ZREM', KEYS[3], ARGV[1])
+redis.call('ZADD', KEYS[4], ARGV[2], ARGV[1])
+redis.call('ZREM', KEYS[5], ARGV[1])
+return 1
+`;
+
+const FAIL_SCRIPT = `
+local raw = redis.call('GET', KEYS[1])
+if not raw then return 0 end
+local job = cjson.decode(raw)
+if job.state == 'cancelled' then return 0 end
+if ARGV[5] ~= '' and job.leaseOwner ~= ARGV[5] then return 0 end
+job.error = ARGV[2]
+job.leaseOwner = nil
+job.leaseUntil = nil
+job.heartbeatAt = nil
+local retryAt = nil
+if ARGV[4] ~= '' then retryAt = tonumber(ARGV[4]) end
+if retryAt and (job.attempts or 0) < (job.maxAttempts or 1) then
+    job.state = 'queued'
+    job.availableAt = retryAt
+    job.startedAt = nil
+    redis.call('SET', KEYS[1], cjson.encode(job))
+    redis.call('ZREM', KEYS[3], ARGV[1])
+    redis.call('ZADD', KEYS[2], retryAt, ARGV[1])
+    return 2
+end
+job.state = 'failed'
+job.completedAt = tonumber(ARGV[3])
+redis.call('SET', KEYS[1], cjson.encode(job))
+redis.call('ZREM', KEYS[2], ARGV[1])
+redis.call('ZREM', KEYS[3], ARGV[1])
+redis.call('ZADD', KEYS[4], ARGV[3], ARGV[1])
+redis.call('ZADD', KEYS[5], ARGV[3], ARGV[1])
+return 1
+`;
+
+const CANCEL_SCRIPT = `
+local raw = redis.call('GET', KEYS[1])
+if not raw then return 0 end
+local job = cjson.decode(raw)
+if job.state == 'succeeded' or job.state == 'failed' or job.state == 'cancelled' then
+    return 0
+end
+job.state = 'cancelled'
+job.completedAt = tonumber(ARGV[2])
+job.leaseOwner = nil
+job.leaseUntil = nil
+job.heartbeatAt = nil
+redis.call('SET', KEYS[1], cjson.encode(job))
+redis.call('ZREM', KEYS[2], ARGV[1])
+redis.call('ZREM', KEYS[3], ARGV[1])
+redis.call('ZADD', KEYS[4], ARGV[2], ARGV[1])
+return 1
+`;
+
+const HEARTBEAT_SCRIPT = `
+local raw = redis.call('GET', KEYS[1])
+if not raw then return 0 end
+local job = cjson.decode(raw)
+if job.state ~= 'running' or job.leaseOwner ~= ARGV[2] then
+    return 0
+end
+job.heartbeatAt = tonumber(ARGV[3])
+job.leaseUntil = tonumber(ARGV[3]) + tonumber(ARGV[4])
+redis.call('SET', KEYS[1], cjson.encode(job))
+redis.call('ZADD', KEYS[2], job.leaseUntil, ARGV[1])
+return 1
+`;
+
+const RECOVER_STALE_SCRIPT = `
+local ids = redis.call('ZRANGEBYSCORE', KEYS[1], '-inf', ARGV[2], 'LIMIT', 0, tonumber(ARGV[3]))
+local recovered = 0
+for _, id in ipairs(ids) do
+    local key = ARGV[1] .. id
+    local raw = redis.call('GET', key)
+    if not raw then
+        redis.call('ZREM', KEYS[1], id)
+    else
+        local job = cjson.decode(raw)
+        if job.state == 'running' and job.leaseUntil and tonumber(job.leaseUntil) <= tonumber(ARGV[2]) then
+            job.state = 'queued'
+            job.availableAt = tonumber(ARGV[2])
+            job.startedAt = nil
+            job.leaseOwner = nil
+            job.leaseUntil = nil
+            job.heartbeatAt = nil
+            job.recoveredAt = tonumber(ARGV[2])
+            redis.call('SET', key, cjson.encode(job))
+            redis.call('ZREM', KEYS[1], id)
+            redis.call('ZADD', KEYS[2], ARGV[2], id)
+            recovered = recovered + 1
+        end
+    end
+end
+return recovered
+`;
+
+const REQUEUE_DEAD_LETTER_SCRIPT = `
+local raw = redis.call('GET', KEYS[1])
+if not raw then return 0 end
+if not redis.call('ZSCORE', KEYS[5], ARGV[1]) then return 0 end
+local job = cjson.decode(raw)
+if job.state ~= 'failed' then return 0 end
+job.state = 'queued'
+job.availableAt = tonumber(ARGV[2]) + tonumber(ARGV[3])
+job.startedAt = nil
+job.completedAt = nil
+job.error = nil
+job.leaseOwner = nil
+job.leaseUntil = nil
+job.heartbeatAt = nil
+job.recoveredAt = nil
+if ARGV[4] == '1' then job.attempts = 0 end
+redis.call('SET', KEYS[1], cjson.encode(job))
+redis.call('ZADD', KEYS[2], job.availableAt, ARGV[1])
+redis.call('ZREM', KEYS[3], ARGV[1])
+redis.call('ZREM', KEYS[4], ARGV[1])
+redis.call('ZREM', KEYS[5], ARGV[1])
+return 1
+`;
+
+const CLEANUP_SCRIPT = `
+local allowed = {}
+for _, state in ipairs(cjson.decode(ARGV[3])) do allowed[state] = true end
+local ids = redis.call('ZRANGEBYSCORE', KEYS[1], '-inf', ARGV[2], 'LIMIT', 0, 1000)
+local removed = 0
+for _, id in ipairs(ids) do
+    local key = ARGV[1] .. id
+    local raw = redis.call('GET', key)
+    if not raw then
+        redis.call('ZREM', KEYS[1], id)
+        redis.call('SREM', KEYS[2], id)
+    else
+        local job = cjson.decode(raw)
+        if allowed[job.state] then
+            redis.call('DEL', key)
+            redis.call('SREM', KEYS[2], id)
+            redis.call('ZREM', KEYS[1], id)
+            redis.call('ZREM', KEYS[3], id)
+            redis.call('ZREM', KEYS[4], id)
+            redis.call('ZREM', KEYS[5], id)
+            removed = removed + 1
+        end
+    end
+end
+return removed
+`;
+
+const UPSERT_SCHEDULE_SCRIPT = `
+redis.call('SET', KEYS[1], ARGV[2])
+redis.call('SADD', KEYS[2], ARGV[1])
+redis.call('ZADD', KEYS[3], ARGV[3], ARGV[1])
+return 1
+`;
+
+const REMOVE_SCHEDULE_SCRIPT = `
+local existed = redis.call('DEL', KEYS[1])
+redis.call('SREM', KEYS[2], ARGV[1])
+redis.call('ZREM', KEYS[3], ARGV[1])
+return existed
+`;
+
+const ACQUIRE_DUE_SCHEDULES_SCRIPT = `
+local ids = redis.call('ZRANGEBYSCORE', KEYS[1], '-inf', ARGV[2], 'LIMIT', 0, tonumber(ARGV[5]) * 4)
+local result = {}
+for _, id in ipairs(ids) do
+    if #result >= tonumber(ARGV[5]) then break end
+    local key = ARGV[1] .. id
+    local raw = redis.call('GET', key)
+    if not raw then
+        redis.call('ZREM', KEYS[1], id)
+    else
+        local schedule = cjson.decode(raw)
+        local leaseUntil = schedule.leaseUntil
+        if schedule.nextRunAt <= tonumber(ARGV[2]) and (not leaseUntil or tonumber(leaseUntil) <= tonumber(ARGV[2])) then
+            schedule.leaseOwner = ARGV[3]
+            schedule.leaseUntil = tonumber(ARGV[2]) + tonumber(ARGV[4])
+            local encoded = cjson.encode(schedule)
+            redis.call('SET', key, encoded)
+            table.insert(result, encoded)
+        end
+    end
+end
+return result
+`;
+
+const COMPLETE_SCHEDULE_SCRIPT = `
+local raw = redis.call('GET', KEYS[1])
+if not raw then return 0 end
+local schedule = cjson.decode(raw)
+if schedule.leaseOwner ~= ARGV[2] then return 0 end
+schedule.lastRunAt = tonumber(ARGV[3])
+schedule.nextRunAt = tonumber(ARGV[4])
+schedule.updatedAt = tonumber(ARGV[5])
+schedule.leaseOwner = nil
+schedule.leaseUntil = nil
+redis.call('SET', KEYS[1], cjson.encode(schedule))
+redis.call('ZADD', KEYS[2], ARGV[4], ARGV[1])
+return 1
+`;
+
+const RELEASE_SCHEDULE_SCRIPT = `
+local raw = redis.call('GET', KEYS[1])
+if not raw then return 0 end
+local schedule = cjson.decode(raw)
+if schedule.leaseOwner ~= ARGV[1] then return 0 end
+schedule.leaseOwner = nil
+schedule.leaseUntil = nil
+redis.call('SET', KEYS[1], cjson.encode(schedule))
+return 1
+`;