@zintrust/queue-cloudflare 2.4.1

package/dist/CloudflareQueueConsumer.js

@@ -0,0 +1,184 @@
+const isEnvelope = (value) => {
+    return (value !== null &&
+        typeof value === 'object' &&
+        value.protocol === 'zintrust.cf.queue.v1' &&
+        typeof value.jobId === 'string');
+};
+const resolveCoordinator = (config, queueName) => {
+    const direct = config?.coordinator;
+    if (direct?.getByName !== undefined)
+        return direct.getByName(queueName ?? 'default');
+    if (direct?.idFromName !== undefined && direct.get !== undefined) {
+        return direct.get(direct.idFromName(queueName ?? 'default'));
+    }
+    const env = globalThis.env;
+    const bindingName = config?.coordinatorBindingName ?? 'QUEUE_COORDINATOR';
+    const binding = env?.[bindingName];
+    if (binding?.getByName !== undefined)
+        return binding.getByName(queueName ?? 'default');
+    if (binding?.idFromName !== undefined && binding.get !== undefined) {
+        return binding.get(binding.idFromName(queueName ?? 'default'));
+    }
+    return null;
+};
+const resolveWorkerId = (config) => {
+    return config?.workerId ?? 'queue-cloudflare-worker';
+};
+const resolveRetryDelay = (job, attempts, baseDelaySeconds) => {
+    const backoff = job.opts.backoff;
+    const baseMs = backoff?.delay ?? baseDelaySeconds * 1000;
+    const delayMs = backoff?.type === 'fixed' ? baseMs : baseMs * 2 ** Math.max(0, attempts);
+    return Math.min(Math.max(0, Math.ceil(delayMs / 1000)), 43200);
+};
+const toEnvelope = (job) => ({
+    protocol: 'zintrust.cf.queue.v1',
+    jobId: job.id,
+    queueName: job.queueName,
+    name: job.name,
+    attempt: job.attemptsMade,
+    availableAt: job.availableAt,
+});
+const serializeError = (error) => {
+    if (error instanceof Error)
+        return { name: error.name, message: error.message, stack: error.stack };
+    return error;
+};
+const createConsumer = (config) => {
+    return {
+        async processMessage(message, batchQueueName) {
+            const envelope = isEnvelope(message.body)
+                ? message.body
+                : {
+                    protocol: 'zintrust.cf.queue.v1',
+                    jobId: message.id,
+                    queueName: config.queueName ?? batchQueueName ?? 'default',
+                    name: 'default',
+                    attempt: message.attempts,
+                    availableAt: new Date().toISOString(),
+                };
+            const queueName = config.queueName ?? envelope.queueName;
+            const coordinator = resolveCoordinator(config.state, queueName);
+            if ((await coordinator?.isPaused?.()) === true) {
+                message.retry({ delaySeconds: config.retryDelaySeconds ?? 30 });
+                return;
+            }
+            const job = await config.store.getJob(queueName, envelope.jobId);
+            if (job === null || job.state === 'canceled') {
+                message.ack();
+                return;
+            }
+            const lease = await coordinator?.acquireLease?.({
+                queueName,
+                jobId: job.id,
+                workerId: resolveWorkerId(config.state),
+                ttlMs: config.state?.heartbeatTtlMs ?? 60_000,
+            });
+            if (lease !== undefined && !lease.acquired) {
+                message.retry({ delaySeconds: config.retryDelaySeconds ?? 30 });
+                return;
+            }
+            await config.store.updateState({
+                queueName,
+                jobId: job.id,
+                state: 'active',
+                incrementAttempts: true,
+            });
+            try {
+                const result = await config.processor(job.data, {
+                    job,
+                    attempt: message.attempts,
+                    updateProgress: async (progress) => {
+                        await config.store.updateProgress(queueName, job.id, progress);
+                    },
+                    log: async (logMessage, data) => {
+                        await config.store.recordLog(queueName, job.id, 'info', logMessage, data);
+                    },
+                    heartbeat: async () => {
+                        await coordinator?.heartbeat?.({
+                            queueName,
+                            jobId: job.id,
+                            workerId: resolveWorkerId(config.state),
+                            ttlMs: config.state?.heartbeatTtlMs ?? 60_000,
+                        });
+                        await config.store.updateState({
+                            queueName,
+                            jobId: job.id,
+                            state: 'active',
+                        });
+                    },
+                });
+                await config.store.updateState({
+                    queueName,
+                    jobId: job.id,
+                    state: 'completed',
+                    result,
+                });
+                const releasedParents = await config.store.markFlowChildCompleted(queueName, job.id);
+                for (const parent of releasedParents) {
+                    await config.queue?.enqueue(parent.queueName, toEnvelope(parent));
+                    await config.store.markDispatched(parent.queueName, parent.id);
+                }
+                await config.store.applyRetention(queueName, job.id, 'completed');
+                await coordinator?.releaseLease?.({
+                    queueName,
+                    jobId: job.id,
+                    workerId: resolveWorkerId(config.state),
+                    ttlMs: config.state?.heartbeatTtlMs ?? 60_000,
+                });
+                message.ack();
+            }
+            catch (error) {
+                const exhausted = job.attemptsMade + 1 >= job.maxAttempts;
+                await config.store.updateState({
+                    queueName,
+                    jobId: job.id,
+                    state: exhausted ? 'failed' : 'retrying',
+                    error: serializeError(error),
+                });
+                if (exhausted) {
+                    await config.store.applyRetention(queueName, job.id, 'failed');
+                    await coordinator?.releaseLease?.({
+                        queueName,
+                        jobId: job.id,
+                        workerId: resolveWorkerId(config.state),
+                        ttlMs: config.state?.heartbeatTtlMs ?? 60_000,
+                    });
+                    message.ack();
+                    return;
+                }
+                message.retry({
+                    delaySeconds: resolveRetryDelay(job, message.attempts, config.retryDelaySeconds ?? 30),
+                });
+            }
+        },
+        async processBatch(batch) {
+            for (const message of batch.messages) {
+                await this.processMessage(message, batch.queue);
+            }
+        },
+    };
+};
+export const CloudflareQueueConsumer = Object.freeze({
+    create: createConsumer,
+    createDeadLetter(input) {
+        return {
+            async processMessage(message, batchQueueName) {
+                const envelope = isEnvelope(message.body) ? message.body : null;
+                const queueName = input.queueName ?? envelope?.queueName ?? batchQueueName ?? 'default';
+                const jobId = envelope?.jobId ?? message.id;
+                await input.store.updateState({
+                    queueName,
+                    jobId,
+                    state: 'dead_lettered',
+                    error: { messageId: message.id, body: message.body, attempts: message.attempts },
+                });
+                message.ack();
+            },
+            async processBatch(batch) {
+                for (const message of batch.messages) {
+                    await this.processMessage(message, batch.queue);
+                }
+            },
+        };
+    },
+});

package/dist/CloudflareQueueCoordinator.d.ts

@@ -0,0 +1,21 @@
+import type { LeaseInput, LeaseResult, RateLimitInput, RateLimitResult } from './types.js';
+type DurableObjectStorageLike = {
+    get: <T = unknown>(key: string) => Promise<T | undefined>;
+    put: (key: string, value: unknown) => Promise<void>;
+    delete: (key: string) => Promise<void>;
+};
+type DurableObjectStateLike = {
+    storage: DurableObjectStorageLike;
+};
+export declare class CloudflareQueueCoordinator {
+    readonly state: DurableObjectStateLike;
+    constructor(state: DurableObjectStateLike);
+    acquireLease(input: LeaseInput): Promise<LeaseResult>;
+    heartbeat(input: LeaseInput): Promise<LeaseResult>;
+    releaseLease(input: LeaseInput): Promise<void>;
+    pause(queueName?: string): Promise<void>;
+    resume(queueName?: string): Promise<void>;
+    isPaused(queueName?: string): Promise<boolean>;
+    rateLimit(input: RateLimitInput): Promise<RateLimitResult>;
+}
+export {};

package/dist/CloudflareQueueCoordinator.js

@@ -0,0 +1,57 @@
+const leaseKey = (queueName, jobId) => `lease:${queueName}:${jobId}`;
+const pausedKey = (queueName) => `paused:${queueName}`;
+const rateLimitKey = (key) => `rate:${key}`;
+export class CloudflareQueueCoordinator {
+    state;
+    constructor(state) {
+        this.state = state;
+    }
+    async acquireLease(input) {
+        const key = leaseKey(input.queueName, input.jobId);
+        const existing = await this.state.storage.get(key);
+        const now = Date.now();
+        if (existing !== undefined && existing.expiresAt > now && existing.owner !== input.workerId) {
+            return { acquired: false, owner: existing.owner, expiresAt: existing.expiresAt };
+        }
+        const expiresAt = now + Math.max(1000, input.ttlMs);
+        await this.state.storage.put(key, { owner: input.workerId, expiresAt });
+        return { acquired: true, owner: input.workerId, expiresAt };
+    }
+    async heartbeat(input) {
+        return await this.acquireLease(input);
+    }
+    async releaseLease(input) {
+        const key = leaseKey(input.queueName, input.jobId);
+        const existing = await this.state.storage.get(key);
+        if (existing === undefined || existing.owner === input.workerId) {
+            await this.state.storage.delete(key);
+        }
+    }
+    async pause(queueName = 'default') {
+        await this.state.storage.put(pausedKey(queueName), true);
+    }
+    async resume(queueName = 'default') {
+        await this.state.storage.delete(pausedKey(queueName));
+    }
+    async isPaused(queueName = 'default') {
+        return (await this.state.storage.get(pausedKey(queueName))) === true;
+    }
+    async rateLimit(input) {
+        const now = Date.now();
+        const max = Math.max(1, Math.floor(input.max));
+        const durationMs = Math.max(1000, Math.floor(input.durationMs));
+        const key = rateLimitKey(input.key);
+        const existing = await this.state.storage.get(key);
+        if (existing === undefined || existing.resetAt <= now) {
+            const record = { remaining: max - 1, resetAt: now + durationMs };
+            await this.state.storage.put(key, record);
+            return { allowed: true, ...record };
+        }
+        if (existing.remaining <= 0) {
+            return { allowed: false, remaining: 0, resetAt: existing.resetAt };
+        }
+        const record = { remaining: existing.remaining - 1, resetAt: existing.resetAt };
+        await this.state.storage.put(key, record);
+        return { allowed: true, ...record };
+    }
+}

package/dist/CloudflareQueueMigrator.d.ts

@@ -0,0 +1,14 @@
+import type { D1DatabaseLike, ZinTrustDatabaseLike } from './types.js';
+export type CloudflareQueueMigrationTarget = {
+    d1: D1DatabaseLike;
+    db?: never;
+} | {
+    db: ZinTrustDatabaseLike;
+    d1?: never;
+};
+export declare const CloudflareQueueMigrator: Readonly<{
+    statements: readonly string[];
+    rollbackStatements: readonly string[];
+    up(target: CloudflareQueueMigrationTarget): Promise<void>;
+    down(target: CloudflareQueueMigrationTarget): Promise<void>;
+}>;

package/dist/CloudflareQueueMigrator.js

@@ -0,0 +1,22 @@
+import { cloudflareQueueMigrationStatements, cloudflareQueueRollbackStatements, } from './migrationSql.js';
+const runStatement = async (target, statement) => {
+    if ('d1' in target && target.d1 !== undefined) {
+        await target.d1.prepare(statement).run();
+        return;
+    }
+    await target.db.execute(statement);
+};
+export const CloudflareQueueMigrator = Object.freeze({
+    statements: cloudflareQueueMigrationStatements,
+    rollbackStatements: cloudflareQueueRollbackStatements,
+    async up(target) {
+        for (const statement of cloudflareQueueMigrationStatements) {
+            await runStatement(target, statement);
+        }
+    },
+    async down(target) {
+        for (const statement of cloudflareQueueRollbackStatements) {
+            await runStatement(target, statement);
+        }
+    },
+});

package/dist/CloudflareQueueScheduler.d.ts

@@ -0,0 +1,43 @@
+import type { CloudflareJobStore } from './CloudflareJobStore.js';
+import type { CloudflareQueueConfig } from './types.js';
+type SchedulerQueueApi = {
+    enqueue<T = unknown>(queue: string, payload: T): Promise<string>;
+};
+export type CloudflareQueueSchedulerConfig = {
+    queueName: string;
+    store: CloudflareJobStore;
+    queue: SchedulerQueueApi;
+    batchSize?: number;
+    stalledAfterMs?: number;
+};
+export declare const CloudflareQueueScheduler: Readonly<{
+    create: (config: CloudflareQueueSchedulerConfig) => {
+        dispatchDueJobs(): Promise<number>;
+        dispatchRepeatables(): Promise<number>;
+        reconcileStalled(): Promise<number>;
+        run(): Promise<{
+            jobs: number;
+            repeatables: number;
+        }>;
+    };
+    createFromQueueConfig(input: {
+        queueName: string;
+        config: CloudflareQueueConfig;
+        store: CloudflareJobStore;
+        queue: SchedulerQueueApi;
+        batchSize?: number;
+    }): {
+        dispatchDueJobs(): Promise<number>;
+        dispatchRepeatables(): Promise<number>;
+        reconcileStalled(): Promise<number>;
+        run(): Promise<{
+            jobs: number;
+            repeatables: number;
+        }>;
+    };
+    createScheduledHandler(config: CloudflareQueueSchedulerConfig): () => Promise<{
+        jobs: number;
+        repeatables: number;
+    }>;
+}>;
+export {};

package/dist/CloudflareQueueScheduler.js

@@ -0,0 +1,99 @@
+const toEnvelope = (input) => ({
+    protocol: 'zintrust.cf.queue.v1',
+    jobId: input.jobId,
+    queueName: input.queueName,
+    name: input.name,
+    attempt: input.attempt,
+    availableAt: input.availableAt,
+});
+const createScheduler = (config) => {
+    return {
+        async dispatchDueJobs() {
+            const jobs = await config.store.getDueJobs(config.queueName, config.batchSize);
+            let dispatched = 0;
+            for (const job of jobs) {
+                await config.queue.enqueue(config.queueName, toEnvelope({
+                    queueName: job.queueName,
+                    jobId: job.id,
+                    name: job.name,
+                    attempt: job.attemptsMade,
+                    availableAt: job.availableAt,
+                }));
+                await config.store.markDispatched(job.queueName, job.id);
+                dispatched += 1;
+            }
+            return dispatched;
+        },
+        async dispatchRepeatables() {
+            const rows = await config.store.getDueRepeatables(config.batchSize);
+            let dispatched = 0;
+            for (const row of rows) {
+                const payload = JSON.parse(row.payload);
+                const job = await config.store.createJob({
+                    queueName: row.queue_name,
+                    name: row.name,
+                    data: payload,
+                });
+                await config.queue.enqueue(row.queue_name, toEnvelope({
+                    queueName: row.queue_name,
+                    jobId: job.id,
+                    name: row.name,
+                    attempt: 0,
+                    availableAt: job.availableAt,
+                }));
+                await config.store.markDispatched(job.queueName, job.id);
+                await config.store.updateRepeatableAfterRun(row);
+                dispatched += 1;
+            }
+            return dispatched;
+        },
+        async reconcileStalled() {
+            const stalled = await config.store.getStalledJobs(config.queueName, config.stalledAfterMs ?? 60_000, config.batchSize);
+            let retried = 0;
+            for (const job of stalled) {
+                await config.store.updateState({
+                    queueName: job.queueName,
+                    jobId: job.id,
+                    state: 'stalled',
+                });
+                await config.store.updateState({
+                    queueName: job.queueName,
+                    jobId: job.id,
+                    state: 'retrying',
+                    availableAt: new Date().toISOString(),
+                });
+                await config.queue.enqueue(job.queueName, toEnvelope({
+                    queueName: job.queueName,
+                    jobId: job.id,
+                    name: job.name,
+                    attempt: job.attemptsMade,
+                    availableAt: job.availableAt,
+                }));
+                await config.store.markDispatched(job.queueName, job.id);
+                retried += 1;
+            }
+            return retried;
+        },
+        async run() {
+            await this.reconcileStalled();
+            const repeatables = await this.dispatchRepeatables();
+            const jobs = await this.dispatchDueJobs();
+            return { jobs, repeatables };
+        },
+    };
+};
+export const CloudflareQueueScheduler = Object.freeze({
+    create: createScheduler,
+    createFromQueueConfig(input) {
+        return createScheduler({
+            queueName: input.queueName,
+            store: input.store,
+            queue: input.queue,
+            batchSize: input.batchSize,
+        });
+    },
+    createScheduledHandler(config) {
+        const scheduler = createScheduler(config);
+        return async () => await scheduler.run();
+    },
+});

package/dist/cli/MigrateCloudflareQueueCommand.d.ts

@@ -0,0 +1,10 @@
+import type { Command } from 'commander';
+type CommandProvider = {
+    getCommand: () => Command;
+    info(message: string): void;
+    success(message: string): void;
+};
+export declare const MigrateCloudflareQueueCommand: Readonly<{
+    create(): CommandProvider;
+}>;
+export {};

package/dist/cli/MigrateCloudflareQueueCommand.js

@@ -0,0 +1,51 @@
+import { BaseCommand } from '@zintrust/core/cli';
+import { execFileSync } from 'node:child_process';
+import { cloudflareQueueMigrationStatements, cloudflareQueueRollbackStatements, } from '../migrationSql.js';
+const addOptions = (command) => {
+    command
+        .option('--database <name>', 'Wrangler D1 database binding/name', 'zintrust_db')
+        .option('--local', 'Run against local D1 database', true)
+        .option('--remote', 'Run against remote D1 database')
+        .option('--rollback', 'Drop Cloudflare queue state tables')
+        .option('--dry-run', 'Print generated SQL without applying it');
+};
+const joinStatements = (statements) => `${statements.join(';\n')};`;
+const runWranglerD1 = (dbName, sql, remote) => {
+    const args = ['d1', 'execute', dbName, remote ? '--remote' : '--local', '--command', sql];
+    return execFileSync('wrangler', args, { encoding: 'utf-8' });
+};
+const execute = async (options, cmd) => {
+    const dbName = typeof options['database'] === 'string' && options['database'].trim() !== ''
+        ? options['database'].trim()
+        : 'zintrust_db';
+    const rollback = options['rollback'] === true;
+    const remote = options['remote'] === true;
+    const statements = rollback
+        ? cloudflareQueueRollbackStatements
+        : cloudflareQueueMigrationStatements;
+    const sql = joinStatements(statements);
+    if (options['dryRun'] === true) {
+        cmd.info(sql);
+        return;
+    }
+    const output = runWranglerD1(dbName, sql, remote);
+    if (output.trim() !== '')
+        cmd.info(output.trim());
+    cmd.success(rollback
+        ? 'Cloudflare queue state tables rolled back.'
+        : 'Cloudflare queue state tables migrated.');
+};
+export const MigrateCloudflareQueueCommand = Object.freeze({
+    create() {
+        return BaseCommand.create({
+            name: 'migrate:queue-cloudflare',
+            description: 'Run @zintrust/queue-cloudflare D1 state migrations',
+            aliases: ['queue-cloudflare:migrate'],
+            addOptions,
+            execute: async (options) => {
+                const command = MigrateCloudflareQueueCommand.create();
+                await execute(options, command);
+            },
+        });
+    },
+});

package/dist/cli-register.d.ts

@@ -0,0 +1,9 @@
+type Registry = {
+    register: (id: string, provider: CliCommandProvider) => void;
+};
+type CliCommandProvider = {
+    getCommand: () => unknown;
+    name?: string;
+};
+export declare function registerCloudflareQueueCliCommands(registry: Registry): void;
+export {};

package/dist/cli-register.js

@@ -0,0 +1,21 @@
+const commandModule = (await import('./cli/MigrateCloudflareQueueCommand.js'));
+export function registerCloudflareQueueCliCommands(registry) {
+    registry.register('migrate:queue-cloudflare', commandModule.MigrateCloudflareQueueCommand.create());
+}
+const globalWithRegistry = globalThis;
+const globalRegistry = globalWithRegistry.__zintrust_cli_command_registry__ ??
+    (globalWithRegistry.__zintrust_cli_command_registry__ = new Map());
+registerCloudflareQueueCliCommands({
+    register: (id, provider) => {
+        globalRegistry.set(id, provider);
+    },
+});
+try {
+    const coreCli = (await import('@zintrust/core/cli'));
+    if (coreCli.OptionalCliCommandRegistry !== undefined) {
+        registerCloudflareQueueCliCommands(coreCli.OptionalCliCommandRegistry);
+    }
+}
+catch {
+    // no-op
+}

package/dist/index.d.ts

@@ -0,0 +1,50 @@
+import { CloudflareQueueConsumer } from './CloudflareQueueConsumer.js';
+import type { CloudflareJobOptions, CloudflareFlowInput, CloudflareFlowResult, CloudflareQueueConfig, CloudflareQueueJob, CloudflareQueueMetrics, CloudflareQueueProcessor, CloudflareQueueState, QueueMessage } from './types.js';
+export type { CloudflareJobOptions, CloudflareFlowInput, CloudflareFlowResult, CloudflareQueueBackoff, CloudflareQueueBinding, CloudflareQueueConfig, CloudflareQueueContentType, CloudflareQueueEnvelope, CloudflareQueueJob, CloudflareQueueMetrics, CloudflareQueueProcessor, CloudflareQueueProcessorContext, CloudflareQueueState, CloudflareQueueStateConfig, CloudflareQueueRetention, CloudflareRepeatOptions, QueueMessage, } from './types.js';
+export { CloudflareJobStore } from './CloudflareJobStore.js';
+export { CloudflareQueueConsumer } from './CloudflareQueueConsumer.js';
+export { CloudflareQueueCoordinator } from './CloudflareQueueCoordinator.js';
+export { CloudflareQueueMigrator } from './CloudflareQueueMigrator.js';
+export { CloudflareQueueScheduler } from './CloudflareQueueScheduler.js';
+type CloudflareQueueDriver = {
+    enqueue<T = unknown>(queue: string, payload: T): Promise<string>;
+    dequeue<T = unknown>(queue: string): Promise<QueueMessage<T> | undefined>;
+    ack(queue: string, id: string): Promise<void>;
+    length(queue: string): Promise<number>;
+    drain(queue: string): Promise<void>;
+    add<T = unknown>(queue: string, name: string, data: T, options?: CloudflareJobOptions): Promise<CloudflareQueueJob<T>>;
+    addBulk<T = unknown>(queue: string, jobs: Array<{
+        name: string;
+        data: T;
+        options?: CloudflareJobOptions;
+    }>): Promise<Array<CloudflareQueueJob<T>>>;
+    getJob<T = unknown>(queue: string, id: string): Promise<CloudflareQueueJob<T> | null>;
+    getJobs<T = unknown>(queue: string, states?: CloudflareQueueState[], limit?: number): Promise<Array<CloudflareQueueJob<T>>>;
+    getJobCounts(queue: string, ...states: CloudflareQueueState[]): Promise<Record<string, number>>;
+    getMetrics(queue: string): Promise<CloudflareQueueMetrics>;
+    updateProgress(queue: string, id: string, progress: unknown): Promise<void>;
+    log(queue: string, id: string, message: string, data?: unknown): Promise<void>;
+    remove(queue: string, id: string): Promise<void>;
+    retry(queue: string, id: string): Promise<void>;
+    promote(queue: string, id: string): Promise<void>;
+    clean(queue: string, states: CloudflareQueueState[], olderThanMs: number): Promise<void>;
+    upsertJobScheduler<T = unknown>(queue: string, name: string, data: T, repeat: NonNullable<CloudflareJobOptions['repeat']>, id?: string): Promise<unknown>;
+    removeJobScheduler(id: string): Promise<void>;
+    runScheduler(queue: string, limit?: number): Promise<{
+        jobs: number;
+        repeatables: number;
+    }>;
+    reconcileStalled(queue: string, olderThanMs?: number, limit?: number): Promise<number>;
+    createFlow<TParent = unknown, TChild = unknown>(input: CloudflareFlowInput<TParent, TChild>): Promise<CloudflareFlowResult<TParent, TChild>>;
+    getFlowChildren<T = unknown>(queue: string, parentJobId: string): Promise<Array<CloudflareQueueJob<T>>>;
+    markFlowChildCompleted(queue: string, childJobId: string): Promise<Array<CloudflareQueueJob>>;
+    createConsumer<T = unknown, TResult = unknown>(processor: CloudflareQueueProcessor<T, TResult>, queueName?: string): ReturnType<typeof CloudflareQueueConsumer.create<T, TResult>>;
+    migrateState(): Promise<void>;
+};
+export declare const CloudflareQueues: Readonly<{
+    create(config?: CloudflareQueueConfig): CloudflareQueueDriver;
+}>;
+export declare const CloudflareQueue: Readonly<{
+    create(config?: CloudflareQueueConfig): CloudflareQueueDriver;
+}>;
+export default CloudflareQueues;