@db3.ai/app 0.1.0-beta.1

package/dist/queue/Queue.js

@@ -0,0 +1,685 @@
+import { randomUUID } from 'node:crypto';
+import { DatabaseQueueDriver } from './drivers/DatabaseQueueDriver.js';
+import { RedisQueueDriver } from './drivers/RedisQueueDriver.js';
+import { enableDefaultQueueMonitor } from './queueMonitor.js';
+import { QueueEvents } from './QueueEvents.js';
+import { queueRetryDeadlineReached, queueRetryDelaySeconds, queueRetryUntil, resolveQueueRetryBackoff } from './RetryPolicy.js';
+import { QueueWorker } from './QueueWorker.js';
+import { isQueueableJob, isQueueableJobClass, queueableJobFromJSON, queueableJobName } from './QueueableJob.js';
+import { isQueueRetryLaterError } from './QueueRetryLaterError.js';
+/**
+ * Coordinates queue drivers, job handlers, dispatching, retries, and one-off processing.
+ *
+ * The Queue owns the durable job semantics. Long-running polling lives in QueueWorker so
+ * callers can either process one job at a time or start a worker loop over the same API.
+ */
+export class Queue {
+    db;
+    options;
+    handlers = new Map();
+    driver;
+    monitor;
+    events;
+    constructor(db, options = {}) {
+        this.db = db;
+        this.options = options;
+        this.driver = this.resolveDriver();
+        this.events = new QueueEvents(failure => {
+            this.reportLifecycleFailure(failure);
+        });
+        this.monitor = enableDefaultQueueMonitor(this.options.queueMonitor);
+        if (this.monitor) {
+            this.events.subscribe(event => {
+                this.monitor?.recordLifecycleEvent(event);
+            });
+        }
+    }
+    /**
+     * Returns the queue lifecycle monitor attached to this queue, when enabled.
+     */
+    get queueMonitor() {
+        return this.monitor;
+    }
+    /**
+     * Returns recent terminal failures from the configured queue driver.
+     *
+     * Drivers without persisted failure inspection return an empty collection so
+     * development tooling can use one queue-level API without knowing the driver.
+     *
+     * @param limit - Maximum number of recent failed jobs to return.
+     * @returns Recent failed jobs ordered from newest to oldest.
+     */
+    async failedJobs(limit = 100) {
+        if (!this.driver.failedJobs)
+            return [];
+        return await this.driver.failedJobs(limit);
+    }
+    /**
+     * Replays one terminal failure as a new active queue job.
+     *
+     * The failed record remains untouched for audit. The replacement receives a
+     * new UUID and a retry reference to the original failure.
+     *
+     * @param id - Driver-owned failed-job record id.
+     * @param options - Optional dispatch overrides for the replacement job.
+     * @returns Queue driver id for the replacement job.
+     */
+    async retryFailed(id, options = {}) {
+        if (!this.driver.failedJob) {
+            throw new Error(`Queue driver "${this.driver.name}" does not support failed-job replay.`);
+        }
+        const failed = await this.driver.failedJob(id);
+        if (!failed) {
+            throw new Error(`Failed queue job "${String(id)}" was not found.`);
+        }
+        return await this.pushJob(failed.payload.job, failed.payload.data, {
+            ...options,
+            queue: options.queue ?? failed.queue,
+            maxTries: options.maxTries ?? failed.payload.maxTries,
+            backoff: options.backoff ?? failed.payload.backoff,
+            origin: options.origin ?? failed.payload.origin,
+        }, failed.payload.displayName, failed.payload.chained ?? [], {
+            failedJobId: failed.id,
+            jobUuid: failed.payload.uuid,
+        });
+    }
+    /**
+     * Registers an async handler for a named job payload.
+     */
+    registerHandler(name, handler) {
+        this.handlers.set(name, job => {
+            return {
+                handle: () => handler(job),
+            };
+        });
+    }
+    /**
+     * Registers a queueable job class by rehydrating it for each queued payload.
+     *
+     * @param Job - Queueable job class to register.
+     */
+    registerJob(Job) {
+        if (!isQueueableJobClass(Job)) {
+            throw new Error('Queue.registerJob requires a QueueableJob class.');
+        }
+        this.handlers.set(queueableJobName(Job), job => {
+            const instance = queueableJobFromJSON(Job, job.payload.data);
+            const context = this.createQueueableJobContext(job);
+            return {
+                handle: () => instance.handle(context),
+                onRetry: (error, delaySeconds) => instance.onRetry({
+                    ...context,
+                    error,
+                    delaySeconds,
+                }),
+                onFinalFailure: error => instance.onFinalFailure({
+                    ...context,
+                    error,
+                }),
+            };
+        });
+    }
+    async dispatch(job, dataOrOptions = {}, maybeOptions = {}) {
+        const serialized = isQueueableJob(job) ? job.serialize() : null;
+        const name = resolveDispatchedJobName(job, serialized?.job);
+        const options = serialized ? dataOrOptions : maybeOptions;
+        if (serialized) {
+            if (!this.handlers.has(name)) {
+                this.registerJob(job.constructor);
+            }
+            return await this.dispatchSerializedJob(serialized, options);
+        }
+        const data = dataOrOptions;
+        return await this.pushJob(name, data, options);
+    }
+    /**
+     * Dispatches the first job in a serialized chain.
+     *
+     * @param jobs - Serializable jobs to run in order.
+     * @param options - Dispatch options for the first job.
+     * @returns Queue id for the first queued job.
+     */
+    async chain(jobs, options = {}) {
+        const [firstJob, ...remainingJobs] = jobs;
+        if (!firstJob) {
+            throw new Error('Queue.chain requires at least one job.');
+        }
+        if (!this.handlers.has(queueableJobName(firstJob.constructor))) {
+            this.registerJob(firstJob.constructor);
+        }
+        return await this.dispatchSerializedJob(firstJob.serialize(), options, remainingJobs.map(job => job.serialize()));
+    }
+    /**
+     * Dispatches several queueable jobs independently.
+     *
+     * @param jobs - Serializable jobs to dispatch.
+     * @param options - Dispatch options applied to each job.
+     * @returns Queue ids for the dispatched jobs.
+     */
+    async batch(jobs, options = {}) {
+        const ids = [];
+        for (const job of jobs) {
+            ids.push(await this.dispatch(job, options));
+        }
+        return ids;
+    }
+    /**
+     * Processes at most one available job and returns whether any work was handled.
+     */
+    async processNextJob(queue = this.options.queue || 'default') {
+        return (await this.workNextJob(queue)) !== null;
+    }
+    /**
+     * Claims one job, runs its handler, and records the resulting success, retry, or failure.
+     */
+    async workNextJob(queue = this.options.queue || 'default', options = {}) {
+        const retryAfterSeconds = this.retryAfterSeconds();
+        const job = await this.driver.pop(queue, {
+            retryAfterSeconds,
+        });
+        if (!job)
+            return null;
+        options.onClaimed?.(job);
+        await this.publishJobEvent('claimed', job);
+        let execution = null;
+        const startedAt = performance.now();
+        const stopLeaseHeartbeat = this.startLeaseHeartbeat(job, retryAfterSeconds);
+        try {
+            const factory = await this.handlerForJob(job.payload.job);
+            if (!factory) {
+                throw new Error(`No handler registered for queued job "${job.payload.job}".`);
+            }
+            execution = await factory(job);
+            await execution.handle();
+            await stopLeaseHeartbeat();
+            const leaseError = await this.confirmLease(job, retryAfterSeconds, 'complete');
+            if (leaseError) {
+                return await this.recordLostLease(job, startedAt, leaseError);
+            }
+            await this.dispatchNextChainedJob(job);
+            const completed = await this.driver.delete(job);
+            if (!completed) {
+                return await this.recordLostLease(job, startedAt, this.lostLeaseError(job, 'complete'));
+            }
+            await this.publishJobEvent('succeeded', job, {
+                durationMs: performance.now() - startedAt,
+            });
+            return {
+                job,
+                status: 'succeeded',
+            };
+        }
+        catch (error) {
+            await stopLeaseHeartbeat();
+            const leaseError = await this.confirmLease(job, retryAfterSeconds, 'transition');
+            if (leaseError) {
+                return await this.recordLostLease(job, startedAt, leaseError);
+            }
+            if (isQueueRetryLaterError(error)) {
+                const delaySeconds = Math.max(0, Math.ceil(error.delaySeconds));
+                const deferredJob = {
+                    ...job,
+                    attempts: Math.max(0, job.attempts - 1),
+                };
+                const deferred = await this.driver.defer(job, delaySeconds);
+                if (!deferred) {
+                    return await this.recordLostLease(job, startedAt, this.lostLeaseError(job, 'defer'));
+                }
+                await this.publishJobEvent('deferred', deferredJob, {
+                    delaySeconds,
+                    durationMs: performance.now() - startedAt,
+                    error,
+                });
+                return {
+                    job: deferredJob,
+                    status: 'deferred',
+                    delaySeconds,
+                    error,
+                };
+            }
+            const delaySeconds = queueRetryDelaySeconds(job.payload.backoff ?? this.resolveRetryBackoff(), job.attempts);
+            const retryDeadlineReached = queueRetryDeadlineReached(job.payload.retryUntil, delaySeconds, unixTimestamp());
+            if (job.attempts >= job.payload.maxTries || retryDeadlineReached) {
+                const failed = await this.driver.fail(job, error);
+                if (!failed) {
+                    return await this.recordLostLease(job, startedAt, this.lostLeaseError(job, 'fail'));
+                }
+                const event = await this.publishJobEvent('failed', job, {
+                    durationMs: performance.now() - startedAt,
+                    error,
+                });
+                await this.runJobHook(execution, 'onFinalFailure', event, error);
+                return {
+                    job,
+                    status: 'failed',
+                    error,
+                };
+            }
+            const released = await this.driver.release(job, delaySeconds);
+            if (!released) {
+                return await this.recordLostLease(job, startedAt, this.lostLeaseError(job, 'release'));
+            }
+            const event = await this.publishJobEvent('released', job, {
+                delaySeconds,
+                durationMs: performance.now() - startedAt,
+                error,
+            });
+            await this.runJobHook(execution, 'onRetry', event, error, delaySeconds);
+            return {
+                job,
+                status: 'released',
+                delaySeconds,
+                error,
+            };
+        }
+    }
+    /**
+     * Periodically renews a claimed job while its handler is still running.
+     *
+     * @param job - Claimed queue job.
+     * @param retryAfterSeconds - Driver lease duration in seconds.
+     * @returns Async stop function that also waits for an in-flight renewal.
+     */
+    startLeaseHeartbeat(job, retryAfterSeconds) {
+        const intervalMs = Math.max(1000, Math.floor(retryAfterSeconds * 1000 / 3));
+        let stopped = false;
+        let timer = null;
+        let inFlight = null;
+        /**
+         * Renews the current lease once and schedules the next heartbeat.
+         */
+        const renew = async () => {
+            if (stopped)
+                return;
+            inFlight = this.driver.touch(job, retryAfterSeconds)
+                .then((renewed) => {
+                if (!renewed) {
+                    stopped = true;
+                    console.error(`[queue] Lost lease for ${job.payload.job} job ${String(job.id)} attempt ${job.attempts}.`);
+                }
+            })
+                .catch((error) => {
+                console.error(`[queue] Failed to renew lease for ${job.payload.job} job ${String(job.id)}.`, error);
+            });
+            await inFlight;
+            inFlight = null;
+            if (!stopped) {
+                timer = setTimeout(renew, intervalMs);
+            }
+        };
+        timer = setTimeout(renew, intervalMs);
+        /**
+         * Stops future heartbeats and waits for the current renewal to settle.
+         */
+        return async () => {
+            stopped = true;
+            if (timer) {
+                clearTimeout(timer);
+                timer = null;
+            }
+            await inFlight;
+        };
+    }
+    /**
+     * Renews a lease immediately before committing an attempt outcome.
+     *
+     * A final renewal is both an ownership check and a fresh fencing window for
+     * the short transition that follows. Renewal errors are treated as unknown
+     * ownership so the worker leaves the durable record for safe recovery.
+     *
+     * @param job - Claimed queue job.
+     * @param retryAfterSeconds - Driver lease duration in seconds.
+     * @param operation - Outcome the worker intends to commit.
+     * @returns Null when ownership is confirmed, otherwise a lease error.
+     */
+    async confirmLease(job, retryAfterSeconds, operation) {
+        try {
+            const renewed = await this.driver.touch(job, retryAfterSeconds);
+            return renewed
+                ? null
+                : this.lostLeaseError(job, operation);
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            return this.lostLeaseError(job, operation, message);
+        }
+    }
+    /**
+     * Records that a stale attempt stopped before changing newer queue state.
+     *
+     * @param job - Claimed job whose lease could not be confirmed.
+     * @param startedAt - Monotonic handler start time.
+     * @param error - Error describing the rejected transition.
+     * @returns Lease-lost queue result for worker logging.
+     */
+    async recordLostLease(job, startedAt, error) {
+        await this.publishJobEvent('lease_lost', job, {
+            durationMs: performance.now() - startedAt,
+            error,
+        });
+        return {
+            job,
+            status: 'lease_lost',
+            error,
+        };
+    }
+    /**
+     * Creates a consistent error for one rejected attempt transition.
+     *
+     * @param job - Claimed queue job.
+     * @param operation - Transition rejected by lease fencing.
+     * @param detail - Optional driver error explaining unknown ownership.
+     * @returns Lease ownership error suitable for monitoring and logs.
+     */
+    lostLeaseError(job, operation, detail) {
+        if (detail) {
+            return new Error(`Could not confirm queue attempt ${job.attempts} still owns ${job.payload.job} job ${String(job.id)}; refused to ${operation}. Driver error: ${detail}`);
+        }
+        return new Error(`Queue attempt ${job.attempts} no longer owns ${job.payload.job} job ${String(job.id)}; refused to ${operation}.`);
+    }
+    /**
+     * Starts a polling worker for a queue unless workers are disabled by configuration.
+     */
+    startWorker(queue = process.env.QUEUE_NAME || this.options.queue || 'default', options = {}) {
+        if (this.workerDisabled() && !options.force)
+            return null;
+        const worker = new QueueWorker(this, queue, options.intervalMs ?? this.workerIntervalMs(), options.maxJobsPerTick ?? this.maxJobsPerTick(), options.logger, options.verbose ?? false);
+        worker.start();
+        return worker;
+    }
+    resolveDriver() {
+        if (this.options.driver && typeof this.options.driver !== 'string') {
+            return this.options.driver;
+        }
+        const driverName = this.options.driver
+            || this.options.driverName
+            || queueDriverFromEnv();
+        if (driverName === 'database') {
+            return new DatabaseQueueDriver(this.db);
+        }
+        if (driverName === 'redis') {
+            return new RedisQueueDriver(this.options.redis);
+        }
+        throw new Error(`Unsupported queue driver "${driverName}".`);
+    }
+    /**
+     * Returns a positive reservation timeout for queue claims and heartbeats.
+     *
+     * @returns Queue lease duration in whole seconds.
+     */
+    retryAfterSeconds() {
+        const configured = this.options.retryAfterSeconds
+            ?? Number(process.env.QUEUE_RETRY_AFTER_SECONDS || 90);
+        if (!Number.isFinite(configured) || configured <= 0)
+            return 90;
+        return Math.max(1, Math.trunc(configured));
+    }
+    /**
+     * Resolves the durable backoff stored on a newly dispatched job.
+     *
+     * @param options - Optional per-dispatch backoff overrides.
+     * @returns Complete validated retry backoff.
+     */
+    resolveRetryBackoff(options) {
+        const configuredStrategy = this.options.retryBackoffStrategy
+            ?? retryBackoffStrategyFromEnv();
+        const defaults = {
+            strategy: configuredStrategy,
+            initialSeconds: this.options.retryDelaySeconds
+                ?? Number(process.env.QUEUE_RETRY_DELAY_SECONDS || 15),
+            maxSeconds: this.options.retryMaxDelaySeconds
+                ?? Number(process.env.QUEUE_RETRY_MAX_DELAY_SECONDS || 3600),
+            jitter: this.options.retryJitter
+                ?? booleanFromEnv('QUEUE_RETRY_JITTER', false),
+        };
+        return resolveQueueRetryBackoff(options, defaults);
+    }
+    workerIntervalMs() {
+        return this.options.workerIntervalMs
+            ?? Number(process.env.QUEUE_WORKER_INTERVAL_MS || 1000);
+    }
+    maxJobsPerTick() {
+        return this.options.maxJobsPerTick
+            ?? Number(process.env.QUEUE_WORKER_MAX_JOBS_PER_TICK || 5);
+    }
+    workerDisabled() {
+        if (this.options.workerEnabled !== undefined) {
+            return !this.options.workerEnabled;
+        }
+        return process.env.QUEUE_WORKER === 'false';
+    }
+    /**
+     * Builds the runtime context passed to a rehydrated queueable job.
+     */
+    createQueueableJobContext(job) {
+        return {
+            job,
+            queue: this,
+        };
+    }
+    /**
+     * Returns a registered handler, resolving a conventional job class when possible.
+     */
+    async handlerForJob(jobName) {
+        const existing = this.handlers.get(jobName);
+        if (existing)
+            return existing;
+        if (!this.options.jobResolver)
+            return null;
+        const resolved = await this.options.jobResolver({
+            jobName,
+        });
+        if (!resolved)
+            return null;
+        this.registerJob(resolved);
+        return this.handlers.get(jobName) ?? null;
+    }
+    /**
+     * Pushes a serialized queueable job to the driver.
+     *
+     * @param serialized - Serialized queueable job payload.
+     * @param options - Dispatch options for this queued job.
+     * @param chained - Optional chain tail to attach to this queued job.
+     * @returns Queue id for the pushed job.
+     */
+    async dispatchSerializedJob(serialized, options, chained = serialized.chained ?? []) {
+        return await this.pushJob(serialized.job, serialized.data, options, serialized.displayName, chained);
+    }
+    /**
+     * Dispatches the next job from a completed job chain.
+     *
+     * @param job - Successfully handled queue job.
+     */
+    async dispatchNextChainedJob(job) {
+        const [nextJob, ...remainingJobs] = job.payload.chained ?? [];
+        if (!nextJob)
+            return;
+        await this.dispatchSerializedJob(nextJob, {
+            queue: job.queue,
+            maxTries: job.payload.maxTries,
+            backoff: job.payload.backoff,
+        }, remainingJobs);
+    }
+    /**
+     * Creates and pushes the durable queue envelope.
+     *
+     * @param name - Queue job name.
+     * @param data - JSON-safe job payload data.
+     * @param options - Dispatch options.
+     * @param displayName - Optional human-readable display name.
+     * @param chained - Serialized jobs to run after this job succeeds.
+     * @param retryOf - Optional terminal failure replayed to create this job.
+     * @returns Queue id returned by the driver.
+     */
+    async pushJob(name, data, options, displayName = name, chained = [], retryOf) {
+        const now = unixTimestamp();
+        const retryUntil = queueRetryUntil(now, options.retryUntilSeconds);
+        const payload = {
+            uuid: randomUUID(),
+            displayName,
+            job: name,
+            maxTries: positiveInteger(options.maxTries, 3),
+            backoff: this.resolveRetryBackoff(options.backoff),
+            ...(retryUntil ? { retryUntil } : {}),
+            data,
+            ...(options.origin ? { origin: options.origin } : {}),
+            ...(retryOf ? { retryOf } : {}),
+            ...(chained.length ? { chained } : {}),
+        };
+        const queue = options.queue || this.options.queue || 'default';
+        const delaySeconds = options.delaySeconds || 0;
+        const jobId = await this.driver.push({
+            queue,
+            payload,
+            delaySeconds,
+            createdAt: now,
+        });
+        await this.events.publish({
+            action: 'dispatched',
+            timestamp: new Date().toISOString(),
+            queue,
+            jobName: name,
+            jobId,
+            jobUuid: payload.uuid,
+            attempts: 0,
+            maxTries: payload.maxTries,
+            delaySeconds,
+            origin: payload.origin,
+            retryOf: payload.retryOf,
+            payloadData: data,
+        });
+        return jobId;
+    }
+    /**
+     * Publishes one lifecycle transition for a claimed queue job.
+     *
+     * @param action - Queue transition that has already occurred.
+     * @param job - Claimed job associated with the transition.
+     * @param details - Optional timing, delay, and error values.
+     * @returns Published immutable lifecycle event.
+     */
+    async publishJobEvent(action, job, details = {}) {
+        const event = {
+            action,
+            timestamp: new Date().toISOString(),
+            queue: job.queue,
+            jobName: job.payload.job,
+            jobId: job.id,
+            jobUuid: job.payload.uuid,
+            attempts: job.attempts,
+            maxTries: job.payload.maxTries,
+            origin: job.payload.origin,
+            retryOf: job.payload.retryOf,
+            payloadData: job.payload.data,
+            ...details,
+        };
+        await this.events.publish(event);
+        return event;
+    }
+    /**
+     * Runs one optional QueueableJob hook without changing the queue outcome.
+     *
+     * @param execution - Prepared job execution containing optional hooks.
+     * @param hook - Hook selected by the committed queue transition.
+     * @param event - Lifecycle event already published for the transition.
+     * @param error - Handler error that caused the transition.
+     * @param delaySeconds - Retry delay supplied to onRetry.
+     */
+    async runJobHook(execution, hook, event, error, delaySeconds) {
+        if (!execution)
+            return;
+        try {
+            if (hook === 'onRetry') {
+                await execution.onRetry?.(error, delaySeconds ?? 0);
+                return;
+            }
+            await execution.onFinalFailure?.(error);
+        }
+        catch (hookError) {
+            this.reportLifecycleFailure({
+                source: 'hook',
+                hook,
+                event,
+                error: hookError,
+            });
+        }
+    }
+    /**
+     * Reports an isolated lifecycle failure without throwing into queue work.
+     *
+     * @param failure - Listener or job-hook failure to report.
+     */
+    reportLifecycleFailure(failure) {
+        if (this.options.onLifecycleError) {
+            try {
+                this.options.onLifecycleError(failure);
+                return;
+            }
+            catch (reportingError) {
+                console.error('[queue] Lifecycle error reporter failed.', reportingError);
+            }
+        }
+        const label = failure.hook
+            ? `job hook ${failure.hook}`
+            : 'lifecycle listener';
+        console.error(`[queue] ${label} failed for ${failure.event.jobName} during ${failure.event.action}.`, failure.error);
+    }
+}
+function queueDriverFromEnv() {
+    const driver = process.env.QUEUE_DRIVER || process.env.QUEUE_CONNECTION || 'database';
+    if (driver === 'database')
+        return driver;
+    if (driver === 'redis')
+        return driver;
+    throw new Error(`Unsupported queue driver "${driver}".`);
+}
+function unixTimestamp() {
+    return Math.floor(Date.now() / 1000);
+}
+/**
+ * Resolves the default retry strategy from queue environment configuration.
+ *
+ * @returns Supported retry strategy, defaulting to exponential.
+ */
+function retryBackoffStrategyFromEnv() {
+    return process.env.QUEUE_RETRY_BACKOFF === 'linear'
+        ? 'linear'
+        : 'exponential';
+}
+/**
+ * Reads a boolean queue environment value with a deterministic fallback.
+ *
+ * @param name - Environment variable name.
+ * @param fallback - Value used when the variable is absent or invalid.
+ * @returns Parsed boolean configuration.
+ */
+function booleanFromEnv(name, fallback) {
+    const value = process.env[name]?.trim().toLowerCase();
+    if (value === 'true' || value === '1')
+        return true;
+    if (value === 'false' || value === '0')
+        return false;
+    return fallback;
+}
+/**
+ * Normalizes one positive integer queue option.
+ *
+ * @param value - Requested option value.
+ * @param fallback - Value used when the request is absent or invalid.
+ * @returns Positive integer.
+ */
+function positiveInteger(value, fallback) {
+    return typeof value === 'number' && Number.isFinite(value) && value > 0
+        ? Math.max(1, Math.trunc(value))
+        : fallback;
+}
+/**
+ * Resolves the durable queue job name for any supported dispatch input.
+ */
+function resolveDispatchedJobName(job, serializedName) {
+    if (serializedName)
+        return serializedName;
+    if (typeof job === 'string')
+        return job;
+    return queueableJobName(job.constructor);
+}