@db3.ai/app 0.1.0-beta.1

package/src/queue/QueueEvents.ts

@@ -0,0 +1,57 @@
+import type * as queue from './contracts';
+
+/**
+ * Typed asynchronous lifecycle dispatcher owned by one Queue instance.
+ *
+ * Queue transitions are already durable when events are published. Listener
+ * failures are therefore reported independently and never reject the queue
+ * operation that produced the event.
+ */
+export class QueueEvents {
+	readonly #listeners = new Set<queue.QueueLifecycleListener>();
+
+	/**
+	 * Creates a queue event dispatcher.
+	 *
+	 * @param onListenerError - Reporter for isolated listener failures.
+	 */
+	constructor(
+		private readonly onListenerError: queue.QueueLifecycleErrorHandler,
+	) {}
+
+	/**
+	 * Subscribes to every queue lifecycle event.
+	 *
+	 * @param listener - Listener invoked for subsequent lifecycle events.
+	 * @returns Function that removes the listener.
+	 */
+	subscribe(listener: queue.QueueLifecycleListener): () => void {
+		this.#listeners.add(listener);
+
+		return () => {
+			this.#listeners.delete(listener);
+		};
+	}
+
+	/**
+	 * Publishes one event and waits for every current listener to settle.
+	 *
+	 * @param event - Immutable lifecycle event to publish.
+	 */
+	async publish(event: queue.QueueLifecycleEvent): Promise<void> {
+		const listeners = [...this.#listeners];
+		const results = await Promise.allSettled(
+			listeners.map(async listener => listener(event)),
+		);
+
+		for (const result of results) {
+			if (result.status === 'fulfilled') continue;
+
+			this.onListenerError({
+				source: 'listener',
+				event,
+				error: result.reason,
+			});
+		}
+	}
+}

package/src/queue/QueueRetryLaterError.ts

@@ -0,0 +1,40 @@
+/**
+ * Error type that asks a queue worker to retry a job at a specific later time.
+ *
+ * Throw this for intentional external backpressure, such as provider rate limits,
+ * where the job should remain queued without consuming an application retry.
+ */
+export class QueueRetryLaterError extends Error {
+	/**
+	 * Create a queue retry-later error.
+	 *
+	 * @param delaySeconds - Number of seconds before the job should be available.
+	 * @param message - Human-readable reason for the deferral.
+	 */
+	constructor(
+		readonly delaySeconds: number,
+		message = 'Queue job should retry later.',
+	) {
+		super(message);
+		this.name = 'QueueRetryLaterError';
+	}
+}
+
+/**
+ * Checks whether an unknown error asks the queue to defer without consuming a try.
+ *
+ * @param error - Unknown thrown value.
+ * @returns True when the error carries a finite retry delay.
+ */
+export function isQueueRetryLaterError(error: unknown): error is QueueRetryLaterError {
+	return (
+		error instanceof QueueRetryLaterError
+		|| (
+			typeof error === 'object'
+			&& error !== null
+			&& (error as { name?: unknown }).name === 'QueueRetryLaterError'
+			&& typeof (error as { delaySeconds?: unknown }).delaySeconds === 'number'
+			&& Number.isFinite((error as { delaySeconds?: number }).delaySeconds)
+		)
+	);
+}

package/src/queue/QueueWorker.ts

@@ -0,0 +1,296 @@
+import { unknownErrorMessage } from '@db3.ai/pure';
+
+import type { Queue } from './Queue';
+import type * as queue from './contracts';
+
+export type { QueueLogger, QueueWorkerLifecycle, QueueWorkerOptions } from './contracts';
+
+const DEFAULT_WORKER_INTERVAL_MS = 1000;
+const MIN_WORKER_INTERVAL_MS = 100;
+const DEFAULT_MAX_JOBS_PER_TICK = 5;
+const MIN_MAX_JOBS_PER_TICK = 1;
+
+/**
+ * Polls one queue and asks Queue to process available jobs.
+ *
+ * The worker intentionally owns only timer and lifecycle behavior. It delegates claims,
+ * handler execution, retries, and failure recording back to Queue.
+ */
+export class QueueWorker implements queue.QueueWorkerLifecycle {
+	#activeTick: Promise<void> | null = null;
+	private timer: NodeJS.Timeout | null = null;
+	private readonly intervalMs: number;
+	private readonly maxJobsPerTick: number;
+
+	constructor(
+		private readonly queueManager: Queue,
+		private readonly queue = 'default',
+		intervalMs = Number(process.env.QUEUE_WORKER_INTERVAL_MS || DEFAULT_WORKER_INTERVAL_MS),
+		maxJobsPerTick = Number(process.env.QUEUE_WORKER_MAX_JOBS_PER_TICK || DEFAULT_MAX_JOBS_PER_TICK),
+		private readonly logger?: queue.QueueLogger,
+		private readonly verbose = false,
+	) {
+		this.intervalMs = normalizeWorkerIntervalMs(intervalMs);
+		this.maxJobsPerTick = normalizeMaxJobsPerTick(maxJobsPerTick);
+	}
+
+	/**
+	 * Starts the worker with an immediate tick followed by interval-based polling.
+	 */
+	start(): void {
+		if (this.timer) return;
+
+		this.logger?.info(this.verbose
+			? `[queue] Worker started for "${this.queue}" (interval ${this.intervalMs}ms, max ${this.maxJobsPerTick} jobs/tick).`
+			: `[queue] Worker started for "${this.queue}".`);
+		this.startTick();
+		this.timer = setInterval(() => {
+			this.startTick();
+		}, this.intervalMs);
+	}
+
+	/**
+	 * Stops future polling ticks. Any already-running tick is allowed to finish.
+	 */
+	stop(): void {
+		if (!this.timer) return;
+
+		clearInterval(this.timer);
+		this.timer = null;
+		this.logger?.info(`[queue] Worker stopped for "${this.queue}".`);
+	}
+
+	/**
+	 * Stops future polling and waits for the active queue tick to finish.
+	 */
+	async stopAndDrain(): Promise<void> {
+		this.stop();
+
+		if (this.#activeTick) {
+			await this.#activeTick;
+		}
+	}
+
+	/**
+	 * Starts one tick when the previous tick is no longer running.
+	 */
+	private startTick(): void {
+		if (this.#activeTick) return;
+
+		const tick = this.tick();
+
+		this.#activeTick = tick;
+		void tick.finally(() => {
+			if (this.#activeTick === tick) {
+				this.#activeTick = null;
+			}
+		});
+	}
+
+	/**
+	 * Claims and processes the bounded set of jobs assigned to one polling tick.
+	 */
+	private async tick(): Promise<void> {
+		let processedThisTick = 0;
+
+		try {
+			for (let processed = 0; processed < this.maxJobsPerTick; processed += 1) {
+				const result = await this.queueManager.workNextJob(this.queue, {
+					onClaimed: job => this.logClaimed(job),
+				});
+
+				if (!result) {
+					this.logIdle(processedThisTick);
+					break;
+				}
+
+				processedThisTick += 1;
+				this.logResult(result);
+			}
+
+			if (processedThisTick >= this.maxJobsPerTick) {
+				this.logTickLimit(processedThisTick);
+			}
+		} catch (error) {
+			this.logger?.error?.(
+				`[queue] Worker tick failed for "${this.queue}". ${unknownErrorMessage(error)}`,
+			);
+		}
+	}
+
+	/**
+	 * Logs that the worker reached an idle poll.
+	 *
+	 * @param processedThisTick - Number of jobs processed before the queue became idle.
+	 */
+	private logIdle(processedThisTick: number): void {
+		if (!this.verbose) return;
+
+		const drained = processedThisTick > 0
+			? ` after processing ${processedThisTick} ${plural('job', processedThisTick)}`
+			: '';
+
+		this.logger?.info(
+			`[queue] Idle on "${this.queue}"${drained}; next check in ${this.intervalMs}ms.`,
+		);
+	}
+
+	/**
+	 * Logs that this tick hit the configured maximum jobs per tick.
+	 *
+	 * @param processedThisTick - Number of jobs processed during this tick.
+	 */
+	private logTickLimit(processedThisTick: number): void {
+		if (!this.verbose) return;
+
+		this.logger?.info(
+			`[queue] Processed ${processedThisTick} ${plural('job', processedThisTick)} this tick; next check in ${this.intervalMs}ms.`,
+		);
+	}
+
+	/**
+	 * Logs the final outcome for a processed queue job.
+	 *
+	 * @param result - Queue processing result returned by the queue manager.
+	 */
+	private logResult(result: queue.QueueProcessResult): void {
+		const job = result.job;
+		const label = `${job.payload.job}#${job.id}`;
+
+		if (result.status === 'succeeded') {
+			this.logger?.info(
+				`[queue] Processed ${label} on "${job.queue}".${this.verboseNextJobMessage(job)}`,
+			);
+			return;
+		}
+
+		if (result.status === 'released') {
+			this.logger?.warn?.(
+				`[queue] Released ${label} after attempt ${job.attempts}/${job.payload.maxTries}; retrying in ${result.delaySeconds ?? 0}s. ${unknownErrorMessage(result.error)}`,
+			);
+			return;
+		}
+
+		if (result.status === 'deferred') {
+			this.logger?.warn?.(
+				`[queue] Deferred ${label}; retrying in ${result.delaySeconds ?? 0}s. ${unknownErrorMessage(result.error)}`,
+			);
+			return;
+		}
+
+		if (result.status === 'lease_lost') {
+			this.logger?.warn?.(
+				`[queue] Stopped fenced ${label} attempt ${job.attempts}; lease ownership could not be confirmed. ${unknownErrorMessage(result.error)}`,
+			);
+			return;
+		}
+
+		this.logger?.error?.(
+			`[queue] Failed ${label} after ${job.attempts}/${job.payload.maxTries} attempts. ${unknownErrorMessage(result.error)}`,
+		);
+	}
+
+	/**
+	 * Logs that a queue job has been claimed and is about to run.
+	 *
+	 * @param job - Claimed queue job.
+	 */
+	private logClaimed(job: queue.QueueJob): void {
+		const label = `${job.payload.job}#${job.id}`;
+
+		this.logger?.info(
+			`[queue] Claimed ${label} on "${job.queue}" attempt ${job.attempts}/${job.payload.maxTries}.`,
+		);
+
+		if (!this.verbose) return;
+
+		this.logger?.info(
+			`[queue] Running ${label}; payload keys: ${payloadKeys(job)}; next on success: ${nextJobLabel(job) ?? 'none'}.`,
+		);
+	}
+
+	/**
+	 * Returns a verbose suffix describing the next chained job.
+	 *
+	 * @param job - Completed queue job.
+	 * @returns Empty string or a sentence fragment for the next queued job.
+	 */
+	private verboseNextJobMessage(job: queue.QueueJob): string {
+		if (!this.verbose) return '';
+
+		const next = nextJobLabel(job);
+
+		return next ? ` Queued next ${next}.` : ' No chained job remains.';
+	}
+}
+
+export { QueueWorker as DatabaseQueueWorker };
+
+/**
+ * Formats a singular or plural word for a count.
+ *
+ * @param word - Singular word to pluralize.
+ * @param count - Number of items.
+ * @returns Word with a simple plural suffix when needed.
+ */
+function plural(word: string, count: number): string {
+	return count === 1 ? word : `${word}s`;
+}
+
+/**
+ * Normalizes worker polling intervals so bad env or CLI values cannot create a hot loop.
+ *
+ * @param value - Requested interval in milliseconds.
+ * @returns A safe polling interval in milliseconds.
+ */
+function normalizeWorkerIntervalMs(value: number): number {
+	if (!Number.isFinite(value) || value <= 0) return DEFAULT_WORKER_INTERVAL_MS;
+
+	return Math.max(MIN_WORKER_INTERVAL_MS, Math.floor(value));
+}
+
+/**
+ * Normalizes the per-tick job limit so workers always make bounded forward progress.
+ *
+ * @param value - Requested number of jobs to process per tick.
+ * @returns A safe integer job limit.
+ */
+function normalizeMaxJobsPerTick(value: number): number {
+	if (!Number.isFinite(value) || value <= 0) return DEFAULT_MAX_JOBS_PER_TICK;
+
+	return Math.max(MIN_MAX_JOBS_PER_TICK, Math.floor(value));
+}
+
+/**
+ * Returns a readable list of payload keys for terminal output.
+ *
+ * @param job - Queue job whose payload data should be described.
+ * @returns Comma-separated payload keys or none.
+ */
+function payloadKeys(job: queue.QueueJob): string {
+	const keys = Object.keys(job.payload.data ?? {});
+
+	return keys.length > 0 ? keys.join(', ') : 'none';
+}
+
+/**
+ * Returns the next chained job name from a queue payload.
+ *
+ * @param job - Queue job whose chain should be inspected.
+ * @returns Next chained job label or null.
+ */
+function nextJobLabel(job: queue.QueueJob): string | null {
+	const [next] = job.payload.chained ?? [];
+
+	return next ? serializedJobLabel(next) : null;
+}
+
+/**
+ * Builds a terminal label for a serialized queued job.
+ *
+ * @param job - Serialized queued job.
+ * @returns Readable job label.
+ */
+function serializedJobLabel(job: queue.SerializedQueuedJob): string {
+	return job.displayName || job.job;
+}

package/src/queue/QueueableJob.ts

@@ -0,0 +1,167 @@
+import type * as queue from './contracts';
+
+export type { QueueableJobClass, QueueableJobContext, QueueableJobFailureContext, QueueableJobRetryContext, SerializedQueueableJob } from './contracts';
+
+/**
+ * Base class for application jobs that serialize their runtime data through JSON.
+ *
+ * Normal jobs only need to declare their data type and implement `handle()`.
+ * Override the constructor to validate or normalize input, and override
+ * `fromJSON()` or `toJSON()` only when the default round trip is insufficient.
+ */
+export abstract class QueueableJob<TData extends Record<string, unknown> = Record<string, unknown>> {
+	private chainedJobs: QueueableJob[] = [];
+
+	/**
+	 * Creates a queueable job from its JSON-safe runtime data.
+	 *
+	 * @param data - Data that will be persisted in the queue payload.
+	 */
+	constructor(readonly data: TData) {}
+
+	/**
+	 * Rehydrates a job by passing persisted JSON data to its constructor.
+	 *
+	 * Subclasses may override this for advanced serialization formats. Most jobs
+	 * should validate or normalize data in their constructor instead.
+	 *
+	 * @param data - JSON-safe data restored from the queue payload.
+	 * @returns Rehydrated job instance.
+	 */
+	static fromJSON<TJob extends QueueableJob>(
+		this: new(data: any) => TJob,
+		data: Record<string, unknown>,
+	): TJob {
+		return new this(data);
+	}
+
+	/**
+	 * Attaches jobs that should run after this job completes successfully.
+	 *
+	 * @param jobs - Serializable jobs to dispatch one at a time after this job.
+	 * @returns This job instance for fluent dispatching.
+	 */
+	chain(jobs: QueueableJob[]): this {
+		this.chainedJobs = jobs;
+
+		return this;
+	}
+
+	/**
+	 * Serializes this job into the durable database queue payload shape.
+	 *
+	 * @returns Serialized queueable job envelope data.
+	 */
+	serialize(): queue.SerializedQueueableJob<TData> {
+		const job = queueableJobName(this.constructor as queue.QueueableJobClass);
+		const chainedJobs = this.chainedJobs.length > 0
+			? this.chainedJobs
+			: this.defaultChain();
+		const chained = chainedJobs.map(chainedJob => chainedJob.serialize());
+
+		return {
+			job,
+			displayName: job,
+			data: this.toJSON(),
+			...(chained.length ? { chained } : {}),
+		};
+	}
+
+	/**
+	 * Handles the job after the worker rehydrates it from queued data.
+	 *
+	 * @param context - Queueable job runtime context.
+	 */
+	abstract handle(context: queue.QueueableJobContext): Promise<void>;
+
+	/**
+	 * Handles a failed attempt after the queue has safely scheduled its retry.
+	 *
+	 * Hook failures are reported independently and do not change the retry.
+	 *
+	 * @param context - Queue metadata, failure, and retry delay.
+	 */
+	async onRetry(_context: queue.QueueableJobRetryContext): Promise<void> {}
+
+	/**
+	 * Handles terminal failure after the queue driver persists the failed job.
+	 *
+	 * Hook failures are reported independently and do not change the durable
+	 * terminal outcome.
+	 *
+	 * @param context - Queue metadata and terminal error.
+	 */
+	async onFinalFailure(_context: queue.QueueableJobFailureContext): Promise<void> {}
+
+	/**
+	 * Returns the JSON-safe data stored as this queued job's payload data.
+	 *
+	 * @returns JSON payload data for this job.
+	 */
+	toJSON(): TData {
+		return this.data;
+	}
+
+	/**
+	 * Returns jobs that should follow this job when no explicit chain is set.
+	 *
+	 * @returns Queueable jobs to run after this job succeeds.
+	 */
+	protected defaultChain(): QueueableJob[] {
+		return [];
+	}
+}
+
+/**
+ * Returns true when a value exposes the queueable-job serialization contract.
+ *
+ * @param value - Unknown value to inspect.
+ * @returns True when the value is a queueable job instance.
+ */
+export function isQueueableJob(value: unknown): value is QueueableJob {
+	return Boolean(
+		value
+		&& typeof value === 'object'
+		&& typeof (value as QueueableJob).serialize === 'function',
+	);
+}
+
+/**
+ * Returns true when a class looks like a QueueableJob subclass.
+ *
+ * @param value - Unknown value to inspect.
+ * @returns True when the value is a queueable job class with JSON rehydration.
+ */
+export function isQueueableJobClass(value: unknown): value is queue.QueueableJobClass {
+	if (typeof value !== 'function') return false;
+
+	const prototype = value.prototype as Record<string, unknown> | undefined;
+
+	return Boolean(
+		prototype
+		&& typeof prototype.serialize === 'function'
+		&& typeof prototype.handle === 'function'
+		&& typeof (value as queue.QueueableJobClass).fromJSON === 'function',
+	);
+}
+
+/**
+ * Resolves the durable queue name for a queueable job class.
+ *
+ * @param Job - Queueable job class or compatible class metadata.
+ * @returns Durable queue job name.
+ */
+export function queueableJobName(Job: queue.QueueableJobClass | { name: string; jobName?: string }): string {
+	return Job.jobName || Job.name;
+}
+
+/**
+ * Rehydrates a queueable job instance from persisted JSON payload data.
+ *
+ * @param Job - Queueable job class that owns the payload data.
+ * @param data - JSON payload data stored in the queue envelope.
+ * @returns Rehydrated queueable job instance.
+ */
+export function queueableJobFromJSON(Job: queue.QueueableJobClass, data: Record<string, unknown>): QueueableJob {
+	return Job.fromJSON(data);
+}

package/src/queue/QueuedJob.ts

@@ -0,0 +1,69 @@
+import { ActiveRecord, type FieldBuilder } from '../db';
+import type * as queue from './contracts';
+
+/**
+ * Active queue record persisted by the database queue driver.
+ */
+export class QueuedJob extends ActiveRecord {
+	static override table = 'jobs';
+	static override primaryKey = 'id';
+
+	/**
+	 * Defines the database schema for active queued jobs.
+	 *
+	 * @param field - Field builder used by ActiveRecord schema installation.
+	 * @returns Queue job field definitions.
+	 */
+	static override fields(field: FieldBuilder) {
+		return {
+			id: field.bigIncrements(),
+
+			queue: field.string({
+				required: true,
+				length: 255,
+				index: true,
+			}),
+
+			payload: field.jsonLongText<queue.JobEnvelope>({
+				required: true,
+			}),
+
+			attempts: field.integer({
+				required: true,
+				unsigned: true,
+				min: 0,
+				default: 0,
+			}),
+
+			reservedAt: field.integer({
+				column: 'reserved_at',
+				unsigned: true,
+				min: 0,
+				index: true,
+			}),
+
+			availableAt: field.integer({
+				column: 'available_at',
+				required: true,
+				unsigned: true,
+				min: 0,
+				index: true,
+			}),
+
+			createdAt: field.integer({
+				column: 'created_at',
+				required: true,
+				unsigned: true,
+				min: 0,
+			}),
+		};
+	}
+
+	declare id: number | null;
+	declare queue: string | null;
+	declare payload: queue.JobEnvelope<Record<string, unknown>> | null;
+	declare attempts: number | null;
+	declare reservedAt: number | null;
+	declare availableAt: number | null;
+	declare createdAt: number | null;
+}