@db3.ai/app 0.1.0-beta.1

package/src/queue/examples/DeferredReportJob.ts

@@ -0,0 +1,25 @@
+import { QueueableJob, QueueRetryLaterError } from '@db3.ai/app/queue';
+import { app } from '@db3.ai/app/server';
+
+/** Durable application deadline, independent of Queue's ordinary retry policy. */
+export interface DeferredReportData extends Record<string, unknown> {
+	/** Epoch milliseconds after which this report must fail instead of deferring. */
+	deadline: number;
+}
+
+/** Demonstrates controlled provider backpressure without contacting a real API. */
+export class DeferredReportJob extends QueueableJob<DeferredReportData> {
+	static readonly jobName = 'reports.defer-example.v1';
+	/** Validates the durable deadline on both producer and worker construction. */
+	constructor(data: DeferredReportData) {
+		if (!data || !Number.isSafeInteger(data.deadline) || data.deadline < 1) throw new Error('A positive deadline in epoch milliseconds is required.');
+		super(data);
+	}
+	/** Applies the application deadline before retry-later can restore an attempt. */
+	async handle(): Promise<void> {
+		if (Date.now() >= this.data.deadline) throw new Error('Report deadline expired.');
+		if (!app().config.get<{ ready: boolean }>('reportProvider')?.ready) throw new QueueRetryLaterError(0, 'Controlled provider backpressure.');
+		// Real provider success would produce a result here. This lab has no charge,
+		// HTTP request or irreversible side effect, so repeated handling is safe.
+	}
+}

package/src/queue/examples/GenerateReportJob.ts

@@ -0,0 +1,37 @@
+import { app } from '@db3.ai/app/server';
+import { QueueableJob } from '@db3.ai/app/queue';
+
+/**
+ * Durable data required to generate one application report.
+ */
+export interface GenerateReportJobData extends Record<string, unknown> {
+	/** Stable application-owned report identifier. */
+	reportId: string;
+}
+
+/**
+ * Example application job restored from its JSON-safe queue payload.
+ */
+export class GenerateReportJob extends QueueableJob<GenerateReportJobData> {
+	/**
+	 * Creates a report job after validating its durable payload.
+	 *
+	 * @param data - Report identity persisted with the queued job.
+	 */
+	constructor(data: GenerateReportJobData) {
+		if (typeof data.reportId !== 'string' || data.reportId.trim() === '') {
+			throw new Error('GenerateReportJob requires a report id.');
+		}
+
+		super(data);
+	}
+
+	/**
+	 * Generates the report through services resolved from the active application.
+	 */
+	async handle(): Promise<void> {
+		app().log.info({
+			reportId: this.data.reportId,
+		}, 'Generating report');
+	}
+}

package/src/queue/examples/WriteReportJob.ts

@@ -0,0 +1,37 @@
+import { QueueableJob } from '@db3.ai/app/queue';
+import { app } from '@db3.ai/app/server';
+
+/** Immutable report revision selected and authorized by the application. */
+export interface WriteReportData extends Record<string, unknown> {
+	/** Safe storage identifier, including a revision when the source can change. */
+	reportId: string;
+}
+
+/** Rebuilds a local report without appending duplicate output on another attempt. */
+export class WriteReportJob extends QueueableJob<WriteReportData> {
+	static readonly jobName = 'reports.write.v1';
+
+	/**
+	 * Validates both newly dispatched and restored payloads.
+	 *
+	 * @param data - Trusted report identity; never a caller-supplied storage path.
+	 */
+	constructor(data: WriteReportData) {
+		if (!data || typeof data.reportId !== 'string' || !/^[a-z0-9][a-z0-9-]{0,63}$/.test(data.reportId)) {
+			throw new Error('reportId must contain 1–64 lowercase letters, digits or hyphens, starting with a letter or digit.');
+		}
+		super(data);
+	}
+
+	/**
+	 * Reads the immutable source and replaces the report at its stable destination.
+	 *
+	 * Missing source bytes throw normally, allowing Queue to apply its retry policy.
+	 * Replacement is repeatable, not an atomic file/database transaction or a
+	 * substitute for provider idempotency when sending mail or charging money.
+	 */
+	async handle(): Promise<void> {
+		const source = await app().storage.readToString(`sources/${this.data.reportId}.txt`);
+		await app().storage.write(`reports/${this.data.reportId}.txt`, `Report: ${source.trim()}\n`);
+	}
+}

package/src/queue/examples/createAndProcessReportJob.ts

@@ -0,0 +1,29 @@
+import { app } from '@db3.ai/app/server';
+import type { QueueJobId, QueueProcessResult } from '@db3.ai/app/queue';
+import { GenerateReportJob } from './GenerateReportJob';
+
+/**
+ * Registers the example job in a worker process.
+ */
+export function registerGenerateReportJob(): void {
+	app().queue.registerJob(GenerateReportJob);
+}
+
+/**
+ * Dispatches one report job through the active application queue.
+ *
+ * @param reportId - Stable application-owned report identifier.
+ * @returns Driver-owned queued job identifier.
+ */
+export async function dispatchGenerateReportJob(reportId: string): Promise<QueueJobId> {
+	return app().queue.dispatch(new GenerateReportJob({ reportId }));
+}
+
+/**
+ * Claims and processes the next job from the default queue.
+ *
+ * @returns Queue result, or null when no job is available.
+ */
+export async function processNextReportJob(): Promise<QueueProcessResult | null> {
+	return app().queue.workNextJob();
+}

package/src/queue/examples/dispatchReportWorkflows.ts

@@ -0,0 +1,40 @@
+import { app } from '@db3.ai/app/server';
+import type { QueueJobId } from '@db3.ai/app/queue';
+import { GenerateReportJob } from './GenerateReportJob';
+
+/**
+ * Dispatches report jobs as an ordered pipeline.
+ *
+ * Each job is persisted only after its predecessor succeeds. A terminal failure
+ * therefore prevents the remaining reports from being dispatched.
+ *
+ * @param reportIds - Report identifiers to process in order.
+ * @returns Driver-owned identifier for the first queued job.
+ */
+export async function dispatchReportPipeline(reportIds: string[]): Promise<QueueJobId> {
+	if (reportIds.length === 0) {
+		throw new Error('A report pipeline requires at least one report.');
+	}
+
+	return app().queue.chain(
+		reportIds.map(reportId => new GenerateReportJob({ reportId })),
+		{
+			queue: 'reports',
+		},
+	);
+}
+
+/**
+ * Dispatches report jobs independently so workers can process them in parallel.
+ *
+ * @param reportIds - Report identifiers that do not depend on one another.
+ * @returns Driver-owned identifiers for every queued job.
+ */
+export async function dispatchReportBatch(reportIds: string[]): Promise<QueueJobId[]> {
+	return app().queue.batch(
+		reportIds.map(reportId => new GenerateReportJob({ reportId })),
+		{
+			queue: 'reports',
+		},
+	);
+}

package/src/queue/examples/manageReportRetries.ts

@@ -0,0 +1,47 @@
+import { app } from '@db3.ai/app/server';
+import { QueueRetryLaterError, type QueueJobId } from '@db3.ai/app/queue';
+import { GenerateReportJob } from './GenerateReportJob';
+
+/**
+ * Dispatches one report with an explicit ordinary-failure retry policy.
+ *
+ * @param reportId - Stable application-owned report identifier.
+ * @returns Driver-owned queued job identifier.
+ */
+export async function dispatchReportWithRetryPolicy(reportId: string): Promise<QueueJobId> {
+	return app().queue.dispatch(new GenerateReportJob({ reportId }), {
+		queue: 'reports',
+		maxTries: 8,
+		backoff: {
+			strategy: 'exponential',
+			initialSeconds: 15,
+			maxSeconds: 900,
+			jitter: true,
+		},
+		retryUntilSeconds: 21_600,
+	});
+}
+
+/**
+ * Defers the current job when a provider supplies an explicit retry delay.
+ *
+ * Throw this from a QueueableJob handler after detecting provider backpressure.
+ * Deferral restores the claimed attempt instead of consuming an ordinary retry.
+ *
+ * @param delaySeconds - Provider-requested delay before another attempt.
+ */
+export function retryReportAfterBackpressure(delaySeconds: number): never {
+	throw new QueueRetryLaterError(delaySeconds, 'The report provider requested backpressure.');
+}
+
+/**
+ * Replays one terminal report failure while preserving its audit record.
+ *
+ * @param failedJobId - Driver-owned identifier from failedJobs().
+ * @returns Driver-owned identifier for the replacement queue job.
+ */
+export async function retryFailedReportJob(failedJobId: QueueJobId): Promise<QueueJobId> {
+	return app().queue.retryFailed(failedJobId, {
+		queue: 'reports',
+	});
+}

package/src/queue/examples/outputs/create-and-process-report-job.json

@@ -0,0 +1,6 @@
+{
+	"jobId": 1,
+	"status": "succeeded",
+	"completedJobs": 1,
+	"remainingJobs": 0
+}

package/src/queue/examples/outputs/dispatch-report-workflows.json

@@ -0,0 +1,14 @@
+{
+	"pipeline": {
+		"initiallyQueued": 1,
+		"completedJobs": 2
+	},
+	"batch": {
+		"jobIds": [
+			3,
+			4
+		],
+		"queuedJobs": 2,
+		"queue": "reports"
+	}
+}

package/src/queue/examples/outputs/manage-report-retries.json

@@ -0,0 +1,24 @@
+{
+	"policy": {
+		"queue": "reports",
+		"maxTries": 8,
+		"backoff": {
+			"strategy": "exponential",
+			"initialSeconds": 15,
+			"maxSeconds": 900,
+			"jitter": true
+		}
+	},
+	"backpressure": {
+		"error": "QueueRetryLaterError",
+		"delaySeconds": 45
+	},
+	"replay": {
+		"replacementId": 2,
+		"failedAuditRecords": 1,
+		"retryOf": {
+			"failedJobId": "failed_report_01",
+			"jobUuid": "failed-report-uuid"
+		}
+	}
+}

package/src/queue/examples/outputs/queue-reports.json

@@ -0,0 +1,11 @@
+{
+	"wrongQueueIdle": true,
+	"firstAttempt": "released",
+	"finalAttempt": "failed",
+	"replayLinked": true,
+	"repeated": "succeeded",
+	"text": "Report: Three notes ready\n",
+	"reportFiles": 1,
+	"remainingJobs": 0,
+	"retainedFailures": 1
+}

package/src/queue/examples/reportConsole.ts

@@ -0,0 +1,47 @@
+import 'dotenv/config';
+import { resolve } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { runQueueConsole, type QueueConsoleOptions } from '@db3.ai/app/queue';
+import { App } from '@db3.ai/app/server';
+import { WriteReportJob } from './WriteReportJob';
+
+/**
+ * Wires the public queue console to a bootstrapped application.
+ *
+ * @param application - Application with migrated queue tables and report storage.
+ * @returns CLI configuration; finite commands close the App, workers drain on signals.
+ */
+export function reportConsoleOptions(application: App): QueueConsoleOptions {
+	return {
+		app: () => application,
+		bootstrap: () => application.queue.registerJob(WriteReportJob),
+		commands: [{
+			command: 'queue:failed',
+			/** Lists bounded operational metadata without exposing payloads or exceptions. */
+			async run({ app }) {
+				const failures = await app.queue.failedJobs(25);
+				console.log(JSON.stringify(failures.map(failure => ({ id: failure.id, queue: failure.queue, job: failure.payload.job, failedAt: failure.failedAt })), null, 2));
+			},
+		}, {
+			command: 'report:seed',
+			/** Writes only the fixed sample input; never accepts a user-controlled file path. */
+			async run() {
+				await application.storage.write('sources/weekly-v1.txt', 'Three notes ready');
+				console.log('Sample source ready: sources/weekly-v1.txt');
+			},
+		}],
+	};
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+	const application = new App({
+		queue: { driver: 'database', queue: 'reports' },
+		storage: { disks: { local: { driver: 'local', root: resolve(process.env.REPORT_STORAGE_ROOT || 'data/queue-reports') } } },
+	});
+	try {
+		await runQueueConsole(reportConsoleOptions(application));
+	} catch (error) {
+		await application.close();
+		throw error;
+	}
+}

package/src/queue/examples/runBackpressure.ts

@@ -0,0 +1,27 @@
+import { pathToFileURL } from 'node:url';
+import { createGeneratedTestDatabase } from '@db3.ai/app/db/test/db';
+import { FailedJob, QueuedJob } from '@db3.ai/app/queue';
+import { App } from '@db3.ai/app/server';
+import { DeferredReportJob } from './DeferredReportJob';
+
+/** Exercises SQL-backed deferral, unchanged attempts, readiness recovery and a hard deadline. */
+export async function runBackpressure() {
+	const database = await createGeneratedTestDatabase('backpressure_guide');
+	const reportProvider = { ready: false };
+	const application = new App({ db: database.db, config: { reportProvider }, queue: { queue: 'reports', queueMonitor: false } });
+	try {
+		await application.db.install(QueuedJob, FailedJob);
+		application.queue.registerJob(DeferredReportJob);
+		const id = await application.queue.dispatch(new DeferredReportJob({ deadline: Date.now() + 60_000 }), { queue: 'reports', maxTries: 1 });
+		const first = await application.queue.workNextJob('reports');
+		const second = await application.queue.workNextJob('reports');
+		const deferred = await QueuedJob.findOrFail(id);
+		reportProvider.ready = true;
+		const recovered = await application.queue.workNextJob('reports');
+		await application.queue.dispatch(new DeferredReportJob({ deadline: 1 }), { queue: 'reports', maxTries: 1 });
+		const expired = await application.queue.workNextJob('reports');
+		return { first: first?.status, second: second?.status, attemptsAfterDeferral: deferred.attempts, recovered: recovered?.status, expired: expired?.status, remainingJobs: await QueuedJob.query().count(), failures: (await application.queue.failedJobs(10)).length };
+	} finally { try { await application.close(); } finally { await database.destroy(); } }
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) console.log(JSON.stringify(await runBackpressure(), null, 2));

package/src/queue/examples/runQueueReports.ts

@@ -0,0 +1,80 @@
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { createGeneratedTestDatabase } from '@db3.ai/app/db/test/db';
+import { FailedJob, QueuedJob } from '@db3.ai/app/queue';
+import { App } from '@db3.ai/app/server';
+import { WriteReportJob } from './WriteReportJob';
+
+/**
+ * Runs durable dispatch, failure repair, replay and duplicate-safe output on SQL.
+ *
+ * The producer is closed before a fresh worker App restores persisted work.
+ * Only this disposable lab uses Database.install(); deployed apps use migrations.
+ * A zero retry delay keeps the failure demonstration deterministic without sleeps.
+ *
+ * @returns Stable observations also asserted by the shipped documentation test.
+ */
+export async function runQueueReports() {
+	const root = await mkdtemp(join(tmpdir(), 'db3-queue-guide-'));
+	try {
+		const database = await createGeneratedTestDatabase('queue_guide');
+		try {
+			const producer = new App({ db: database.db, queue: { driver: 'database', queue: 'reports', queueMonitor: false } });
+			try {
+				await producer.db.install(QueuedJob, FailedJob);
+				await producer.queue.dispatch(new WriteReportJob({ reportId: 'weekly-v1' }), {
+					maxTries: 2, backoff: { strategy: 'linear', initialSeconds: 0, maxSeconds: 0, jitter: false },
+				});
+			} finally {
+				await producer.close();
+			}
+
+			const consumer = new App({
+				db: database.db,
+				queue: { driver: 'database', queue: 'reports', queueMonitor: false },
+				storage: { disks: { local: { driver: 'local', root } } },
+			});
+			try {
+				consumer.queue.registerJob(WriteReportJob);
+				const wrongQueueIdle = await consumer.queue.workNextJob('default') === null;
+				const firstAttempt = await consumer.queue.workNextJob('reports');
+				const finalAttempt = await consumer.queue.workNextJob('reports');
+				const [failed] = await consumer.queue.failedJobs(10);
+				if (!failed) throw new Error('Expected the missing source to produce a persisted terminal failure.');
+
+				// Repair the cause before replaying. No queued payload or audit row is edited.
+				await consumer.storage.write('sources/weekly-v1.txt', 'Three notes ready');
+				const replacementId = await consumer.queue.retryFailed(failed.id);
+				const replacement = await QueuedJob.findOrFail(replacementId);
+				const replayLinked = replacement.payload?.retryOf?.failedJobId === failed.id;
+				const worker = consumer.queue.startWorker('reports', { force: true, maxJobsPerTick: 1 });
+				if (!worker) throw new Error('The report worker did not start.');
+				await worker.stopAndDrain();
+
+				await consumer.queue.dispatch(new WriteReportJob({ reportId: 'weekly-v1' }));
+				const repeated = await consumer.queue.workNextJob('reports');
+				const files = [];
+				for await (const file of consumer.storage.list('reports')) files.push(file);
+				return {
+					wrongQueueIdle, firstAttempt: firstAttempt?.status, finalAttempt: finalAttempt?.status,
+					replayLinked, repeated: repeated?.status,
+					text: await consumer.storage.readToString('reports/weekly-v1.txt'),
+					reportFiles: files.length, remainingJobs: await QueuedJob.query().count(),
+					retainedFailures: (await consumer.queue.failedJobs(10)).length,
+				};
+			} finally {
+				await consumer.close();
+			}
+		} finally {
+			await database.destroy();
+		}
+	} finally {
+		await rm(root, { recursive: true, force: true });
+	}
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+	console.log(JSON.stringify(await runQueueReports(), null, 2));
+}

package/src/queue/examples/runQueueWorkflows.ts

@@ -0,0 +1,43 @@
+import { pathToFileURL } from 'node:url';
+import { createGeneratedTestDatabase } from '@db3.ai/app/db/test/db';
+import { FailedJob, QueuedJob } from '@db3.ai/app/queue';
+import { App } from '@db3.ai/app/server';
+import { registerGenerateReportJob } from './createAndProcessReportJob';
+import { dispatchReportBatch, dispatchReportPipeline } from './dispatchReportWorkflows';
+
+/**
+ * Compares sequential chains with immediately queued batches using real SQL.
+ *
+ * The example job only logs a report identity. This lab proves orchestration,
+ * not a reporting provider. It creates and destroys its own test database.
+ *
+ * @returns Stable queue counts, validation and processing observations.
+ */
+export async function runQueueWorkflows() {
+	const database = await createGeneratedTestDatabase('queue_workflows');
+	const application = new App({ db: database.db, queue: { queue: 'reports', queueMonitor: false } });
+	try {
+		application.log.level = 'silent';
+		await application.db.install(QueuedJob, FailedJob);
+		registerGenerateReportJob();
+		let invalidRejected = false;
+		try { await dispatchReportPipeline([]); } catch (error) { invalidRejected = error instanceof Error && error.message.includes('at least one report'); }
+		const afterInvalid = await QueuedJob.query().count();
+		await dispatchReportPipeline(['first', 'second']);
+		const chainInitial = await QueuedJob.query().count();
+		const first = await application.queue.workNextJob('reports');
+		const chainRemaining = await QueuedJob.query().count();
+		const second = await application.queue.workNextJob('reports');
+		const batch = await dispatchReportBatch(['third', 'fourth']);
+		const batchInitial = await QueuedJob.query().count();
+		const third = await application.queue.workNextJob('reports');
+		const fourth = await application.queue.workNextJob('reports');
+		return { invalidRejected, afterInvalid, chainInitial, chainRemaining, batchInitial, batchIds: batch.length, statuses: [first?.status, second?.status, third?.status, fourth?.status], remaining: await QueuedJob.query().count() };
+	} finally {
+		try { await application.close(); } finally { await database.destroy(); }
+	}
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+	console.log(JSON.stringify(await runQueueWorkflows(), null, 2));
+}

package/src/queue/index.ts

@@ -0,0 +1,13 @@
+export * from './FailedJob';
+export * from './QueuedJob';
+export * from './Queue';
+export * from './QueueEvents';
+export * from './QueueableJob';
+export * from './QueueRetryLaterError';
+export * from './QueueWorker';
+export * from './queueMonitor';
+export * from './console';
+export * from './drivers/DatabaseQueueDriver';
+export * from './drivers/RedisQueueDriver';
+export * from './contracts';
+export * from './createJob';