@db3.ai/app 0.1.0-beta.1 → 0.1.0-beta.4

package/src/queue/README.md

@@ -134,7 +134,7 @@ For a continuous worker, use a separate terminal:
 npx tsx examples/reportConsole.ts queue:work --queue=reports --max-jobs=1
 ```
 
-Ctrl-C drains the active tick before closing the App. To replay a repaired
+Ctrl-C finishes the active job before closing the App. To replay a repaired
 terminal failure, use its failed-record ID, not its original active-job ID:
 
 ```sh
@@ -177,8 +177,53 @@ work per tick; it is not concurrency. Use more supervised worker processes for
 parallelism. Polling defaults to 1000 ms and valid intervals are clamped to at
 least 100 ms. `queue:work --once` checks once and exits, even if delayed work exists.
 
+### Share worker capacity
+
+Jobs still target one queue. Workers can consume several exact names or all
+names in their database/Redis namespace:
+
+```ts
+const general = app.queue.startWorker({
+	queues: '*',
+	excludeQueues: ['articles', 'article-images'],
+});
+const shared = app.queue.startWorker({ queues: '*' });
+// Fixed selection: { queues: ['default', 'articles'] }.
+// At shutdown:
+await Promise.all([general?.stopAndDrain(), shared?.stopAndDrain()]);
+```
+
+Exclusions win before claiming. Names are exact, without patterns; duplicates
+are removed. Empty names/arrays and fully excluded explicit selections fail.
+Omitting selection retains the configured default. Each worker executes one
+job at a time, rotates after a claim and skips unavailable queues. Round-robin
+shares claim opportunities, not processing time; running jobs are not preempted.
+
+Wildcard discovery refreshes before each claim/poll and includes delayed and
+reserved queues. Claims still enforce availability and lease ownership.
+Database and Redis support discovery; custom drivers need `queueNames()`.
+Redis uses SCAN over existing queue keys. Prefer explicit names on large shared
+keyspaces. Newly introduced queues automatically become eligible for `'*'`.
+
+```sh
+node --import tsx server/console.ts queue:work --queues=default,articles
+node --import tsx server/console.ts queue:work --queues='*' --exclude-queues=articles,article-images
+```
+
+Pass selections in `runQueueConsole({ app, workers: { general: selection } })`
+and use `--pool=general`, without queue flags. `--once` applies the same rules
+and closes the app on success/failure. Existing `--queue=reports` still works.
+For timer-free processing, use `new QueueWorker(app.queue, selection).workOnce()`;
+concurrent calls on one worker fail. See the disposable SQL
+[selection lab](examples/runQueueSelection.ts).
+
+Routing changes affect new jobs only: retain consumers for old names during
+rollout/rollback. Monitor each workload queue independently of general health.
+
 ## Retries and repair
 
+`QueueTerminalError` stops immediately regardless of remaining tries, retaining the failed job and invoking final-failure hooks. Use it when ordinary retries cannot repair the cause.
+
 Ordinary errors consume attempts. Dispatch defaults are three attempts,
 exponential backoff starting at 15 seconds, capped at 3600 seconds, with jitter
 off unless changed in queue options/environment. The supported strategies are
@@ -366,3 +411,99 @@ handler throws so a generated skeleton cannot silently consume real work.
 Editor or Studio integrations can use `jobTemplate(name)` to preview the same
 source and `createJob({ appDirectory, name })` to create the file. These are
 shared framework operations; a Studio screen is not included in this package.
+
+## Atomic application dispatch
+
+The database driver's `push` participates in an explicit `ActiveRecord.withDb`
+scope, including one established by `app().db.transaction`. Application records
+and queued jobs can therefore commit or roll back together. Outside a scope it
+uses its configured database. Redis dispatch is not transactional with SQL.
+Dispatch lifecycle events may run before the surrounding transaction commits;
+observers must not perform irreversible external work on those events.
+
+## Retained runnable records
+
+Use `RunnableJob` when an operation needs durable progress and history. It is an
+ActiveRecord implementing the queue contract; concrete subclasses share
+`job_runs`. Register `RunnableJob`, `JobRunAttempt` and `JobRunEvent` with the
+application's migration registry, alongside `QueuedJob` and `FailedJob`.
+
+```ts
+import { RunnableJob, type QueueableJobContext } from '@db3.ai/app/queue';
+
+/** Retains one organization's progress between delivery attempts. */
+class CountItemsJob extends RunnableJob {
+	/** Saves a checkpoint after each bounded unit of work. */
+	override async handle(_context: QueueableJobContext): Promise<void> {
+		this.total = 10;
+		for (let index = this.completed ?? 0; index < 10; index++) {
+			await this.assertActive();
+			await this.checkpoint(run => {
+				run.completed = index + 1;
+				run.state = { lastItem: index };
+			});
+		}
+		await this.log('All items processed');
+	}
+}
+
+const run = CountItemsJob.create({ tenantId: organizationId, input: {} });
+await app.queue.dispatch(run); // persists before execution, then queues its ID
+```
+
+`prepare()` can resolve tenant/input scope before initial persistence. `save()`
+during `handle()` saves application state under the current execution fence.
+Use `checkpoint(callback)` when concurrent requests merge counters or JSON.
+The application authorizes launching/reading runs; a tenant ID alone is not an
+access-control policy. `input`, tenant and subject are immutable during execution.
+
+The runtime records waiting/running/completed/failed/blocked/cancelled outcomes,
+start/end/heartbeat timestamps, and separate attempts. Queue retries reconstruct
+the same run, retaining its checkpoint. A recovered expired reservation marks
+the preceding attempt interrupted. The queue still delivers at least once:
+external effects must be idempotent and jobs must check ownership before
+publishing them. `assertActive()` also detects an operator-set
+`cancelRequestedAt`; `signal` allows requests to cooperate with cancellation.
+`block(reason)` retains a stopped outcome without retrying a known bad condition.
+
+Run-specific concurrency, request budgets, exception thresholds and cost
+accounting belong to the concrete job. Keep JSON checkpoints bounded (1 MB) and
+use `log(message, data, level)` for bounded events (16 KB each). Omit secrets and
+provider bodies. Historical evidence needs an application retention policy.
+
+With the database queue, reserving the dispatch generation and inserting its
+reference are atomic. The run is saved first, so a failed enqueue remains
+inspectable. An external broker cannot share that SQL transaction; adopting one
+requires a transactional outbox/reconciler for the publish gap. Tracked terminal
+failures remain in `job_runs` instead of adding failed queue-delivery rows.
+Ordinary `QueueableJob` classes and their existing failure behavior are unchanged.
+
+### Active run context
+
+`RunnableJob.current()` returns the run executing in the current asynchronous
+context. Dispatching a retained child automatically records its parent ID and
+checks that the parent still owns its execution. Cancellation is checked through
+the parent ancestry before further checkpoints and work. Applications own which
+job types support operator cancellation and any external-effect cleanup.
+
+A registered retained implementation also handles named dispatches and ordinary
+queue producers registered under the same job name. Producers must register the
+retained implementation before dispatch. The scheduler accepts either retained
+or lightweight queue contracts; it does not poll retained history.
+
+### Live worker presence
+
+Set `QueueOptions.onWorkerHeartbeat` to persist or publish `QueueWorkerSnapshot`
+records in the application's own monitoring store. Every worker has a unique ID,
+host/PID, exact queue selection and exclusions, current job identity, start time
+and heartbeat. Polling workers report idle/busy/stopping/stopped transitions and
+a heartbeat every ten seconds, including during long jobs. One-shot workers
+report a final stopped state. Sink failures do not fail jobs; stale heartbeats
+must be treated as unavailable rather than counted as idle capacity.
+The sink is optional, coalesces pending updates, and never receives job payloads.
+Applications own access control, expiry and retention. Counts across overlapping
+queue selections are shared capacity, not additional worker processes.
+
+App queues report listener/hook exceptions and job identity through `app.log`;
+`onLifecycleError` overrides this. Standalone queues use stderr. Committed queue
+transitions remain unchanged: essential settlement must be idempotent and recoverable.

package/src/queue/RunnableJob.ts

@@ -0,0 +1,272 @@
+import { AsyncLocalStorage } from 'node:async_hooks';
+import { randomUUID } from 'node:crypto';
+import { ActiveRecord } from '../db/ActiveRecord';
+import { JobRunAttempt } from './JobRunAttempt';
+import { JobRunEvent } from './JobRunEvent';
+import { JobRunStoppedError } from './JobRunStoppedError';
+import { QueueTerminalError } from './QueueTerminalError';
+import { isQueueRetryLaterError } from './QueueRetryLaterError';
+import type { Queue } from './Queue';
+import type { DispatchOptions, QueueJobId, QueueableJobContext, QueueableJobContract, QueueableJobFailureContext, QueueableJobRetryContext, SerializedQueueableJob } from './contracts';
+
+/** Persisted lifecycle values; waiting includes scheduled retries, blocked requires an explicit new run. */
+export const JOB_RUN_STATUSES = ['waiting', 'running', 'blocked', 'completed', 'failed', 'cancelled'] as const;
+
+/**
+ * A retained run with application-owned input, state and behaviour. Register only
+ * this base model in migrations; concrete job classes share its table. Jobs must
+ * use checkpoint/save for durable work and make external effects idempotent.
+ * Tenant and subject identities are immutable; applications authorize launches
+ * and reads. Queue serialization contains only the saved run ID and generation.
+ */
+export class RunnableJob extends ActiveRecord.define({
+	table: 'job_runs',
+	fields: field => ({
+		id: field.ulid(),
+		jobType: field.string({ column: 'job_type', required: true, length: 200, index: true }),
+		tenantId: field.string({ column: 'tenant_id', required: true, length: 100, indexes: [{ name: 'job_runs_tenant_history', columns: ['tenant_id', 'id'] }] }),
+		subjectId: field.string({ column: 'subject_id', length: 100, indexes: [{ name: 'job_runs_subject_history', columns: ['subject_id', 'id'] }] }),
+		actorId: field.string({ column: 'actor_id', length: 100 }),
+		parentRunId: field.string({ column: 'parent_run_id', length: 26, index: true }),
+		trigger: field.string({ required: true, default: 'manual', length: 40 }),
+		status: field.choice({ choices: [...JOB_RUN_STATUSES], required: true, default: 'waiting', index: true }),
+		input: field.json<Record<string, unknown>>({ required: true, default: {}, comment: 'Immutable application inputs captured when the run is created.' }),
+		state: field.json<Record<string, unknown>>({ required: true, default: {}, comment: 'Mutable application checkpoint for progress, diagnosis and explicit recovery; may contain private data.' }),
+		result: field.json<Record<string, unknown>>({ comment: 'Application outcome or result summary, separate from mutable progress state.' }),
+		error: field.text(),
+		stage: field.string({ length: 200, comment: 'Job-owned current phase; lifecycle outcome is stored in status.' }),
+		completed: field.integer({ required: true, default: 0, comment: 'Job-owned processed unit count, e.g. pages scanned or 1 for one operation; not a completion flag.' }),
+		total: field.integer({ comment: 'Planned units in the same measure as completed; null when unknown.' }),
+		failures: field.integer({ required: true, default: 0, comment: 'Job-owned failed unit count; units may also be counted as completed. Job failure is recorded in status.' }),
+		attemptCount: field.integer({ column: 'attempt_count', required: true, default: 0 }),
+		deliveryKey: field.string({ column: 'delivery_key', length: 36 }),
+		leaseToken: field.string({ column: 'lease_token', length: 36 }),
+		cancelRequestedAt: field.timestamp({ column: 'cancel_requested_at' }),
+		queuedAt: field.timestamp({ column: 'queued_at' }),
+		startedAt: field.timestamp({ column: 'started_at' }),
+		heartbeatAt: field.timestamp({ column: 'heartbeat_at' }),
+		completedAt: field.timestamp({ column: 'completed_at' }),
+		createdAt: field.timestamp({ column: 'created_at', auto: 'create' }),
+		updatedAt: field.timestamp({ column: 'updated_at', auto: 'update' }),
+	}),
+}) implements QueueableJobContract {
+	/** Stable name may be overridden to preserve queued identities across class renames. */
+	static jobName?: string;
+	static #active = new AsyncLocalStorage<RunnableJob>();
+
+	/** Returns the active run for child dispatch correlation within this asynchronous execution only. */
+	static current(): RunnableJob | undefined { return RunnableJob.#active.getStore(); }
+	#context: QueueableJobContext | null = null;
+	#token: string | null = null;
+	#updates: Promise<unknown> = Promise.resolve();
+	#stop = new AbortController();
+	#publishing = false;
+
+	/** Internal dispatch guard; a saved delivery cannot be enqueued twice through the public API. */
+	get publishingDelivery(): boolean { return this.#publishing; }
+
+	/** Cooperative abort signal. Application requests should combine it with their own deadline. */
+	get signal(): AbortSignal { return this.#stop.signal; }
+
+	/** Loads the authoritative run through its concrete registered class, rejecting mismatched deliveries. */
+	static async fromJSON<T extends typeof RunnableJob>(this: T, data: Record<string, unknown>): Promise<InstanceType<T>> {
+		if (typeof data.runId !== 'string' || typeof data.deliveryKey !== 'string') throw new Error('Invalid runnable job reference.');
+		const run = await this.findOrFail(data.runId);
+		if (run.jobType !== (this.jobName ?? this.name) || run.deliveryKey !== data.deliveryKey) throw new Error('Runnable job identity does not match its delivery.');
+		return run as InstanceType<T>;
+	}
+
+	/** Saves a new run before dispatch. During execution, saves only application-owned fields under the attempt fence. */
+	override async save(): Promise<this> {
+		if (!this.isPersisted()) {
+			if (this.constructor === RunnableJob) throw new Error('Create a concrete RunnableJob subclass.');
+			this.jobType = (this.constructor as typeof RunnableJob).jobName ?? this.constructor.name;
+			return super.save();
+		}
+		if (!this.#token) throw new Error('Existing run records may only be saved by their owning execution.');
+		const values = structuredClone({ state: this.state, result: this.result, stage: this.stage, completed: this.completed, total: this.total, failures: this.failures });
+		await this.#serializeUpdate(() => this.#write(values));
+		return this;
+	}
+
+	/** Serializes concurrent progress callbacks so state merges cannot lose another completed item's update. */
+	async checkpoint(update: (run: this) => void): Promise<void> {
+		await this.#serializeUpdate(async () => {
+			await this.assertActive();
+			update(this);
+			await this.#write({ state: this.state, result: this.result, stage: this.stage, completed: this.completed, total: this.total, failures: this.failures });
+		});
+	}
+
+	/** Checks delivery ownership and operator cancellation before launching or publishing another unit of work. */
+	async assertActive(): Promise<void> {
+		this.signal.throwIfAborted();
+		await this.#context?.assertOwnership?.();
+		const row = await RunnableJob.where({ id: this.id, leaseToken: this.#token, status: 'running' }).first();
+		if (!this.#token || !row) throw new Error('Runnable job execution ownership was lost.');
+		let cancellation = Boolean(row.cancelRequestedAt);
+		let parentId = row.parentRunId;
+		const visited = new Set<string>();
+		while (!cancellation && parentId) {
+			if (visited.has(parentId) || visited.size >= 32) throw new Error('Invalid run ancestry.');
+			visited.add(parentId);
+			const parent = await RunnableJob.find(parentId);
+			cancellation = Boolean(parent?.cancelRequestedAt);
+			parentId = parent?.parentRunId ?? null;
+		}
+		if (cancellation) {
+			const reason = new JobRunStoppedError('cancelled', 'Cancelled by the operator.');
+			this.#stop.abort(reason);
+			throw reason;
+		}
+	}
+
+	/** Appends bounded structured diagnostics; callers must omit credentials and private provider payloads. */
+	async log(message: string, data: Record<string, unknown> = {}, level: 'info' | 'warning' | 'error' = 'info'): Promise<void> {
+		await this.assertActive();
+		if (JSON.stringify(data).length > 16000) throw new Error('Job event data exceeds 16,000 characters.');
+		await JobRunEvent.create({ runId: this.id, tenantId: this.tenantId, attempt: this.attemptCount, message: message.slice(0, 1024), data, level }).save();
+	}
+
+	/** Stops launching further work until an operator creates a resumed run using the saved checkpoint. */
+	block(reason: string): never { throw new JobRunStoppedError('blocked', reason); }
+
+	/** Resolves application-owned input/scope before first persistence. Override in a tenant-aware application base class. */
+	async prepare(): Promise<void> {}
+
+	/** Saves a run and its dispatch generation before delivery. Failed dispatches remain visible in history. */
+	async enqueue(queue: Queue, options: DispatchOptions = {}): Promise<QueueJobId> {
+		if (this.deliveryKey) throw new Error('This run has already been dispatched.');
+		if (!this.isPersisted()) { await this.prepare(); await this.save(); }
+		const key = randomUUID();
+		let reserved = false;
+		/** Reserves the delivery generation and publishes its small queue reference. */
+		const publish = async (): Promise<QueueJobId> => {
+			const changed = await RunnableJob.where({ id: this.id, status: 'waiting' }).whereNull('deliveryKey').patch({ deliveryKey: key, queuedAt: new Date() });
+			if (changed !== 1) throw new Error('This run has already been dispatched.');
+			reserved = true;
+			this.deliveryKey = key;
+			this.#publishing = true;
+			try { return await queue.dispatch(this, options); } finally { this.#publishing = false; }
+		};
+		try {
+			return queue.driverName === 'database'
+				? await RunnableJob.getDb().transaction(transaction => ActiveRecord.withDb(transaction, publish))
+				: await publish();
+		}
+		catch (error) {
+			if (!reserved) throw error;
+			const failedDispatch = RunnableJob.where({ id: this.id, status: 'waiting' });
+			if (queue.driverName === 'database') failedDispatch.whereNull('deliveryKey'); else failedDispatch.where('deliveryKey', key);
+			await failedDispatch.patch({ status: 'failed', error: `Dispatch failed: ${error instanceof Error ? error.message : String(error)}`, completedAt: new Date() });
+			throw error;
+		}
+	}
+
+	/** Produces a small immutable queue reference; never copies mutable model JSON into delivery storage. */
+	serialize(): SerializedQueueableJob {
+		if (!this.isPersisted() || !this.deliveryKey) throw new Error('Call run.enqueue(queue) before queueing a runnable record.');
+		return { job: this.jobType!, displayName: this.jobType!, data: { runId: this.id, deliveryKey: this.deliveryKey } };
+	}
+
+	/** Concrete application jobs implement their own bounded work, checkpoints and result publication. */
+	async handle(_context: QueueableJobContext): Promise<void> { throw new Error('RunnableJob.handle must be implemented.'); }
+
+	/** Lifecycle is saved before delivery transitions; ordinary queue retry hooks are intentionally empty. */
+	async onRetry(_context: QueueableJobRetryContext): Promise<void> {}
+	/** Lifecycle is saved before delivery transitions, including terminal failure. */
+	async onFinalFailure(context: QueueableJobFailureContext): Promise<void> {
+		// A deadline can pass between handler failure and the driver's final disposition.
+		const error = context.error instanceof Error ? context.error.message : String(context.error);
+		await RunnableJob.getDb().transaction(transaction => ActiveRecord.withDb(transaction, async () => {
+			const changed = await RunnableJob.where({ id: this.id, status: 'waiting', attemptCount: this.attemptCount }).patch({ status: 'failed', error, completedAt: new Date() });
+			if (changed) {
+				await JobRunAttempt.where({ runId: this.id, attempt: this.attemptCount, status: 'retrying' }).patch({ status: 'failed', error });
+				await JobRunEvent.create({ runId: this.id, tenantId: this.tenantId, attempt: this.attemptCount, level: 'error', message: error.slice(0, 1024), data: { status: 'failed', queue: context.job.queue, deliveryId: context.job.id, jobUuid: context.job.payload.uuid, maxTries: context.job.payload.maxTries } }).save();
+				this.status = 'failed';
+			}
+		}));
+	}
+
+	/** Runs one fenced attempt. Expired delivery recovery preserves checkpoints and records the interrupted attempt. */
+	async execute(context: QueueableJobContext): Promise<void> {
+		await context.assertOwnership?.();
+		this.#context = context;
+		this.#updates = Promise.resolve();
+		const token = randomUUID();
+		const claimed = await RunnableJob.getDb().transaction(transaction => ActiveRecord.withDb(transaction, async () => {
+			const raw = await RunnableJob.where('id', this.id).toKnex().forUpdate().first();
+			if (!raw) throw new Error('Run no longer exists.');
+			const current = RunnableJob.fromDb(raw);
+			if (current.deliveryKey !== this.deliveryKey) throw new Error('Run delivery was replaced.');
+			if (!['waiting', 'running'].includes(current.status ?? '')) return false;
+			if (current.status === 'running') await JobRunAttempt.where({ runId: this.id, status: 'running' }).patch({ status: 'interrupted', completedAt: new Date(), error: 'Worker reservation expired before completion.' });
+			const now = new Date();
+			const values = { status: 'running' as const, leaseToken: token, attemptCount: (current.attemptCount ?? 0) + 1, startedAt: current.startedAt ?? now, heartbeatAt: now, completedAt: null, error: null };
+			await RunnableJob.where('id', this.id).patch(values);
+			this.assign({ ...current.toJSON(), ...values });
+			await JobRunAttempt.create({ runId: this.id, tenantId: this.tenantId, attempt: this.attemptCount, deliveryId: String(context.job.id), status: 'running', startedAt: now }).save();
+			return true;
+		}));
+		if (!claimed) return;
+		this.#token = token;
+		this.#stop = new AbortController();
+		let heartbeat: Promise<void> = Promise.resolve();
+		const timer = setInterval(() => {
+			heartbeat = heartbeat.then(async () => {
+				await this.assertActive();
+				await RunnableJob.where({ id: this.id, leaseToken: token }).patch({ heartbeatAt: new Date() });
+			}).catch(error => { this.#stop.abort(error); });
+		}, 10000);
+		try {
+			await this.log('Attempt started');
+			await RunnableJob.#active.run(this, () => this.handle(context));
+			await this.#updates;
+			await this.assertActive();
+			await this.#finish('completed', null);
+		} catch (error) {
+			await this.#updates.catch(() => undefined);
+			await context.assertOwnership?.();
+			const stopped = error instanceof JobRunStoppedError;
+			const retry = context.willRetry ? context.willRetry(error) : isQueueRetryLaterError(error) || !(error instanceof QueueTerminalError) && context.job.attempts < context.job.payload.maxTries;
+			const status = stopped ? error.status : retry ? 'waiting' : 'failed';
+			await this.#finish(status, error instanceof Error ? error.message : String(error));
+			if (!stopped) throw error;
+		} finally {
+			clearInterval(timer);
+			await heartbeat;
+			this.#context = null;
+			this.#token = null;
+		}
+	}
+
+	/** Writes application-owned state with a database fence; unrelated runtime fields cannot be overwritten. */
+	async #write(values: Record<string, unknown>): Promise<void> {
+		await this.assertActive();
+		if (JSON.stringify(values).length > 1000000) throw new Error('Run checkpoint exceeds one megabyte. Store large evidence separately.');
+		const count = await RunnableJob.where({ id: this.id, leaseToken: this.#token, status: 'running' }).whereNull('cancelRequestedAt').patch({ ...values, heartbeatAt: new Date(), updatedAt: new Date() });
+		if (count !== 1) {
+			await this.assertActive();
+			throw new Error('Runnable job checkpoint ownership was lost.');
+		}
+	}
+
+	/** Records terminal/retry state, attempt outcome and timeline in the same transaction. */
+	async #finish(status: typeof JOB_RUN_STATUSES[number], error: string | null): Promise<void> {
+		await RunnableJob.getDb().transaction(transaction => ActiveRecord.withDb(transaction, async () => {
+			const now = new Date();
+			const changed = await RunnableJob.where({ id: this.id, leaseToken: this.#token, status: 'running' }).patch({ status, error, completedAt: status === 'waiting' ? null : now, heartbeatAt: now, leaseToken: null, updatedAt: now });
+			if (changed !== 1) throw new Error('Runnable job completion ownership was lost.');
+			await JobRunAttempt.where({ runId: this.id, attempt: this.attemptCount, status: 'running' }).patch({ status: status === 'waiting' ? 'retrying' : status, error, completedAt: now });
+			await JobRunEvent.create({ runId: this.id, tenantId: this.tenantId, attempt: this.attemptCount, level: status === 'failed' ? 'error' : error ? 'warning' : 'info', message: error?.slice(0, 1024) ?? 'Run completed', data: { status, queue: this.#context?.job.queue, deliveryId: this.#context?.job.id, jobUuid: this.#context?.job.payload.uuid, maxTries: this.#context?.job.payload.maxTries } }).save();
+			this.status = status;
+		}));
+	}
+
+	/** Queues state writes on this instance while allowing callers to observe their own failures. */
+	#serializeUpdate(action: () => Promise<void>): Promise<void> {
+		const next = this.#updates.then(action);
+		this.#updates = next;
+		return next;
+	}
+}

package/src/queue/console.ts

@@ -3,6 +3,7 @@ import { parseJsonRecord, unknownErrorMessage } from '@db3.ai/pure';
 import type { Logger } from '../logging';
 import type * as queue from './contracts';
 import type { Queue } from './Queue';
+import { QueueWorker } from './QueueWorker';
 
 interface QueueConsoleLogOptions {
 	verbose?: boolean;
@@ -37,8 +38,11 @@ export interface QueueConsoleCommand {
 	run(context: QueueConsoleCommandContext): void | Promise<void>;
 }
 
+/** Application bootstrap, named worker selections and extra commands shared by queue CLI invocations. */
 export interface QueueConsoleOptions {
 	app: () => QueueConsoleApp;
+	/** Checked-in worker selections selected with --pool=<name>; credentials remain in the application environment. */
+	workers?: Readonly<Record<string, queue.QueueSelection>>;
 	bootstrap?: () => void | Promise<void>;
 	commands?: QueueConsoleCommand[];
 	helpText?: string;
@@ -283,14 +287,13 @@ async function bootstrap(options: QueueConsoleOptions): Promise<void> {
 	await options.bootstrap?.();
 }
 
+/** Runs a configured worker selection, with identical admission for one-shot and polling execution. */
 async function workQueue(
 	options: QueueConsoleOptions,
 	parsed: ParsedQueueConsoleArgs,
 ): Promise<void> {
-	const queue = queueConsoleOptionString(parsed, 'queue')
-		|| parsed.args[0]
-		|| process.env.QUEUE_NAME
-		|| 'default';
+	const selection = queueConsoleSelection(parsed, options.workers);
+	const queue = selection.queues === '*' ? '*' : selection.queues?.join(', ') ?? 'default';
 	const once = queueConsoleOptionBoolean(parsed, 'once');
 	const verbose = queueConsoleVerbose(parsed);
 
@@ -301,30 +304,32 @@ async function workQueue(
 	const app = options.app();
 
 	if (once) {
-		if (verbose) {
-			console.log(`[queue] Checking "${queue}" once.`);
-		}
+		try {
+			if (verbose) {
+				console.log(`[queue] Checking "${queue}" once.`);
+			}
 
-		const result = await app.queue.workNextJob(queue, {
-			onClaimed: job => logQueueClaimed(job, {
-				verbose,
-			}),
-		});
+			const result = await new QueueWorker(app.queue, selection).workOnce({
+				onClaimed: job => logQueueClaimed(job, {
+					verbose,
+				}),
+			});
+
+			if (!result) {
+				console.log(`[queue] No jobs available on "${queue}".`);
+				return;
+			}
 
-		if (!result) {
-			console.log(`[queue] No jobs available on "${queue}".`);
+			logQueueProcessResult(result, {
+				verbose,
+			});
+		} finally {
 			await app.close();
-			return;
 		}
-
-		logQueueProcessResult(result, {
-			verbose,
-		});
-		await app.close();
 		return;
 	}
 
-	const worker = app.queue.startWorker(queue, {
+	const worker = app.queue.startWorker(selection, {
 		force: true,
 		logger: app.log?.child({
 			component: 'queue-worker',
@@ -357,6 +362,31 @@ async function workQueue(
 	});
 }
 
+/**
+ * Resolves one worker selection from explicit CLI flags or a checked-in named pool.
+ * Pool admission cannot be silently broadened by combining it with queue flags.
+ */
+export function queueConsoleSelection(parsed: ParsedQueueConsoleArgs, workers: QueueConsoleOptions['workers'] = {}): queue.QueueSelection {
+	for (const flag of ['pool', 'queues', 'queue', 'exclude-queues']) {
+		if (Object.hasOwn(parsed.options, flag) && queueConsoleOptionString(parsed, flag) === undefined) throw new Error(`--${flag} requires a non-empty value.`);
+	}
+	const pool = queueConsoleOptionString(parsed, 'pool');
+	const queues = queueConsoleOptionString(parsed, 'queues');
+	const single = queueConsoleOptionString(parsed, 'queue') || parsed.args[0];
+	const excluded = queueConsoleOptionString(parsed, 'exclude-queues');
+	if (pool) {
+		if (queues !== undefined || single !== undefined || excluded !== undefined) throw new Error('--pool cannot be combined with queue selection flags.');
+		if (!Object.hasOwn(workers, pool)) throw new Error(`Unknown worker pool "${pool}".`);
+		return workers[pool]!;
+	}
+	if (queues !== undefined && single !== undefined) throw new Error('Use --queues or --queue, not both.');
+	const names = queues ?? single ?? process.env.QUEUE_NAME ?? 'default';
+	return {
+		queues: names === '*' ? '*' : names.split(',').map(name => name.trim()),
+		excludeQueues: excluded === undefined ? [] : excluded.split(',').map(name => name.trim()),
+	};
+}
+
 async function dispatchQueueJob(
 	app: QueueConsoleApp,
 	parsed: ParsedQueueConsoleArgs,
@@ -429,6 +459,9 @@ function defaultHelpText(): string {
 
 Options:
   --queue=<name>       Queue name, defaults to "default".
+  --queues=<names|*>  Worker queue names, comma-separated; quote '*'.
+  --exclude-queues=<names>  Exact queue names this worker cannot claim.
+  --pool=<name>        Checked-in worker selection; cannot combine with queue flags.
   --delay=<seconds>    Delay before the job is available.
   --tries=<count>      Maximum attempts before jobs_failed.
   --once               Process one available job, then exit.

package/src/queue/contracts/QueueDriver.ts

@@ -50,9 +50,18 @@ export interface QueueFailedJob {
  * recording. Queue owns handler execution and retry decisions.
  */
 export interface QueueDriver {
+	/** Inspects pending, delayed, reserved and chained work for maintenance; callers must quiesce producers first. */
+	hasPendingJobs?(jobPrefix: string): Promise<boolean>;
 	/** Driver name stored on failed job records and shown in diagnostics. */
 	readonly name: QueueDriverName | string;
 
+	/**
+	 * Discovers names with pending, delayed or reserved work in this driver's namespace.
+	 * Required for wildcard workers; explicit selections do not need discovery.
+	 * Returned names may disappear before a claim, which must still be atomic.
+	 */
+	queueNames?(): Promise<string[]>;
+
 	/**
 	 * Persists a job so a worker can claim it later.
 	 *

package/src/queue/contracts/QueueJobRunner.ts

@@ -0,0 +1,4 @@
+/** Executes a complete claimed-job attempt, including retries and hooks, inside its owner's lifecycle gate. */
+export interface QueueJobRunner {
+	<T>(operation: () => Promise<T>): Promise<T>;
+}

package/src/queue/contracts/QueuePayload.ts

@@ -89,7 +89,7 @@ export interface DispatchOptions {
 	/**
 	 * Named queue/channel the job should be pushed onto.
 	 *
-	 * Workers process one named queue at a time, so this lets callers route
+	 * Workers select eligible named queues, so this lets callers route
 	 * different classes of work to different worker pools. When omitted, the
 	 * queue service uses its configured default queue name.
 	 */

package/src/queue/contracts/QueueSelection.ts

@@ -0,0 +1,11 @@
+/**
+ * Named queues a worker may consume. Selection never changes a job's queue.
+ * Wildcards discover queues within the configured driver connection/namespace;
+ * exclusions always win. Names are exact, with no glob or job-class matching.
+ */
+export interface QueueSelection {
+	/** Exact queue names, or all currently discoverable names. Defaults to the configured default queue. */
+	queues?: readonly string[] | '*';
+	/** Exact names that this worker must never claim, including when using '*'. */
+	excludeQueues?: readonly string[];
+}