@db3.ai/app 0.1.0-beta.1 → 0.1.0

package/src/scheduler/SchedulerCheckpointRecord.ts

@@ -0,0 +1,51 @@
+import { ActiveRecord } from '../db';
+import type { SchedulerCheckpoint } from './contracts';
+
+/** Durable scheduler restart progress; register this model in the application's migration registry. */
+export class SchedulerCheckpointRecord extends ActiveRecord.define({
+	table: 'scheduler_checkpoints',
+	fields: field => ({
+		id: field.ulid(),
+		name: field.string({ required: true, length: 120, unique: true }),
+		value: field.json<{ evaluatedFor: string }>({ required: true }),
+		updatedAt: field.timestamp({ column: 'updated_at', auto: 'update' }),
+	}),
+}) {
+	/** Returns restart persistence for one stable scheduler name; concurrent saves never regress progress. */
+	static checkpoint(name = 'default'): SchedulerCheckpoint {
+		return {
+			/** Establishes coverage before the first dispatch without resetting an existing scheduler. */
+			async load(firstMinute: Date): Promise<Date> {
+				let record = await SchedulerCheckpointRecord.where('name', name).first();
+				if (!record) {
+					try {
+						record = await SchedulerCheckpointRecord.create({ name, value: { evaluatedFor: new Date(firstMinute.getTime() - 60_000).toISOString() } }).save();
+					} catch (error) {
+						record = await SchedulerCheckpointRecord.where('name', name).first();
+						if (!record) throw error;
+					}
+				}
+				return checkpointMinute(record.value!.evaluatedFor);
+			},
+			/** Advances only a valid, initialized UTC-minute cursor under a database row lock. */
+			async save(evaluatedFor: Date): Promise<void> {
+				const minute = checkpointMinute(evaluatedFor.toISOString());
+				await SchedulerCheckpointRecord.getDb().transaction(transaction => ActiveRecord.withDb(transaction, async () => {
+					const row = await SchedulerCheckpointRecord.where('name', name).toKnex().forUpdate().first();
+					if (!row) throw new Error('Scheduler checkpoint must be initialized before advancing it.');
+					const record = SchedulerCheckpointRecord.fromDb(row);
+					if (checkpointMinute(record.value!.evaluatedFor).getTime() >= minute.getTime()) return;
+					record.value = { evaluatedFor: minute.toISOString() };
+					await record.save();
+				}));
+			},
+		};
+	}
+}
+
+/** Rejects corrupt progress rather than silently skipping an unknown scheduling window. */
+function checkpointMinute(value: unknown): Date {
+	const minute = new Date(typeof value === 'string' ? value : NaN);
+	if (!Number.isFinite(minute.getTime()) || minute.getTime() % 60_000 !== 0) throw new Error('Scheduler checkpoint requires a valid UTC minute.');
+	return minute;
+}

package/src/scheduler/SchedulerWorker.ts

@@ -1,7 +1,11 @@
 import type * as scheduler from './contracts';
 import { Scheduler } from './Scheduler';
+import { utcMinute } from './scheduleTime';
 
 const DEFAULT_SLOW_TICK_WARNING_MS = 50_000;
+const MINUTE_MS = 60_000;
+const MAX_MINUTES_PER_BATCH = 60;
+const FAILED_TICK_RETRY_MS = 1_000;
 
 /**
  * Long-running minute-aligned process adapter around Scheduler.runDue().
@@ -10,6 +14,9 @@ export class SchedulerWorker {
 	#running = false;
 	#abortController: AbortController | null = null;
 	readonly #now: () => Date;
+	#nextMinute: number | null = null;
+	readonly #onTick?: scheduler.SchedulerWorkerOptions['onTick'];
+	readonly #checkpoint?: scheduler.SchedulerCheckpoint;
 	readonly #sleep: scheduler.SchedulerSleep;
 	readonly #logger: scheduler.SchedulerLogger;
 	readonly #slowTickWarningMs: number;
@@ -24,6 +31,8 @@ export class SchedulerWorker {
 		private readonly schedulerService: Scheduler,
 		options: scheduler.SchedulerWorkerOptions = {},
 	) {
+		this.#onTick = options.onTick;
+		this.#checkpoint = options.checkpoint;
 		this.#now = options.now ?? (() => new Date());
 		this.#sleep = options.sleep ?? schedulerSleep;
 		this.#logger = options.logger ?? consoleSchedulerLogger();
@@ -32,7 +41,9 @@ export class SchedulerWorker {
 	}
 
 	/**
-	 * Runs immediately, then evaluates at each subsequent wall-clock minute.
+	 * Evaluates every uncovered minute in order, including time spent dispatching
+	 * earlier batches. Catch-up yields after sixty minutes without discarding work.
+	 * A durable checkpoint resumes coverage across replacement processes.
 	 *
 	 * @throws Error when the same worker instance is started twice.
 	 */
@@ -45,15 +56,28 @@ export class SchedulerWorker {
 		this.#abortController = new AbortController();
 
 		try {
+			if (this.#nextMinute === null) {
+				const firstMinute = utcMinute(this.#now());
+				const previous = await this.#checkpoint?.load(firstMinute);
+				this.#nextMinute = previous ? utcMinute(previous).getTime() + MINUTE_MS : firstMinute.getTime();
+			}
+			let batchSize = 0;
 			while (this.#running) {
-				await this.runTick();
-
-				if (!this.#running) break;
-
-				await this.#sleep(
-					millisecondsUntilNextMinute(this.#now()),
-					this.#abortController.signal,
-				);
+				const now = this.#now();
+				const currentMinute = utcMinute(now).getTime();
+				if (this.#nextMinute <= currentMinute) {
+					const completed = await this.#runTick(new Date(this.#nextMinute));
+					if (completed) this.#nextMinute += MINUTE_MS;
+					if (!this.#running) break;
+					if (!completed || ++batchSize >= MAX_MINUTES_PER_BATCH) {
+						await this.#sleep(completed ? 1 : FAILED_TICK_RETRY_MS, this.#abortController.signal);
+						batchSize = 0;
+					}
+					continue;
+				}
+				batchSize = 0;
+				// An early timer or backward clock step must not skip or repeat a minute.
+				await this.#sleep(Math.min(MINUTE_MS, Math.max(1, this.#nextMinute - now.getTime())), this.#abortController.signal);
 			}
 		} finally {
 			this.#running = false;
@@ -70,13 +94,22 @@ export class SchedulerWorker {
 	}
 
 	/**
-	 * Evaluates one scheduler minute and reports failures without ending work.
+	 * Evaluates one fixed minute, preserving progress only after the whole batch.
+	 * Recorded event failures remain visible and do not prevent later minutes;
+	 * incomplete evaluations and checkpoint failures retry this same minute.
+	 *
+	 * @param evaluatedFor - Original scheduled minute, independent of dispatch duration.
+	 * @returns Whether the complete minute was evaluated and its checkpoint saved.
 	 */
-	private async runTick(): Promise<void> {
+	async #runTick(evaluatedFor: Date): Promise<boolean> {
 		const startedAt = performance.now();
+		let completed = false;
 
 		try {
-			const result = await this.schedulerService.runDue(this.#now());
+			const result = await this.schedulerService.runDue(evaluatedFor);
+			if (result.failures.length === 0) await this.#onTick?.(result);
+			await this.#checkpoint?.save(evaluatedFor);
+			completed = true;
 
 			if (result.due > 0 || result.failures.length > 0) {
 				this.#logger.info(
@@ -90,7 +123,7 @@ export class SchedulerWorker {
 				);
 			}
 		} catch (error) {
-			this.#logger.error('[scheduler] Tick failed; the worker will continue.', error);
+			this.#logger.error(`[scheduler] Evaluation for ${evaluatedFor.toISOString()} did not finish; retrying the same minute.`, error);
 		}
 
 		const durationMs = performance.now() - startedAt;
@@ -100,6 +133,7 @@ export class SchedulerWorker {
 				`[scheduler] Tick took ${Math.round(durationMs)}ms; inline calls should remain short.`,
 			);
 		}
+		return completed;
 	}
 }
 
@@ -126,14 +160,15 @@ async function schedulerSleep(
 	signal: AbortSignal,
 ): Promise<void> {
 	await new Promise<void>(resolve => {
-		const timeout = setTimeout(resolve, delayMs);
-
-		signal.addEventListener('abort', () => {
+		/** Clears the timer and its abort subscription on either completion path. */
+		const finish = (): void => {
 			clearTimeout(timeout);
+			signal.removeEventListener('abort', finish);
 			resolve();
-		}, {
-			once: true,
-		});
+		};
+		const timeout = setTimeout(finish, delayMs);
+		signal.addEventListener('abort', finish, { once: true });
+		if (signal.aborted) finish();
 	});
 }
 

package/src/scheduler/console.ts

@@ -43,7 +43,7 @@ export async function runSchedulerConsole(
 			await runWithApp(options, app => {
 				for (const definition of app.scheduler.definitions()) {
 					console.log(
-						`${definition.name}\t${definition.kind}\tdaily at ${definition.frequency.time}\t${definition.timezone}${definition.jobName ? `\t${definition.jobName}` : ''}`,
+						`${definition.name}\t${definition.kind}\t${definition.frequency.type === 'minute' ? 'every minute' : definition.frequency.type === 'hourly' ? 'hourly (UTC)' : `daily at ${definition.frequency.time}`}\t${definition.timezone}${definition.jobName ? `\t${definition.jobName}` : ''}`,
 					);
 				}
 			});
@@ -153,6 +153,8 @@ async function workScheduler(
 	app.scheduler.definitions();
 
 	const worker = new SchedulerWorker(app.scheduler, {
+		onTick: options.onTick,
+		checkpoint: options.checkpoint,
 		logger: app.log ? schedulerLogger(app.log) : undefined,
 	});
 	let shuttingDown = false;

package/src/scheduler/contracts/Schedule.ts

@@ -0,0 +1,22 @@
+import type { ScheduledJob } from '../ScheduledJob';
+import type { ScheduledCall } from '../ScheduledCall';
+import type { ScheduledJobSource, ScheduledCallHandler, ScheduledTaskDefinition } from './Scheduler';
+
+/** Fluent declaration surface shared by the host scheduler and app-owned schedule.ts files. */
+export interface Schedule {
+	/** Declares queued work; workers execute the expensive operation. */
+	job(source: ScheduledJobSource): ScheduledJob;
+	/** Declares a short inline operation executed by the scheduler. */
+	call(handler: ScheduledCallHandler): ScheduledCall;
+}
+
+/** Ownership handle for one group's registrations. Closing removes only that group. */
+export interface ScheduleRegistration {
+	/** Browser-safe definitions, including the owning app identifier. */
+	definitions: ScheduledTaskDefinition[];
+	/** Idempotently removes the group's runtime definitions. Durable history remains. */
+	close(): void;
+}
+
+/** Wraps one complete occurrence, including claiming and dispatch, in the owner's lifecycle gate. */
+export type ScheduleRunner = (operation: () => Promise<void>) => Promise<void>;

package/src/scheduler/contracts/Scheduler.ts

@@ -1,4 +1,4 @@
-import type { QueueableJob } from '../../queue';
+import type { QueueableJobContract as QueueableJob } from '../../queue';
 
 /**
  * Supported scheduled task kinds.
@@ -18,25 +18,40 @@ export type ScheduledOccurrenceStatus =
 	| 'failed';
 
 /**
- * Daily frequency supported by the initial scheduler implementation.
+ * Daily frequency interpreted in the configured timezone.
  */
 export interface DailyScheduleFrequency {
-	/** Frequency discriminator reserved for future schedule types. */
+	/** Frequency discriminator. */
 	type: 'daily';
 	/** Local wall-clock time in 24-hour HH:mm format. */
 	time: string;
 }
 
+/** Runs once for each evaluated UTC minute, including catch-up after downtime. */
+export interface MinuteScheduleFrequency {
+	type: 'minute';
+}
+
+/** Supported calendar frequencies; durable claims remain scoped to UTC minutes. */
+export type ScheduleFrequency = DailyScheduleFrequency | MinuteScheduleFrequency | HourlyScheduleFrequency;
+
+/** Runs at minute zero of each UTC hour, including catch-up; independent of local daylight-saving changes. */
+export interface HourlyScheduleFrequency {
+	type: 'hourly';
+}
+
 /**
  * Normalized public description of one registered schedule.
  */
 export interface ScheduledTaskDefinition {
+	/** App or other runtime group that owns this registration; absent for host declarations. */
+	owner?: string;
 	/** Stable task name used for deduplication and history. */
 	name: string;
 	/** Whether the task dispatches a job or executes an inline callback. */
 	kind: ScheduledTaskKind;
-	/** Daily frequency and local wall-clock time. */
-	frequency: DailyScheduleFrequency;
+	/** Minute/hour cadence or daily local wall-clock time. */
+	frequency: ScheduleFrequency;
 	/** IANA timezone used to interpret the frequency. */
 	timezone: string;
 	/** Durable queue job name when known without invoking a factory. */
@@ -61,7 +76,7 @@ export interface ScheduledJobClass {
 	 * @param data - JSON-safe queue payload.
 	 * @returns Rehydrated job instance.
 	 */
-	fromJSON(data: Record<string, unknown>): QueueableJob;
+	fromJSON(data: Record<string, unknown>): QueueableJob | Promise<QueueableJob>;
 }
 
 /**

package/src/scheduler/contracts/SchedulerCheckpoint.ts

@@ -0,0 +1,23 @@
+/**
+ * Application-owned durable progress for one registered scheduler.
+ * Share this checkpoint between replacement processes using the same definitions.
+ */
+export interface SchedulerCheckpoint {
+	/**
+	 * Loads the last fully evaluated UTC minute. On first use, atomically persist
+	 * and return the minute immediately before firstMinute, establishing the start
+	 * of coverage before any scheduled work is attempted.
+	 *
+	 * @param firstMinute - Current UTC minute at the worker's first startup.
+	 * @returns Last evaluated minute, or the newly established starting boundary.
+	 */
+	load(firstMinute: Date): Promise<Date>;
+	/**
+	 * Advances progress after all events in a minute have been considered.
+	 * Concurrent saves must never move the stored minute backwards. A rejected
+	 * save causes the worker to reconsider that minute using occurrence deduplication.
+	 *
+	 * @param evaluatedFor - UTC minute whose complete evaluation was recorded.
+	 */
+	save(evaluatedFor: Date): Promise<void>;
+}

package/src/scheduler/contracts/SchedulerConsole.ts

@@ -1,5 +1,6 @@
 import type { Scheduler } from '../Scheduler';
 import type { Logger } from '../../logging';
+import type { SchedulerWorkerOptions } from './SchedulerWorker';
 
 /**
  * Parsed scheduler command name, positional arguments, and long options.
@@ -29,6 +30,10 @@ export interface SchedulerConsoleApp {
  * Application hooks and help customisation for scheduler console commands.
  */
 export interface SchedulerConsoleOptions {
+	/** Called after a successful scheduler worker tick for application-owned health reporting. */
+	onTick?: SchedulerWorkerOptions['onTick'];
+	/** Durable worker progress, initialized only by scheduler:work. One-off commands leave it alone. */
+	checkpoint?: SchedulerWorkerOptions['checkpoint'];
 	/** Resolves the active application instance after bootstrap. */
 	app: () => SchedulerConsoleApp;
 	/** Prepares schema, queue jobs, and schedule definitions. */

package/src/scheduler/contracts/SchedulerWorker.ts

@@ -1,4 +1,5 @@
-import type { SchedulerLogger } from './Scheduler';
+import type { SchedulerLogger, SchedulerRunResult } from './Scheduler';
+import type { SchedulerCheckpoint } from './SchedulerCheckpoint';
 
 /**
  * Abort-aware sleep function used by the long-running scheduler worker.
@@ -12,6 +13,10 @@ export type SchedulerSleep = (
  * Runtime dependencies and logging overrides for SchedulerWorker.
  */
 export interface SchedulerWorkerOptions {
+	/** Durable coverage across process restarts; without it, coverage starts in the current process. */
+	checkpoint?: SchedulerCheckpoint;
+	/** Records a successful evaluation and its actual minute; hook failures retry that minute. */
+	onTick?: (result: SchedulerRunResult) => Promise<void>;
 	/** Clock used to align and evaluate scheduler ticks. */
 	now?: () => Date;
 	/** Abort-aware sleeper used between minute boundaries. */

package/src/scheduler/contracts/index.ts

@@ -1,3 +1,5 @@
 export * from './Scheduler';
 export * from './SchedulerConsole';
 export * from './SchedulerWorker';
+export * from './SchedulerCheckpoint';
+export * from './Schedule';

package/src/scheduler/examples/runSchedulerWindow.ts

@@ -0,0 +1,54 @@
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { createGeneratedTestDatabase } from '@db3.ai/app/db/test/db';
+import { FailedJob, QueuedJob } from '@db3.ai/app/queue';
+import { ScheduledOccurrence, SchedulerWorker } from '@db3.ai/app/scheduler';
+import { App } from '@db3.ai/app/server';
+import { WriteDailySummaryJob } from './WriteDailySummaryJob';
+
+/**
+ * Demonstrates elapsed-minute scheduling with a real SQL queue and a controlled clock.
+ * Crossing the first boundary during a heartbeat, then spending seventy seconds
+ * dispatching two jobs, must also cover the following minute. This isolated lab
+ * removes its test database and files and makes no external provider calls.
+ *
+ * @returns Original due minutes and final outcomes after queue processing.
+ */
+export async function runSchedulerWindow() {
+	const root = await mkdtemp(join(tmpdir(), 'db3-scheduler-window-'));
+	try {
+		const database = await createGeneratedTestDatabase('scheduler_window_guide');
+		const application = new App({ db: database.db, storage: { disks: { local: { driver: 'local', root } } } });
+		try {
+			await application.db.install(QueuedJob, FailedJob, ScheduledOccurrence);
+			application.scheduler.job(WriteDailySummaryJob).name('first-summary').dailyAt('09:00');
+			application.scheduler.job(WriteDailySummaryJob).name('second-summary').dailyAt('09:00');
+			application.scheduler.job(WriteDailySummaryJob).name('next-summary').dailyAt('09:01');
+			let clock = Date.parse('2026-01-01T08:59:59.999Z');
+			const unsubscribe = application.queue.events.subscribe(async event => {
+				if (event.action === 'dispatched') clock += 35_000;
+			});
+			const worker = new SchedulerWorker(application.scheduler, {
+				now: () => new Date(clock),
+				onTick: async result => {
+					clock += 3;
+					if (result.evaluatedFor.getTime() === Date.parse('2026-01-01T09:01:00Z')) worker.stop();
+				},
+			});
+			try { await worker.start(); } finally { worker.stop(); unsubscribe(); }
+			while (await application.queue.workNextJob()) { /* Drain this lab's three jobs. */ }
+			const occurrences = await ScheduledOccurrence.query().orderBy('scheduledFor').orderBy('name').all();
+			return occurrences.map(occurrence => ({ name: occurrence.name, scheduledFor: occurrence.scheduledFor?.toISOString() ?? null, status: occurrence.status }));
+		} finally {
+			try { await application.close(); } finally { await database.destroy(); }
+		}
+	} finally {
+		await rm(root, { recursive: true, force: true });
+	}
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+	console.log(JSON.stringify(await runSchedulerWindow(), null, 2));
+}

package/src/scheduler/index.ts

@@ -7,3 +7,4 @@ export * from './ScheduledJob';
 export * from './ScheduledOccurrence';
 export * from './ScheduledOccurrenceRecorder';
 export * from './SchedulerWorker';
+export * from './SchedulerCheckpointRecord';

package/src/server/App.ts

@@ -1,7 +1,9 @@
+import type { WebSocketServiceOptions } from '../websocket';
 import type { Knex } from 'knex';
 import { resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { Ai, type AIOptions } from '../ai';
+import { Apps, type AppDefinitions, type AppServices, type InstalledApps } from '../apps';
 
 import {
 	Auth,
@@ -23,6 +25,12 @@ import {
 	Database,
 	type DatabaseOptions,
 } from '../db';
+import { InApp, type InAppOptions } from '../in-app';
+import { Health, type HealthOptions } from '../health';
+import { Mail, type MailOptions } from '../mail';
+import { Notifications } from '../notifications';
+import { WebSockets } from '../websocket';
+import { UserIdentity } from '../auth';
 import { Events } from '../events';
 import { UrlGenerator, type UrlGeneratorOptions } from '../url';
 import {
@@ -48,7 +56,9 @@ import {
 } from './appContext';
 
 /** Application root, service configuration and optional dependencies supplied at bootstrap. */
-export interface AppOptions {
+export interface AppOptions<TApps extends AppDefinitions = {}> {
+	/** Optional explicit composition. Omit to discover local apps and marked npm dependencies. */
+	apps?: TApps;
 	/** App root as a path or file URL; defaults to the working directory at construction. */
 	directory?: string | URL;
 	/** Provider adapters and extensible ActiveRecord models for app().ai. */
@@ -62,6 +72,14 @@ export interface AppOptions {
 	/** Canonical application URL configuration used across all runtimes. */
 	url?: UrlGeneratorOptions;
 	queue?: QueueOptions;
+	/** Authorization policy for application-scoped in-app inboxes. */
+	inApp?: InAppOptions;
+	/** Optional shared transport for channel events from API and worker processes. */
+	webSockets?: WebSocketServiceOptions;
+	/** Application health identity and initial component checks. */
+	health?: HealthOptions;
+	/** Provider-neutral mail delivery used directly and by app().notifications. */
+	mail?: MailOptions;
 	/** Serializable classes and ActiveRecord models registered with app().serializer. */
 	serializer?: SerializerOptions;
 	storage?: StorageOptions;
@@ -73,7 +91,7 @@ export interface AppOptions {
  * Keep this as the place where top-level framework capabilities are discovered
  * and wired, not where domain logic itself accumulates.
  */
-export class App {
+export class App<TApps extends AppDefinitions = {}> {
 	private readonly services = new Map<string, unknown>();
 	/** Absolute app root used for conventional server paths, independent of later working-directory changes. */
 	readonly directory: string;
@@ -83,13 +101,19 @@ export class App {
 	 *
 	 * @param options - Framework services and application configuration.
 	 */
-	constructor(protected readonly options: AppOptions = {}) {
+	constructor(protected readonly options: AppOptions<TApps> = {}) {
 		this.directory = resolve(options.directory instanceof URL ? fileURLToPath(options.directory) : options.directory ?? process.cwd());
+		if (options.apps) void this.apps;
 		setActiveApp(this);
 
 		if (this.config.has('security')) void this.security;
 	}
 
+	/** Returns the app composition registry with optional, inferred public service accessors. */
+	get apps(): Apps<TApps> & AppServices<TApps> {
+		return this.service('apps', () => new Apps(this, this.options.apps)) as Apps<TApps> & AppServices<TApps>;
+	}
+
 	/** Returns the lazy database service with this app's connection, model registry and root directory. */
 	get db(): Database {
 		return this.service('db', () => {
@@ -101,6 +125,31 @@ export class App {
 		});
 	}
 
+	/** Returns the durable inbox service using the configured Auth identity model. */
+	get inApp(): InApp {
+		return this.service('inApp', () => new InApp(this.options.inApp, this.options.auth?.identityModel ?? UserIdentity));
+	}
+
+	/** Returns the extensible application health registry. */
+	get health(): Health {
+		return this.service('health', () => new Health(this.options.health));
+	}
+
+	/** Returns the provider-neutral mail service configured for this application. */
+	get mail(): Mail {
+		return this.service('mail', () => new Mail(this.options.mail));
+	}
+
+	/** Returns the notification coordinator for application-owned channel renderers. */
+	get notifications(): Notifications<any> {
+		return this.service('notifications', () => new Notifications(this.inApp, this.mail, this.options.auth?.identityModel ?? UserIdentity));
+	}
+
+	/** Returns authenticated WebSocket endpoints owned by this application's lifecycle. */
+	get webSockets(): WebSockets {
+		return this.service('webSockets', () => new WebSockets(this, this.options.webSockets));
+	}
+
 	get auth(): Auth<any> {
 		return this.service('auth', () => new Auth<any>({
 			...this.options.auth,
@@ -148,7 +197,7 @@ export class App {
 	 * @returns Pino-backed logging service with app lifecycle ownership.
 	 */
 	get log(): Log {
-		return this.service('log', () => new Log(this.options.log));
+		return this.service('log', () => new Log(this.options.log, () => this.mail));
 	}
 
 	/**
@@ -179,7 +228,10 @@ export class App {
 	get queue(): Queue {
 		return this.service('queue', () => new Queue(
 			this.db,
-			this.options.queue,
+			{
+				onLifecycleError: failure => this.log.error({ err: failure.error, component: 'queue', source: failure.source, hook: failure.hook, jobId: failure.event.jobId, jobName: failure.event.jobName, action: failure.event.action }, 'Queue lifecycle handler failed'),
+				...this.options.queue,
+			},
 		));
 	}
 
@@ -268,18 +320,26 @@ export class App {
 	 * Closes shared resources owned by the default app.
 	 */
 	async close(): Promise<void> {
+		const errors: unknown[] = [];
+		/** Continues releasing host resources when an app or transport disposer fails. */
+		const release = async (cleanup: () => void | Promise<void>): Promise<void> => {
+			try { await cleanup(); } catch (error) { errors.push(error); }
+		};
+		await release(() => (this.services.get('apps') as Apps | undefined)?.close());
+		await release(() => (this.services.get('webSockets') as WebSockets | undefined)?.close());
 		clearActiveApp(this);
 
-		(this.services.get('scheduler') as Scheduler | undefined)?.close();
-		(this.services.get('events') as Events | undefined)?.clear();
-		await (this.services.get('cache') as Cache | undefined)?.close();
-		await (this.services.get('log') as Log | undefined)?.close();
+		await release(() => (this.services.get('scheduler') as Scheduler | undefined)?.close());
+		await release(() => (this.services.get('events') as Events | undefined)?.clear());
+		await release(() => (this.services.get('cache') as Cache | undefined)?.close());
+		await release(() => (this.services.get('log') as Log | undefined)?.close());
 
 		if (!this.options.db) {
-			await destroyDatabase();
+			await release(() => destroyDatabase());
 		}
 
 		this.services.clear();
+		if (errors.length) throw new AggregateError(errors, 'Application shutdown failed.');
 	}
 
 	private resolveDbConnection(): Knex {
@@ -324,3 +384,6 @@ function parseOptionalBoolean(input: string | undefined): boolean | undefined {
 			);
 	}
 }
+
+/** Optional discovered app services, generated from root App.ts classes by DB3 tooling. */
+export interface App<TApps extends AppDefinitions = {}> extends InstalledApps {}